diff --git a/.github/FUNDING.yml b/.github/FUNDING.yml deleted file mode 100644 index df40ac8..0000000 --- a/.github/FUNDING.yml +++ /dev/null @@ -1,11 +0,0 @@ -github: [SegoCode] -community_bridge: # Replace with a single Community Bridge project-name e.g., cloud-foundry -issuehunt: # Replace with a single IssueHunt username -ko_fi: # Replace with a single ko_fi username -liberapay: # Replace with a single Liberapay username -open_collective: # Replace with a single open_collective username -patreon: # Replace with a single Patreon username -tidelift: # Replace with a single Tidelift platform-name/package-name e.g., npm/babel -polar: # Replace with a single polar username -buy_me_a_coffee: # Replace with a single buy_me_a_coffee username -custom: # Replace with up to 4 custom sponsorship URLs e.g., ['link1', 'link2'] diff --git a/.github/ISSUE_TEMPLATE/bug_report.yml b/.github/ISSUE_TEMPLATE/bug_report.yml index 761c8d2..d039ba4 100644 --- a/.github/ISSUE_TEMPLATE/bug_report.yml +++ b/.github/ISSUE_TEMPLATE/bug_report.yml @@ -42,14 +42,21 @@ body: - type: dropdown attributes: - label: OS version + label: Platform + description: Where does the issue occur? options: - - Windows 11 - - Windows 10 - - Windows Other - - Linux Debian - - Linux Arch - - Linux Other + - Web — Chrome (Blink) + - Web — Firefox (Gecko) + - Web — Safari (WebKit) + - Web — Other + - Desktop — macOS + - Desktop — Windows + - Desktop — Linux + - Desktop — Other + - Mobile — iOS + - Mobile — Android + - Mobile — Other + - Other validations: required: true diff --git a/.github/ISSUE_TEMPLATE/config.yml b/.github/ISSUE_TEMPLATE/config.yml index b242a1b..2056bc5 100644 --- a/.github/ISSUE_TEMPLATE/config.yml +++ b/.github/ISSUE_TEMPLATE/config.yml @@ -1,5 +1,5 @@ blank_issues_enabled: false contact_links: - name: Contact the developer - url: https://SegoCode.github.io/SegoCode/ + url: https://SegoCode.github.io/SegoCode/ about: To discuss any type of related topic diff --git a/.github/workflows/do-not-merge.yml b/.github/workflows/do-not-merge.yml deleted file mode 100644 index a6c326b..0000000 --- a/.github/workflows/do-not-merge.yml +++ /dev/null @@ -1,64 +0,0 @@ -name: Do not merge - -on: - pull_request: - types: [opened, edited, labeled, unlabeled, synchronize, ready_for_review] - -jobs: - checks: - runs-on: ubuntu-latest - steps: - - name: Check if PR is ready to be merged - uses: actions/github-script@v6 - with: - script: | - const forbiddenWords = ['wip', '[wip]', 'work in progress', 'in progress', 'do not merge', 'do-not-merge', 'do_not_merge', 'dont merge', 'draft', '🚧', 'needs work']; - const pr = context.payload.pull_request; - const prTitle = pr.title.toLowerCase(); - const labels = pr.labels.map(label => label.name.toLowerCase()); - const isDraft = pr.draft; - const hasForbiddenLabel = labels.some(label => forbiddenWords.some(word => label.includes(word))); - const hasForbiddenTitle = forbiddenWords.some(word => prTitle.includes(word)); - if (isDraft || hasForbiddenLabel || hasForbiddenTitle) { - core.setFailed('The Pull Request is in draft mode or has labels or title indicating it should not be merged.'); - } else { - console.log('The Pull Request is ready to be merged.'); - } - - - name: Check if PR is from develop to main - uses: actions/github-script@v6 - with: - github-token: ${{ secrets.GITHUB_TOKEN }} - script: | - const pr = context.payload.pull_request; - const baseBranch = pr.base.ref; - const headBranch = pr.head.ref; - - if (headBranch === 'develop' && baseBranch === 'main') { - const repoFullName = context.payload.repository.full_name; - const syncWorkflowLink = `https://github.com/${repoFullName}/actions/workflows/sync-from-develop-to-main.yml`; - - const message = ` - ## ⚠️ Direct PR from develop to main detected - - This pull request is merging changes directly from \`develop\` to \`main\`. - - ### Important Note - Please consider using our [sync workflow](${syncWorkflowLink}) instead of this direct PR. - - **If you don't use the sync workflow:** - - This PR will not trigger a release - - You'll need to manually handle release processes - `; - - await github.rest.issues.createComment({ - owner: context.repo.owner, - repo: context.repo.repo, - issue_number: pr.number, - body: message - }); - - console.log('Added comment to PR from develop to main'); - } else { - console.log('PR is not from develop to main, no action needed'); - } diff --git a/.github/workflows/generate-tag.yml b/.github/workflows/generate-tag.yml index 79de8b0..7560e2c 100644 --- a/.github/workflows/generate-tag.yml +++ b/.github/workflows/generate-tag.yml @@ -1,98 +1,63 @@ name: Generate tag -on: +on: pull_request: types: [closed] +concurrency: + group: generate-tag + jobs: create_tag: - if: github.event.pull_request.merged == true && contains(github.event.pull_request.labels.*.name, 'auto-tag') + if: github.event.pull_request.merged == true && github.event.pull_request.base.ref == 'main' && contains(github.event.pull_request.labels.*.name, 'auto-tag') runs-on: ubuntu-latest + permissions: + contents: write + outputs: + new_version: ${{ steps.determine_version.outputs.new_version }} steps: - name: Checkout repository - uses: actions/checkout@v4 + uses: actions/checkout@v7 with: - fetch-depth: 0 # Fetch all history for tag retrieval + fetch-depth: 0 - name: Set up git run: | - git config --global user.name "github-actions" - git config --global user.email "github-actions@github.com" - - - name: Fetch all tags - run: git fetch --tags - - - name: Get latest tag - id: get_latest_tag - run: | - # Get the latest tag - latest_tag=$(git describe --tags `git rev-list --tags --max-count=1` 2>/dev/null || echo "") - echo "latest_tag=$latest_tag" >> $GITHUB_ENV + git config user.name "github-actions[bot]" + git config user.email "41898282+github-actions[bot]@users.noreply.github.com" - name: Determine new version id: determine_version run: | - latest_tag=${{ env.latest_tag }} + latest_tag=$(git tag --list --sort=-version:refname | grep -m1 -E '^[0-9]+\.[0-9]+$' || true) if [ -z "$latest_tag" ]; then - # Initialize the version to 1.0 if no tags exist new_version="1.0" else - # Extract the major and minor version and increment the minor version - major_version=$(echo $latest_tag | cut -d. -f1) - minor_version=$(echo $latest_tag | cut -d. -f2) - new_minor_version=$((minor_version + 1)) - new_version="$major_version.$new_minor_version" - - # Check if the new version tag already exists - while git rev-parse "refs/tags/$new_version" >/dev/null 2>&1; do - new_minor_version=$((new_minor_version + 1)) - new_version="$major_version.$new_minor_version" - done + IFS=. read -r major minor <<< "$latest_tag" + if [[ "${{ contains(github.event.pull_request.labels.*.name, 'semver:major') }}" == "true" ]]; then + new_version="$((major + 1)).0" + else + new_version="$major.$((minor + 1))" + fi fi - echo "new_version=$new_version" >> $GITHUB_ENV - - - name: Checkout main branch - run: | - git checkout main + echo "new_version=$new_version" >> "$GITHUB_OUTPUT" - name: Create new tag - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | - new_version=${{ env.new_version }} - git tag -a $new_version -m "Automatically generated version $new_version" - git push origin $new_version + new_version="${{ steps.determine_version.outputs.new_version }}" + git tag -a "$new_version" -m "Release $new_version" "${{ github.event.pull_request.merge_commit_sha }}" + git push origin "$new_version" create_issue: needs: create_tag runs-on: ubuntu-latest + permissions: + issues: write steps: - - name: Checkout repository - uses: actions/checkout@v4 - with: - fetch-depth: 0 # Fetch all history for tag retrieval - - - name: Set up git - run: | - git config --global user.name "github-actions" - git config --global user.email "github-actions@github.com" - - - name: Fetch all tags - run: git fetch --tags - - - name: Get the latest tag - id: get_latest_tag - run: | - # Get the latest tag - latest_tag=$(git describe --tags `git rev-list --tags --max-count=1` 2>/dev/null || echo "") - echo "latest_tag=$latest_tag" >> $GITHUB_ENV - - name: Create issue for new tag env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - latest_tag: ${{ env.latest_tag }} + GH_TOKEN: ${{ github.token }} run: | - repository=${{ github.repository }} - issue_title="The tag \`${{ env.latest_tag }}\` was created" - issue_body=$'The **${{ env.latest_tag }}** tag for the **main branch** has been created. Please consider creating a release of this tag, if a release isn\'t needed, you can close this issue.\n\n[Click here to create a release of **${{ env.latest_tag }}** tag](../releases/new?tag=${{ env.latest_tag }})' - gh issue create --title "$issue_title" --body "$issue_body" --label "auto-tag" + issue_title="The tag \`${{ needs.create_tag.outputs.new_version }}\` was created" + issue_body=$'The **${{ needs.create_tag.outputs.new_version }}** tag for the **main branch** has been created.\nPlease consider creating a release of this tag, if a release isn\'t needed, you can close this issue.\n\n[Click here to create a release of **${{ needs.create_tag.outputs.new_version }}** tag](../releases/new?tag=${{ needs.create_tag.outputs.new_version }})' + gh issue create --repo "${{ github.repository }}" --title "$issue_title" --body "$issue_body" --label auto-tag diff --git a/.github/workflows/gitleaks.yml b/.github/workflows/gitleaks.yml index f53dea4..f16f194 100644 --- a/.github/workflows/gitleaks.yml +++ b/.github/workflows/gitleaks.yml @@ -1,46 +1,42 @@ name: Gitleaks on: + push: pull_request: workflow_dispatch: - schedule: - # Run at 00:00 UTC on the first day of each month - - cron: '0 0 1 * *' jobs: scan: name: Gitleaks Scan runs-on: ubuntu-latest - # Only run scheduled jobs on main branch - if: github.event_name != 'schedule' || github.ref == 'refs/heads/main' + permissions: + contents: read + issues: write steps: - name: Checkout code - uses: actions/checkout@v4 + uses: actions/checkout@v7 with: fetch-depth: 0 - name: Run Gitleaks id: gitleaks - uses: gitleaks/gitleaks-action@v2 + uses: gitleaks/gitleaks-action@v3 env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GITHUB_TOKEN: ${{ github.token }} continue-on-error: true - name: Create issue if leaks found if: steps.gitleaks.outcome == 'failure' && (github.ref == 'refs/heads/main' || github.event_name == 'workflow_dispatch') env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GH_TOKEN: ${{ github.token }} run: | echo "Creating detailed security issue report..." - # Get current timestamp TIMESTAMP=$(date -u +"%Y-%m-%d %H:%M:%S UTC") - # Create an enriched issue body ISSUE_BODY="## 🚨 Security Alert: Potential Secrets Detected **Detection Time:** ${TIMESTAMP} **Branch:** ${GITHUB_REF#refs/heads/} - **Triggered by:** ${{ github.event_name == 'workflow_dispatch' && 'Manual workflow run' || (github.event_name == 'schedule' && 'Monthly scheduled scan' || 'Automated scan') }} **Detected by:** Gitleaks Security Scanner ### Details @@ -55,13 +51,14 @@ jobs: 3. Remove the secrets from the codebase 4. Consider using GitHub Secrets or environment variables instead - ### Contact - Please reach out to the security team for assistance if needed. - --- *This issue was automatically generated by the Gitleaks security scanning workflow.*" gh issue create \ - --repo ${{ github.repository }} \ - --title "Security Alert: Potential secrets detected in ${{ github.event_name == 'pull_request' && format('PR #{0}', github.event.pull_request.number) || 'main branch' }}" \ + --repo "${{ github.repository }}" \ + --title "Security Alert: Potential secrets detected in main branch" \ --body "${ISSUE_BODY}" + + - name: Fail if leaks were found + if: steps.gitleaks.outcome == 'failure' + run: exit 1 diff --git a/.github/workflows/greetings.yml b/.github/workflows/greetings.yml index 724954b..2450362 100644 --- a/.github/workflows/greetings.yml +++ b/.github/workflows/greetings.yml @@ -9,8 +9,8 @@ jobs: issues: write pull-requests: write steps: - - uses: actions/first-interaction@v1 + - uses: actions/first-interaction@v3.1.0 with: - repo-token: ${{ secrets.GITHUB_TOKEN }} - issue-message: "Thank you for your first issue. To better understand your request or the problem you've encountered, please provide as many details as possible. If the behavior changes or if you have new information about your request, don't hesitate to add it. It will be reviewed ASAP." - pr-message: "Thank you for your first pull request to the repository! We're grateful for your contribution and will review it ASAP." + repo_token: ${{ secrets.GITHUB_TOKEN }} + issue_message: "Thank you for your first issue. To better understand your request or the problem you've encountered, please provide as many details as possible. If the behavior changes or if you have new information about your request, don't hesitate to add it. It will be reviewed ASAP." + pr_message: "Thank you for your first pull request to the repository! We're grateful for your contribution and will review it ASAP." diff --git a/.github/workflows/java-verify.yml b/.github/workflows/java-verify.yml new file mode 100644 index 0000000..ce90609 --- /dev/null +++ b/.github/workflows/java-verify.yml @@ -0,0 +1,33 @@ +name: Java verify + +on: + pull_request: + +permissions: + contents: read + +concurrency: + group: java-verify-${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + verify: + name: Compile and test + runs-on: ubuntu-latest + defaults: + run: + working-directory: code + steps: + - name: Check out repository + uses: actions/checkout@v5 + + - name: Set up Java + uses: actions/setup-java@v5 + with: + distribution: temurin + java-version: '21' + cache: maven + cache-dependency-path: code/pom.xml + + - name: Verify Maven project + run: mvn --batch-mode --no-transfer-progress verify diff --git a/.github/workflows/sync-from-develop-to-main.yml b/.github/workflows/sync-from-develop-to-main.yml index 7baa0b8..d684b83 100644 --- a/.github/workflows/sync-from-develop-to-main.yml +++ b/.github/workflows/sync-from-develop-to-main.yml @@ -1,52 +1,68 @@ name: Sync from develop to main -on: +on: workflow_dispatch: + inputs: + release: + description: Version to publish when the pull request is merged + required: true + default: minor + type: choice + options: + - minor + - major + - none + +concurrency: + group: promote-develop-to-main jobs: create_pr: runs-on: ubuntu-latest + permissions: + contents: read + pull-requests: write + issues: write steps: - - name: Checkout repository - uses: actions/checkout@v2 - - - name: Setup git - run: | - git config --global user.name "github-actions" - git config --global user.email "github-actions@github.com" - - - name: Install GitHub CLI - run: | - sudo apt-get update - sudo apt-get install gh -y - - name: Check and create labels env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GH_TOKEN: ${{ github.token }} run: | - # Check if the 'auto-sync' label exists - AUTO_SYNC_LABEL=$(gh api repos/${{ github.repository }}/labels --jq '.[] | select(.name=="auto-sync")') - if [ -z "$AUTO_SYNC_LABEL" ]; then - # Create the 'auto-sync' label if it doesn't exist - gh api repos/${{ github.repository }}/labels -f name='auto-sync' -f color='C5DEF5' - fi - - # Check if the 'auto-tag' label exists - AUTO_TAG_LABEL=$(gh api repos/${{ github.repository }}/labels --jq '.[] | select(.name=="auto-tag")') - if [ -z "$AUTO_TAG_LABEL" ]; then - # Create the 'auto-tag' label if it doesn't exist - gh api repos/${{ github.repository }}/labels -f name='auto-tag' -f color='BFDADC' - fi - - - name: Create pull request + gh label create auto-sync --repo "${{ github.repository }}" --color C5DEF5 --force + gh label create auto-tag --repo "${{ github.repository }}" --color BFDADC --force + gh label create semver:major --repo "${{ github.repository }}" --color B60205 --force + + - name: Replace pull request env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GH_TOKEN: ${{ github.token }} run: | - # Create the pull request - PR_URL=$(gh pr create --base main --head develop --title "Sync from develop to main" --body $'This is an automated pull request to sync changes from the develop branch to the main branch. \n >[!WARNING]\n> If this pull request is merged with the \'**auto-tag**\' label, it will create a **new tag**. To prevent this, manually remove the \'**auto-tag**\' label.') - - # Extract PR number from URL - PR_NUMBER=$(basename $PR_URL) - - # Add the labels to the pull request - gh pr edit $PR_NUMBER --add-label "auto-sync" --add-label "auto-tag" + PR_BODY="> [!WARNING] + > This is an automated pull request to sync changes from the \`develop\` branch to the \`main\` branch. + > If this pull request is merged with the **auto-tag** label, it will create a **new tag**. + > To prevent this, manually remove the **auto-tag** label." + + gh pr list --repo "${{ github.repository }}" --base main --head develop --state open \ + --json number --jq '.[].number' | while read -r PR_NUMBER; do + gh pr close "$PR_NUMBER" --repo "${{ github.repository }}" + done + + PR_NUMBER=$(gh pr create --repo "${{ github.repository }}" --base main --head develop \ + --title "chore: promote develop to main" \ + --body "$PR_BODY" | sed 's#.*/##') + + gh pr edit "$PR_NUMBER" --repo "${{ github.repository }}" --add-label auto-sync + case "${{ inputs.release }}" in + major) + gh pr edit "$PR_NUMBER" --repo "${{ github.repository }}" --add-label auto-tag --add-label semver:major + ;; + minor) + gh pr edit "$PR_NUMBER" --repo "${{ github.repository }}" --remove-label semver:major 2>/dev/null || true + gh pr edit "$PR_NUMBER" --repo "${{ github.repository }}" --add-label auto-tag + ;; + none) + gh pr edit "$PR_NUMBER" --repo "${{ github.repository }}" --remove-label auto-tag 2>/dev/null || true + gh pr edit "$PR_NUMBER" --repo "${{ github.repository }}" --remove-label semver:major 2>/dev/null || true + ;; + esac + + gh pr view "$PR_NUMBER" --repo "${{ github.repository }}" --json url --jq .url diff --git a/.github/workflows/sync-from-main-to-develop.yml b/.github/workflows/sync-from-main-to-develop.yml index 7d43073..b17a2b9 100644 --- a/.github/workflows/sync-from-main-to-develop.yml +++ b/.github/workflows/sync-from-main-to-develop.yml @@ -5,28 +5,33 @@ on: branches: - main +concurrency: + group: sync-main-to-develop + jobs: sync_main_into_develop: runs-on: ubuntu-latest permissions: - contents: write + contents: write steps: - - name: Checkout develop branch - uses: actions/checkout@v4 + - name: Checkout repository + uses: actions/checkout@v7 with: - ref: develop + ref: main fetch-depth: 0 - name: Configure Git run: | - git config --global user.name "github-actions" - git config --global user.email "github-actions@github.com" + git config user.name "github-actions[bot]" + git config user.email "41898282+github-actions[bot]@users.noreply.github.com" - name: Merge main into develop run: | - git fetch origin main - git merge origin/main --no-edit - - - name: Push changes to develop - run: | + if ! git ls-remote --exit-code --heads origin develop; then + echo "develop does not exist yet; initialization will create it." + exit 0 + fi + git fetch origin develop + git checkout -B develop origin/develop + git merge "$GITHUB_SHA" --no-edit git push origin develop diff --git a/.github/workflows/update-license.yml b/.github/workflows/update-license.yml deleted file mode 100644 index 43f5ac4..0000000 --- a/.github/workflows/update-license.yml +++ /dev/null @@ -1,59 +0,0 @@ -name: Update license - -on: - workflow_dispatch: - schedule: - # Run at 00:00 UTC on the first day of each month - - cron: '0 0 1 * *' - -jobs: - update-license: - runs-on: ubuntu-latest - permissions: - contents: write - - steps: - - name: Checkout repository - uses: actions/checkout@v4 - with: - ref: develop - - - name: Setup git - run: | - git config --global user.name "github-actions" - git config --global user.email "github-actions@github.com" - - - name: Extract repository and username - id: extract - run: | - REPO_NAME="${{ github.repository }}" - USERNAME=$(echo $REPO_NAME | cut -d'/' -f1) - REPO_NAME_ONLY=$(echo $REPO_NAME | cut -d'/' -f2) - echo "username=$USERNAME" >> $GITHUB_OUTPUT - echo "reponame=$REPO_NAME_ONLY" >> $GITHUB_OUTPUT - - - name: Download LICENSE template - run: | - curl -s https://raw.githubusercontent.com/SegoCode/template/main/LICENSE -o LICENSE - - - name: Replace placeholders in LICENSE file - run: | - REPO_NAME_ONLY="${{ steps.extract.outputs.reponame }}" - USERNAME="${{ steps.extract.outputs.username }}" - REPO_NAME_ESCAPED=$(echo $REPO_NAME_ONLY | sed 's/\//\\\//g') - USERNAME_ESCAPED=$(echo $USERNAME | sed 's/\//\\\//g') - - # Replace placeholders with actual values - sed -i "s/{reponame}/$REPO_NAME_ESCAPED/g" LICENSE - sed -i "s/{username}/$USERNAME_ESCAPED/g" LICENSE - - - name: Commit and push changes - run: | - # Add LICENSE file - git add LICENSE - - # Only commit if there are changes - git diff --staged --quiet || git commit -m "Update LICENSE with repository information" - - # Push directly to develop branch - git push origin develop diff --git a/.github/workflows/warn-direct-pr.yml b/.github/workflows/warn-direct-pr.yml new file mode 100644 index 0000000..8d4c155 --- /dev/null +++ b/.github/workflows/warn-direct-pr.yml @@ -0,0 +1,61 @@ +name: Warn direct PR + +on: + pull_request_target: + types: [opened] + +concurrency: + group: warn-manual-develop-to-main-${{ github.event.pull_request.number }} + +jobs: + warn: + runs-on: ubuntu-latest + permissions: + issues: write + pull-requests: write + steps: + - name: Warn about manual promotion PR + uses: actions/github-script@v9 + with: + script: | + const pr = context.payload.pull_request; + const isPromotion = pr.base.ref === 'main' && pr.head.ref === 'develop'; + const isAutomated = pr.user.type === 'Bot'; + const hasAutoSync = pr.labels.some(({ name }) => name.toLowerCase() === 'auto-sync'); + + if (!isPromotion || isAutomated || hasAutoSync) { + return; + } + + const marker = ''; + const comments = await github.paginate(github.rest.issues.listComments, { + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: pr.number, + per_page: 100 + }); + + if (comments.some(({ body }) => body?.includes(marker))) { + return; + } + + const workflowUrl = `${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/workflows/sync-from-develop-to-main.yml`; + const body = [ + marker, + '> [!WARNING]', + '> This pull request was opened manually from `develop` to `main` without the `auto-sync` label.', + '>', + `> The [Sync from develop to main workflow](${workflowUrl}) will close this PR and create a new promotion PR.`, + '> Select the release type:', + '>', + '> - `minor`: create the next minor tag.', + '> - `major`: create the next major tag.', + '> - `none`: merge without creating a tag.' + ].join('\n'); + + await github.rest.issues.createComment({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: pr.number, + body + }); diff --git a/.gitignore b/.gitignore index 8c11a6b..4f9b440 100644 --- a/.gitignore +++ b/.gitignore @@ -1,8 +1,8 @@ ### Custom ### code/.idea/* -code/storage/* code/target/* +code/downloads/* # Created by https://www.toptal.com/developers/gitignore/api/git,gpg,ssh,vim,linux,macos,windows,notepadpp,sublimetext,intellij+all,visualstudiocode,dotenv # Edit at https://www.toptal.com/developers/gitignore?templates=git,gpg,ssh,vim,linux,macos,windows,notepadpp,sublimetext,intellij+all,visualstudiocode,dotenv diff --git a/LICENSE b/LICENSE index 6ab3d8e..1b2ebe6 100644 --- a/LICENSE +++ b/LICENSE @@ -1,75 +1,227 @@ -Copyright (c) SegoCode -All rights reserved. - -Section 1 - Definitions - -1.1 "NonCommercial" pertains to any use, distribution, or modification of the - licensed material that does not primarily aim to achieve commercial - advantage or generate monetary compensation. This includes, but is not - limited to, activities such as distributing the licensed material as part - of an application or product that is sold, using the licensed material in - advertising, and creating products or services with the licensed material - that are subsequently sold. - -1.2 "Adapted Material" refers to any work derived from or based upon the licensed - material. This includes, but is not limited to, translations, alterations, - rearrangements, transformations, source code modifications, compiled code - alterations, architectural redesigns, or the integration of the licensed - material into other software projects. - -1.3 "NonAdapted Material" refers to exact copies of the licensed material, either - in source code or binary form, which are reproduced without any changes, - modifications, or transformations. - -Section 2 - License Conditions - -2.1 Distribution and usage in source or binary forms are permitted solely for - NonCommercial purposes for both NonAdapted Material and Adapted Material - excluding NonAdapted Material cases in 2.4 section. - -2.2 Redistributions for any NonAdapted Material or Adapted Material, either - in source code or binary form, must include the original copyright notice, - this license, the disclaimer, and comply with the requirements specified - herein. For binary form redistributions, these documents must be included - in any provided documentation or materials or a clearly accessible link - to this license ensuring that recipients can easily review the license terms. - -2.3 For any Adapted Material, whether in source code or binary form, and for any - instance of the licensed material utilized in applications accessible over - the internet that run on a server, the source code must be made accessible - through a public repository, a downloadable archive, or an equivalent - method, ensuring that users have the capability to access, review, and - download the code, and must clearly credit the original work and author. - -2.4 For any NonAdapted Material that are offered on download sites, marketplaces, or - software distribution platforms, are permitted under the condition that any - economic benefit derived from such distribution is strictly indirect, including - but not limited to advertisements or link redirectors. Direct sales or charges - for access to the licensed material are not permitted under this license. Upon - the original author distributing the material on the same platform, all alternative - downloads and any associated revenue-generating mechanisms must be discontinued - immediately. Acceptance of this license constitutes agreement that the copyright - holder may request the immediate cessation of such downloads and activities on - such platforms, and the distributor must comply with such request without delay. - -2.5 Distributing the licensed material alongside unrelated or harmful software, such as adware, - malware, or spyware or implying endorsement or association with the original author - or recognized entities without permission, is prohibited. - -Section 3 - Updates and Revisions - -3.1 The copyright holder retains the right to amend, update, or otherwise modify this - license at any time without prior notice. Users should periodically review the - license terms to stay informed of any changes. - -3.2 Continued use of the licensed material after such changes signifies acceptance of the - revised license terms. It is the user's responsibility to ensure ongoing compliance - with the most current version of the license. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. +# Source Available PolyForm Noncommercial + GNU AGPL-3.0 License + + + +## Acceptance + +In order to get any license under these terms, you must agree +to them as both strict obligations and conditions to all +your licenses. + +## Copyright License + +The licensor grants you a copyright license for the +software to do everything you might do with the software +that would otherwise infringe the licensor's copyright +in it for any permitted purpose. However, you may +only distribute the software according to [Distribution +License](#distribution-license) and make changes or new works +based on the software according to [Changes and New Works +License](#changes-and-new-works-license), subject to +[Source Availability Obligations](#source-availability-obligations). + +## Distribution License + +The licensor grants you an additional copyright license +to distribute copies of the software. Your license +to distribute covers distributing the software with +changes and new works permitted by [Changes and New Works +License](#changes-and-new-works-license), subject to +[Source Availability Obligations](#source-availability-obligations). + +## Notices + +You must ensure that anyone who gets a copy of any part of +the software from you also gets a copy of these terms or the +URL for them above, as well as copies of any plain-text lines +beginning with `Required Notice:` that the licensor provided +with the software. For example: + +> Required Notice: Copyright [SegoCode] ([https://github.com/SegoCode/webdl]) + +## Changes and New Works License + +The licensor grants you an additional copyright license to +make changes and new works based on the software for any +permitted purpose, subject to +[Source Availability Obligations](#source-availability-obligations). + +## Patent License + +The licensor grants you a patent license for the software that +covers patent claims the licensor can license, or becomes able +to license, that you would infringe by using the software. + +## Noncommercial Purposes + +Any noncommercial purpose is a permitted purpose. + +## Personal Uses + +Personal use for research, experiment, and testing for +the benefit of public knowledge, personal study, private +entertainment, hobby projects, amateur pursuits, or religious +observance, without any anticipated commercial application, +is use for a permitted purpose. + +## Noncommercial Organizations + +Use by any charitable organization, educational institution, +public research organization, public safety or health +organization, environmental protection organization, +or government institution is use for a permitted purpose +regardless of the source of funding or obligations resulting +from the funding. + +## Fair Use + +You may have "fair use" rights for the software under the +law. These terms do not limit them. + +## No Other Rights + +These terms do not allow you to sublicense or transfer any of +your licenses to anyone else, or prevent the licensor from +granting licenses to anyone else. These terms do not imply +any other licenses. + +## Patent Defense + +If you make any written claim that the software infringes or +contributes to infringement of any patent, your patent license +for the software granted under these terms ends immediately. If +your company makes such a claim, your patent license ends +immediately for work on behalf of your company. + +## Source Availability Obligations + +These obligations apply in addition to all other terms of this +license and are conditions of every license granted hereunder. + +### Definitions + +“Source Code” means the preferred form of the work for making +modifications to it. + +“Object Code” means any non-source form of a work. + +“Corresponding Source” means all the Source Code needed to +generate, install, and (for an executable work) run the Object +Code and to modify the work, including scripts to control those +activities. However, it does not include the work’s System +Libraries, or general-purpose tools or generally available free +programs which are used unmodified in performing those activities +but which are not part of the work. The Corresponding Source for +a work in Source Code form is that same work. + +“Convey” means any kind of propagation that enables other parties +to make or receive copies. Mere interaction with a user through a +computer network, with no transfer of a copy, is not conveying. + +“Make Available as a Network Service” means enabling third parties +to interact with the functionality of the software or a modified +version remotely through a computer network, or offering a service +whose value entirely or primarily derives from the software or a +modified version, or that accomplishes for users the primary purpose +of the software or a modified version. + +### Obligation when Conveying + +If you Convey the software or any modified version (whether in +Source Code or Object Code form), you must: + +- license the entire work, as a whole, under these terms to anyone + who comes into possession of a copy; +- provide the Corresponding Source of the work under these same + terms, either accompanying the Object Code on a durable physical + medium customarily used for software interchange, or offering + equivalent access to the Corresponding Source from a network + server at no charge, with clear directions next to the Object + Code stating where to find it, and keeping that access available + for as long as needed to satisfy these requirements; and +- ensure that any interactive user interfaces display Appropriate + Legal Notices (copyright notice, absence of warranty, and how to + view a copy of these terms). + +### Obligation for Network Interaction and Services + +If you modify the software and Make Available as a Network Service +a version that supports remote interaction, or if you Make Available +as a Network Service the software or any modified version, you must +prominently offer all users interacting with it remotely an +opportunity to receive the Corresponding Source of your version by +providing access to that Corresponding Source from a network server +at no charge, through some standard or customary means of +facilitating copying of software. This obligation applies even if +you do not Convey copies. + +### Same License and Notices + +Any Corresponding Source you provide under these obligations must +itself be licensed under these terms and must include all Required +Notices and a copy of (or URL to) these terms. + +## Limited Control over Third-Party Hosting + +The licensor may at any time, by any reasonable means (including +direct notification to the third-party site, public announcement on +the licensor’s website, repository, social media, or any other public +communication), make known that distribution of the software or any +modified version on a particular third-party website, platform, +repository, mirror, or download service is not authorized under these +terms. + +Once the licensor has made such communication in a manner reasonably +capable of coming to the attention of the public or of the operators +of the site, any person who has uploaded, instructed the upload of, +hosts, or continues to distribute the software or a modified version +on that site must promptly cease the distribution and remove all +copies under their control. + +Failure to cease distribution and remove the copies after the +licensor’s communication has become reasonably known constitutes a +violation of these terms. + +This obligation applies only to copies that the person controls or +that they caused to be placed on the site. It does not impose liability +for actions of independent third parties beyond the person’s control. + +## Violations + +The first time you are notified in writing that you have +violated any of these terms, or done anything with the software +not covered by your licenses, your licenses can nonetheless +continue if you come into full compliance with these terms, +and take practical steps to correct past violations, within +32 days of receiving notice. Otherwise, all your licenses +end immediately. + +## No Liability + +***As far as the law allows, the software comes as is, without +any warranty or condition, and the licensor will not be liable +to you for any damages arising out of these terms or the use +or nature of the software, under any kind of legal claim.*** + +## Definitions + +The **licensor** is the individual or entity offering these +terms, and the **software** is the software the licensor makes +available under these terms. + +**You** refers to the individual or entity agreeing to these +terms. + +**Your company** is any legal entity, sole proprietorship, +or other kind of organization that you work for, plus all +organizations that have control over, are under the control of, +or are under common control with that organization. **Control** +means ownership of substantially all the assets of an entity, +or the power to direct its management and policies by vote, +contract, or otherwise. Control can be direct or indirect. + +**Your licenses** are all the licenses granted to you for the +software under these terms. + +**Use** means anything you do with the software requiring one +of your licenses. diff --git a/README.md b/README.md index 2f64978..8c97ee1 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ -# webdl +

- +

About • @@ -8,27 +8,31 @@ Quick Start & Information

- ## About -Telegram bot in Java for downloading social media videos using yt-dlp +[![Top language](https://img.shields.io/github/languages/top/SegoCode/webdl?style=flat-square)](https://github.com/SegoCode/webdl) +[![Repository size](https://img.shields.io/github/repo-size/SegoCode/webdl?style=flat-square&label=repo%20size)](https://github.com/SegoCode/webdl) +[![Commit activity per year](https://img.shields.io/github/commit-activity/y/SegoCode/webdl?style=flat-square&label=commits)](https://github.com/SegoCode/webdl/graphs/commit-activity) +[![Licencia: PolyForm Noncommercial + GNU AGPL-3.0](https://img.shields.io/badge/License-PolyForm%20Noncommercial%20%2B%20GNU%20AGPL--3.0-blue?style=flat-square)](https://github.com/SegoCode/webdl/blob/main/LICENSE) +[![Bitcoin BTC](https://img.shields.io/badge/buy_me_a_coffee-BTC-F7931A?style=flat-square&logo=bitcoin&logoColor=white)](https://github.com/SegoCode/SegoCode/discussions/2) + + +Telegram bot in Java for downloading social media videos using [yt-dlp](https://github.com/yt-dlp/yt-dlp). Send a video URL, get the file back as a Telegram video message. ## Features - Non-blocking message queue processing with virtual threads -- Dynamic interaction with Telegram messages (send, delete, edit) - -- Web panel with usage statistics on port 8080 +- Dynamic interaction with Telegram messages (send and delete) - Automatic retry on download failures ## Quick Start & Information -Webdl accepts a video URL, downloads it using [yt-dlp](https://github.com/yt-dlp/yt-dlp), and sends it back to the user as a video message. +Requires Java 21, Maven, and [yt-dlp](https://github.com/yt-dlp/yt-dlp) available on `PATH`. Set `BOT_TOKEN` to your Telegram bot token. ### From source -``` +```shell git clone https://github.com/SegoCode/webdl cd webdl/code mvn clean package -DskipTests @@ -37,7 +41,7 @@ java -jar target/webdl.jar ### Docker -``` +```shell cd webdl/code mvn clean package -DskipTests docker build -t webdl-image . @@ -45,32 +49,9 @@ docker run -d \ --name webdl \ --restart unless-stopped \ -e BOT_TOKEN=your-bot-token \ - -p 8080:8080 \ - -v /mnt/drive/data/webdl:/downloads \ webdl-image ``` -### Project structure - -``` -code/src/main/java/org/segocode/webdl/ -├── Main.java # Entry point -├── bot/ -│ ├── Webdlbot.java # Telegram long-polling bot -│ ├── constants/Messages.java # User-facing message strings -│ ├── model/{User,DataRootContainer}.java # EclipseStore persistence -│ ├── service/{MessageService,VideoService}.java -│ └── util/MessageUtil.java -├── panel/ -│ ├── PanelApplication.java # Javalin web server bootstrap -│ └── AdminController.java # Admin panel route handler -└── system/ - ├── command/CommandExecutor.java # yt-dlp subprocess with retry - └── util/FileUtil.java -``` - - - ---

diff --git a/code/Dockerfile b/code/Dockerfile index 63189a7..bd1111b 100644 --- a/code/Dockerfile +++ b/code/Dockerfile @@ -15,5 +15,6 @@ RUN apk add --no-cache ffmpeg python3 && \ chmod a+rx /usr/local/bin/yt-dlp WORKDIR /app +RUN mkdir downloads COPY --from=build /app/target/webdl.jar /app/webdl.jar ENTRYPOINT ["java", "-jar", "/app/webdl.jar"] diff --git a/code/pom.xml b/code/pom.xml index b4c1b7d..8b8c7a8 100644 --- a/code/pom.xml +++ b/code/pom.xml @@ -9,8 +9,7 @@ 1.0 - 21 - 21 + 21 UTF-8 @@ -26,31 +25,21 @@ 1.4.14 - org.projectlombok - lombok - 1.18.36 - provided - - - org.eclipse.store - storage-embedded - 2.1.2 - - - io.javalin - javalin - 6.5.0 - - - com.google.code.gson - gson - 2.12.1 + org.junit.jupiter + junit-jupiter + 6.1.2 + test webdl + + org.apache.maven.plugins + maven-surefire-plugin + 3.5.4 + org.apache.maven.plugins maven-shade-plugin @@ -62,6 +51,7 @@ shade + false diff --git a/code/src/main/java/org/segocode/webdl/Main.java b/code/src/main/java/org/segocode/webdl/Main.java index a20f823..76344cd 100644 --- a/code/src/main/java/org/segocode/webdl/Main.java +++ b/code/src/main/java/org/segocode/webdl/Main.java @@ -1,12 +1,14 @@ package org.segocode.webdl; -import org.eclipse.store.storage.embedded.types.EmbeddedStorage; -import org.eclipse.store.storage.embedded.types.EmbeddedStorageManager; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; import org.segocode.webdl.bot.Webdlbot; -import org.segocode.webdl.panel.PanelApplication; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.telegram.telegrambots.meta.TelegramBotsApi; +import org.telegram.telegrambots.meta.api.methods.GetMe; +import org.telegram.telegrambots.meta.generics.BotSession; import org.telegram.telegrambots.updatesreceivers.DefaultBotSession; public class Main { @@ -14,16 +16,40 @@ public class Main { public static void main(String[] args) { try { - LOGGER.info("Starting storage manager..."); - final EmbeddedStorageManager storageManager = EmbeddedStorage.start(); TelegramBotsApi botsApi = new TelegramBotsApi(DefaultBotSession.class); LOGGER.info("Starting the video download bot..."); - botsApi.registerBot(new Webdlbot(storageManager)); + Webdlbot bot = new Webdlbot(); + BotSession session = botsApi.registerBot(bot); + startWatchdog(bot, session); LOGGER.info("Bot started successfully and ready to download videos 🚀"); - LOGGER.info("Starting webp anel app..."); - PanelApplication.start(storageManager); } catch (Exception e) { LOGGER.error("Error while attempting to start the bot. Error details:", e); + throw new IllegalStateException("Unable to start the bot", e); } } + + private static void startWatchdog(Webdlbot bot, BotSession session) { + AtomicInteger failures = new AtomicInteger(); + Executors.newSingleThreadScheduledExecutor(Thread.ofVirtual().factory()) + .scheduleWithFixedDelay( + () -> { + try { + if (!session.isRunning()) { + throw new IllegalStateException("Telegram bot session stopped"); + } + bot.execute(new GetMe()); + failures.set(0); + } catch (Exception e) { + int currentFailures = failures.incrementAndGet(); + LOGGER.warn("Telegram health check failed ({}/3): {}", currentFailures, e.getMessage()); + if (currentFailures >= 3) { + LOGGER.error("Telegram connection is unhealthy; exiting for Docker restart"); + System.exit(1); + } + } + }, + 1, + 1, + TimeUnit.MINUTES); + } } diff --git a/code/src/main/java/org/segocode/webdl/bot/DownloadQueue.java b/code/src/main/java/org/segocode/webdl/bot/DownloadQueue.java new file mode 100644 index 0000000..ea04fa1 --- /dev/null +++ b/code/src/main/java/org/segocode/webdl/bot/DownloadQueue.java @@ -0,0 +1,28 @@ +package org.segocode.webdl.bot; + +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; + +final class DownloadQueue implements AutoCloseable { + private final ThreadPoolExecutor executor = new ThreadPoolExecutor( + 1, + 1, + 0L, + TimeUnit.MILLISECONDS, + new LinkedBlockingQueue<>(), + Thread.ofVirtual().factory()); + + int pendingTasks() { + return executor.getActiveCount() + executor.getQueue().size(); + } + + void submit(Runnable task) { + executor.submit(task); + } + + @Override + public void close() { + executor.shutdownNow(); + } +} diff --git a/code/src/main/java/org/segocode/webdl/bot/Webdlbot.java b/code/src/main/java/org/segocode/webdl/bot/Webdlbot.java index 7f77268..0830576 100644 --- a/code/src/main/java/org/segocode/webdl/bot/Webdlbot.java +++ b/code/src/main/java/org/segocode/webdl/bot/Webdlbot.java @@ -4,12 +4,6 @@ import static org.segocode.webdl.bot.util.MessageUtil.*; import static org.segocode.webdl.system.util.FileUtil.*; -import java.time.LocalDateTime; -import java.util.Optional; -import java.util.concurrent.*; -import org.eclipse.store.storage.types.StorageManager; -import org.segocode.webdl.bot.model.DataRootContainer; -import org.segocode.webdl.bot.model.User; import org.segocode.webdl.bot.service.MessageService; import org.segocode.webdl.bot.service.VideoService; import org.segocode.webdl.system.command.CommandExecutor; @@ -24,33 +18,7 @@ public class Webdlbot extends TelegramLongPollingBot { private static final Logger LOGGER = LoggerFactory.getLogger(Webdlbot.class); private static final String BOT_TOKEN = System.getenv("BOT_TOKEN"); - private final StorageManager storageManager; - - // Create a ThreadPoolExecutor with a single virtual thread - private final ThreadPoolExecutor executorService = new ThreadPoolExecutor( - 1, // corePoolSize: the number of threads to keep in the pool, even if they are idle - 1, // maximumPoolSize: the maximum number of threads to allow in the pool - 0L, // keepAliveTime: when the number of threads is greater than the core - TimeUnit.MILLISECONDS, // the time unit for the keepAliveTime argument - new LinkedBlockingQueue<>(), // the queue to use for holding tasks before they are executed - Thread.ofVirtual().factory() // the factory to use when creating new threads - ); - - public Webdlbot(StorageManager storageManager) { - this.storageManager = storageManager; - initializeRootContainer(); - } - - private void initializeRootContainer() { - Object root = storageManager.root(); - if (!(root instanceof DataRootContainer)) { - storageManager.setRoot(new DataRootContainer()); - LOGGER.info("Initialized new DataRootContainer as db root object"); - } - LOGGER.info( - "Root db object contains {} users", - ((DataRootContainer) storageManager.root()).getUsers().size()); - } + private final DownloadQueue downloadQueue = new DownloadQueue(); @Override public String getBotUsername() { @@ -59,7 +27,7 @@ public String getBotUsername() { @Override public String getBotToken() { - if (BOT_TOKEN == null || BOT_TOKEN.isEmpty()) { + if (BOT_TOKEN == null || BOT_TOKEN.isBlank()) { LOGGER.error("BOT_TOKEN is not set in the environment variables."); throw new IllegalStateException("BOT_TOKEN is not set in the environment variables."); } @@ -70,7 +38,7 @@ public String getBotToken() { public void onUpdateReceived(Update update) { if (update.hasMessage() && update.getMessage().hasText()) { try { - long queuedTasks = executorService.getTaskCount() - executorService.getCompletedTaskCount(); + int queuedTasks = downloadQueue.pendingTasks(); Integer queuedMessageId; if (queuedTasks > 0) { String messageTime = DOWNLOAD_REQUEST_QUEUED + " (<" + queuedTasks + "m)"; @@ -86,56 +54,51 @@ public void onUpdateReceived(Update update) { queuedMessageId = null; } - executorService.submit(() -> { + downloadQueue.submit(() -> { try { if (queuedMessageId != null) { - execute(MessageService.deleteMessage( - update.getMessage().getChatId(), queuedMessageId)); + deleteMessage(update.getMessage().getChatId(), queuedMessageId); } - // Entry point download flow dispatch(update); } catch (Exception e) { LOGGER.error("Failed to launch dispatch, error: {}", e.getMessage(), e); - handleDispatchError(update, e); + handleDispatchError(update); } finally { - synchronized (storageManager) { - storageManager.setRoot(loadMetricsData(update)); - storageManager.storeRoot(); - } + cleanDownloadsFolder(); } }); } catch (Exception e) { LOGGER.error("Failed on onUpdateReceived, error: {}", e.getMessage(), e); - handleDispatchError(update, e); + handleDispatchError(update); } } } private void dispatch(Update update) throws Exception { - String url = ""; Message message = update.getMessage(); LOGGER.info( "Starting message processing from @{}: {}", message.getFrom().getUserName(), message.getText()); - if (!update.getMessage().getText().contains("http")) { + String url = extractUrlFromMessage(update.getMessage().getText()); + if (url == null) { execute(MessageService.sendTextMessage(message.getChatId(), message.getMessageId(), NOT_VALID_LINK)) .getMessageId(); return; - } else { - url = extractUrlFromMessage(update.getMessage().getText()); - LOGGER.info("Extracted URL: {} from {}", url, message.getFrom().getUserName()); } + LOGGER.info("Extracted URL: {} from {}", url, message.getFrom().getUserName()); final Integer responseId = execute( MessageService.sendTextMessage(message.getChatId(), message.getMessageId(), DOWNLOAD_REQUEST)) .getMessageId(); - CommandExecutor.executeCommand(url, String.valueOf(message.getMessageId())); - execute(VideoService.sendVideo(message.getChatId(), message.getMessageId())); - execute(MessageService.deleteMessage(message.getChatId(), responseId)); - cleanDownloadsFolder(); + try { + CommandExecutor.executeCommand(url, String.valueOf(message.getMessageId())); + execute(VideoService.sendVideo(message.getChatId(), message.getMessageId())); + } finally { + deleteMessage(message.getChatId(), responseId); + } } - private void handleDispatchError(Update update, Exception e) { + private void handleDispatchError(Update update) { try { execute(MessageService.sendTextMessage( update.getMessage().getChatId(), @@ -144,39 +107,14 @@ private void handleDispatchError(Update update, Exception e) { .getMessageId(); } catch (TelegramApiException ex) { LOGGER.error("Failed to send error message, error: {}", ex.getMessage(), ex); - } finally { - cleanDownloadsFolder(); } } - private DataRootContainer loadMetricsData(Update update) { - // Create or update user data from the Telegram update - org.telegram.telegrambots.meta.api.objects.User telegramUser = - update.getMessage().getFrom(); - - DataRootContainer rootContainer = (DataRootContainer) storageManager.root(); - - // Find existing user or create new one - String userId = telegramUser.getId().toString(); - Optional existingUser = rootContainer.findUserById(userId); - - if (existingUser.isPresent()) { - User user = existingUser.get(); - user.recordNewMessage(); - } else { - User newUser = new User( - userId, - update.getMessage().getChatId(), - telegramUser.getUserName(), - telegramUser.getFirstName(), - telegramUser.getLastName(), - telegramUser.getLanguageCode(), - telegramUser.getIsPremium(), - 1, - LocalDateTime.now().toString()); - rootContainer.getUsers().add(newUser); - LOGGER.info("New user registered: {}", newUser.getUserName()); + private void deleteMessage(Long chatId, Integer messageId) { + try { + execute(MessageService.deleteMessage(chatId, messageId)); + } catch (TelegramApiException e) { + LOGGER.warn("Failed to delete temporary message {}: {}", messageId, e.getMessage()); } - return rootContainer; } } diff --git a/code/src/main/java/org/segocode/webdl/bot/model/DataRootContainer.java b/code/src/main/java/org/segocode/webdl/bot/model/DataRootContainer.java deleted file mode 100644 index 23c0951..0000000 --- a/code/src/main/java/org/segocode/webdl/bot/model/DataRootContainer.java +++ /dev/null @@ -1,25 +0,0 @@ -package org.segocode.webdl.bot.model; - -import java.util.ArrayList; -import java.util.List; -import java.util.Optional; -import lombok.AllArgsConstructor; -import lombok.Data; -import lombok.NoArgsConstructor; - -@Data -@NoArgsConstructor -@AllArgsConstructor -public class DataRootContainer { - private List users = new ArrayList<>(); - - /** - * Finds a user by their ID - * - * @param id The ID of the user to find - * @return An Optional containing the user if found, empty otherwise - */ - public Optional findUserById(String id) { - return users.stream().filter(user -> id.equals(user.getId())).findFirst(); - } -} diff --git a/code/src/main/java/org/segocode/webdl/bot/model/User.java b/code/src/main/java/org/segocode/webdl/bot/model/User.java deleted file mode 100644 index 481a310..0000000 --- a/code/src/main/java/org/segocode/webdl/bot/model/User.java +++ /dev/null @@ -1,39 +0,0 @@ -package org.segocode.webdl.bot.model; - -import java.time.LocalDateTime; -import lombok.AllArgsConstructor; -import lombok.Data; -import lombok.NoArgsConstructor; - -@Data -@NoArgsConstructor -@AllArgsConstructor -public class User { - // Unique identifiers - private String id; - private Long chatId; - private String userName; - - // Personal information - private String firstName; - private String lastName; - - // Preferences and metadata - private String languageCode; - private Boolean isPremium; - - // Usage statistics - private Integer messageCount; - private String lastMessageTime; - - /** - * Increments the message count and updates the last message time to current time - */ - public void recordNewMessage() { - if (this.messageCount == null) { - this.messageCount = 0; - } - this.messageCount++; - this.lastMessageTime = LocalDateTime.now().toString(); - } -} diff --git a/code/src/main/java/org/segocode/webdl/bot/service/VideoService.java b/code/src/main/java/org/segocode/webdl/bot/service/VideoService.java index 3019d9e..4c6cf42 100644 --- a/code/src/main/java/org/segocode/webdl/bot/service/VideoService.java +++ b/code/src/main/java/org/segocode/webdl/bot/service/VideoService.java @@ -1,6 +1,8 @@ package org.segocode.webdl.bot.service; import java.io.File; +import java.nio.file.Path; +import java.nio.file.Paths; import org.segocode.webdl.system.util.FileUtil; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -18,7 +20,12 @@ public class VideoService { * @return The SendVideo object configured with the chat ID and reply to message ID. */ public static SendVideo sendVideo(Long chatId, Integer replyToMessageId) { - String filePath = buildFilePath(replyToMessageId); + return sendVideo(chatId, replyToMessageId, Paths.get("./downloads")); + } + + static SendVideo sendVideo(Long chatId, Integer replyToMessageId, Path downloadsDirectory) { + String filePath = + downloadsDirectory.resolve(replyToMessageId.toString()).toString(); LOGGER.info("Locating video file for message ID {}: {}", replyToMessageId, filePath); File videoFile = FileUtil.locateVideoFile(filePath); @@ -48,14 +55,4 @@ private static SendVideo createSendVideoRequest(Long chatId, Integer replyToMess sendVideoRequest.setVideo(new InputFile(videoFile)); return sendVideoRequest; } - - /** - * Builds the file path for the video file based on the given replyToMessageId. - * - * @param replyToMessageId The ID of the message to which this video will be a reply. - * @return The file path as a String. - */ - private static String buildFilePath(Integer replyToMessageId) { - return "./downloads/" + replyToMessageId + ".mp4"; - } } diff --git a/code/src/main/java/org/segocode/webdl/bot/util/MessageUtil.java b/code/src/main/java/org/segocode/webdl/bot/util/MessageUtil.java index 576f565..2fa1e2e 100644 --- a/code/src/main/java/org/segocode/webdl/bot/util/MessageUtil.java +++ b/code/src/main/java/org/segocode/webdl/bot/util/MessageUtil.java @@ -1,12 +1,13 @@ package org.segocode.webdl.bot.util; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + public class MessageUtil { + private static final Pattern URL_PATTERN = Pattern.compile("https?://\\S+"); + public static String extractUrlFromMessage(String messageText) { - int startIndex = messageText.indexOf("http"); - int endIndex = messageText.indexOf(" ", startIndex); - if (endIndex == -1) { - endIndex = messageText.length(); - } - return messageText.substring(startIndex, endIndex); + Matcher matcher = URL_PATTERN.matcher(messageText); + return matcher.find() ? matcher.group() : null; } } diff --git a/code/src/main/java/org/segocode/webdl/panel/AdminController.java b/code/src/main/java/org/segocode/webdl/panel/AdminController.java deleted file mode 100644 index c3e86f8..0000000 --- a/code/src/main/java/org/segocode/webdl/panel/AdminController.java +++ /dev/null @@ -1,40 +0,0 @@ -package org.segocode.webdl.panel; - -import com.google.gson.Gson; -import io.javalin.http.Context; -import java.io.IOException; -import java.io.InputStream; -import java.nio.charset.StandardCharsets; -import org.eclipse.store.storage.embedded.types.EmbeddedStorageManager; -import org.segocode.webdl.bot.model.DataRootContainer; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -public class AdminController { - private static final Logger LOGGER = LoggerFactory.getLogger(AdminController.class); - private final EmbeddedStorageManager storageManager; - - public AdminController(EmbeddedStorageManager storageManager) { - this.storageManager = storageManager; - } - - public void handleAdminRequest(Context ctx) { - ctx.contentType("text/html"); - - try { - InputStream inputStream = getClass().getResourceAsStream("/views/admin.html"); - String htmlContent = new String(inputStream.readAllBytes(), StandardCharsets.UTF_8); - String jsonData; - synchronized (storageManager) { - jsonData = new Gson().toJson(((DataRootContainer) storageManager.root()).getUsers()); - } - htmlContent = htmlContent.replace("{{!user_data}}", jsonData); - - ctx.result(htmlContent); - } catch (IOException e) { - LOGGER.error("Failed to read admin HTML file", e); - ctx.status(500); - ctx.result("Error loading admin panel: " + e.getMessage()); - } - } -} diff --git a/code/src/main/java/org/segocode/webdl/panel/PanelApplication.java b/code/src/main/java/org/segocode/webdl/panel/PanelApplication.java deleted file mode 100644 index ab354cd..0000000 --- a/code/src/main/java/org/segocode/webdl/panel/PanelApplication.java +++ /dev/null @@ -1,28 +0,0 @@ -package org.segocode.webdl.panel; - -import io.javalin.Javalin; -import org.eclipse.store.storage.embedded.types.EmbeddedStorageManager; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -public class PanelApplication { - private static final Logger LOGGER = LoggerFactory.getLogger(PanelApplication.class); - private static PanelApplication instance; - - private PanelApplication(EmbeddedStorageManager storageManager) { - AdminController adminController = new AdminController(storageManager); - Javalin app = Javalin.create().start(8080); - configureRoutes(app, adminController); - LOGGER.info("Panel application started on port 8080"); - } - - public static synchronized void start(EmbeddedStorageManager storageManager) { - if (instance == null) { - instance = new PanelApplication(storageManager); - } - } - - private void configureRoutes(Javalin app, AdminController adminController) { - app.get("/", adminController::handleAdminRequest); - } -} diff --git a/code/src/main/java/org/segocode/webdl/system/command/CommandExecutor.java b/code/src/main/java/org/segocode/webdl/system/command/CommandExecutor.java index be467a6..f179f76 100644 --- a/code/src/main/java/org/segocode/webdl/system/command/CommandExecutor.java +++ b/code/src/main/java/org/segocode/webdl/system/command/CommandExecutor.java @@ -8,8 +8,9 @@ public class CommandExecutor { private static final Logger LOGGER = LoggerFactory.getLogger(CommandExecutor.class); - private static final int MAX_RETRIES = 5; + private static final int MAX_RETRIES = 2; private static final int TIMEOUT = 120; // seconds + private static final int TERMINATION_TIMEOUT = 5; // seconds /** * Executes a command in the system's command line to download a video using yt-dlp. @@ -24,32 +25,47 @@ public static void executeCommand(String url, String uuid) throws Exception { String ytDlpCommand = osName.contains("win") ? "yt-dlp.exe" : "yt-dlp"; // This is needed? String outputPath = "." + File.separator + "downloads" + File.separator + uuid + ".%(ext)s"; - // The -q option is important to prevent deadlocks by ensuring the output buffer is not filled. - // TODO: make a StreamGobbler to handle buffered output and avoid deadlocks. - String[] command = {ytDlpCommand, "-q", "-S", "ext,res:720", "-o", outputPath, url}; + String[] command = { + ytDlpCommand, "-q", "--no-playlist", "-S", "res:720", "--recode-video", "mp4", "-o", outputPath, url + }; - int attempt = 0; - while (attempt++ < MAX_RETRIES) { + for (int attempt = 1; attempt <= MAX_RETRIES; attempt++) { LOGGER.info("Attempt {} of {}", attempt, MAX_RETRIES); + Process process; try { - Process process = new ProcessBuilder(command).start(); - if (process.waitFor(TIMEOUT, TimeUnit.SECONDS) - ? process.exitValue() == 0 - : process.destroyForcibly() == null) { + process = new ProcessBuilder(command) + .redirectOutput(ProcessBuilder.Redirect.INHERIT) + .redirectError(ProcessBuilder.Redirect.INHERIT) + .start(); + } catch (IOException e) { + throw new IOException("Unable to start yt-dlp", e); + } + + try { + if (!process.waitFor(TIMEOUT, TimeUnit.SECONDS)) { + terminate(process); + LOGGER.warn("Download attempt {} timed out.", attempt); + } else if (process.exitValue() == 0) { LOGGER.info("Download successful for URL: {}", url); return; } else { - LOGGER.warn("Download attempt {} failed or timed out.", attempt); - process.destroyForcibly(); - } - } catch (IOException e) { - LOGGER.error("I/O error occurred on attempt {} of {}", attempt, MAX_RETRIES, e); - if (attempt >= MAX_RETRIES) { - throw e; + LOGGER.warn("Download attempt {} failed with exit code {}.", attempt, process.exitValue()); } + } catch (InterruptedException e) { + terminate(process); + Thread.currentThread().interrupt(); + throw e; } } LOGGER.error("Max retries reached. Command failed for URL: {}", url); throw new RuntimeException("Max retries reached. Command failed."); } + + private static void terminate(Process process) throws InterruptedException { + process.descendants().forEach(ProcessHandle::destroyForcibly); + process.destroyForcibly(); + if (!process.waitFor(TERMINATION_TIMEOUT, TimeUnit.SECONDS)) { + LOGGER.error("yt-dlp did not terminate after {} seconds", TERMINATION_TIMEOUT); + } + } } diff --git a/code/src/main/java/org/segocode/webdl/system/util/FileUtil.java b/code/src/main/java/org/segocode/webdl/system/util/FileUtil.java index dfcb8ee..2f764d7 100644 --- a/code/src/main/java/org/segocode/webdl/system/util/FileUtil.java +++ b/code/src/main/java/org/segocode/webdl/system/util/FileUtil.java @@ -16,10 +16,16 @@ public class FileUtil { * Deletes all files and folders within the 'downloads' folder. */ public static void cleanDownloadsFolder() { - Path downloadsFolder = Paths.get("./downloads/"); - try (Stream paths = Files.walk(downloadsFolder)) { + cleanDirectory(Paths.get("./downloads/")); + } + + static void cleanDirectory(Path directory) { + if (Files.notExists(directory)) { + return; + } + try (Stream paths = Files.walk(directory)) { paths.sorted(Comparator.reverseOrder()) - .filter(p -> !p.equals(downloadsFolder)) + .filter(path -> !path.equals(directory)) .forEach(p -> { try { Files.delete(p); @@ -29,27 +35,18 @@ public static void cleanDownloadsFolder() { } }); } catch (Exception e) { - LOGGER.error("Error cleaning the downloads folder: {}", downloadsFolder, e); + LOGGER.error("Error cleaning directory: {}", directory, e); } } /** - * Locates the video file, attempting to find or rename it with a .mp4 extension. + * Locates the MP4 file generated by yt-dlp. * * @param filePath The path of the video file. * @return The video file if found or renamed successfully, null otherwise. */ public static File locateVideoFile(String filePath) { - File videoFile = new File(filePath); - if (videoFile.exists()) return videoFile; - - // Attempt to locate the file without the .mp4 extension - String filePathWithoutExtension = filePath.replace(".mp4", ""); - videoFile = new File(filePathWithoutExtension); - if (videoFile.exists()) { - File renamedFile = new File(filePathWithoutExtension + ".mp4"); - if (videoFile.renameTo(renamedFile)) return renamedFile; - } - return null; + Path video = Paths.get(filePath + ".mp4"); + return Files.isRegularFile(video) ? video.toFile() : null; } } diff --git a/code/src/main/resources/views/admin.html b/code/src/main/resources/views/admin.html deleted file mode 100644 index d5a472c..0000000 --- a/code/src/main/resources/views/admin.html +++ /dev/null @@ -1,457 +0,0 @@ - - - - - - Dashboard - - - - - - -

-
-
-

DASHBOARD

-

Bot activity & user statistics

-
-
-
PANEL
-
UPDATED --
- -
-
- -
-
-

Total messages processed

-

0

-
-
-
-

Total users

-

0

-
-
-

Average / user

-

0

-
-
-

Active languages

-

0

-
-
-
- -
-
-

User activity table

-

Ordered by message volume

-
- - - - - - - - - - - -
NameUsernameLanguageMessagesLast active
-
-
- - - - diff --git a/code/src/test/java/org/segocode/webdl/bot/DownloadQueueTest.java b/code/src/test/java/org/segocode/webdl/bot/DownloadQueueTest.java new file mode 100644 index 0000000..888b359 --- /dev/null +++ b/code/src/test/java/org/segocode/webdl/bot/DownloadQueueTest.java @@ -0,0 +1,65 @@ +package org.segocode.webdl.bot; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +import java.time.Duration; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; +import org.junit.jupiter.api.Test; + +class DownloadQueueTest { + @Test + void processesTasksOneAtATimeInSubmissionOrder() throws Exception { + List executionOrder = Collections.synchronizedList(new ArrayList<>()); + AtomicInteger activeTasks = new AtomicInteger(); + AtomicInteger maximumActiveTasks = new AtomicInteger(); + CountDownLatch firstTaskStarted = new CountDownLatch(1); + CountDownLatch releaseFirstTask = new CountDownLatch(1); + CountDownLatch completed = new CountDownLatch(3); + + try (DownloadQueue queue = new DownloadQueue()) { + queue.submit(() -> runTask( + 1, executionOrder, activeTasks, maximumActiveTasks, firstTaskStarted, releaseFirstTask, completed)); + firstTaskStarted.await(1, TimeUnit.SECONDS); + queue.submit(() -> runTask(2, executionOrder, activeTasks, maximumActiveTasks, null, null, completed)); + queue.submit(() -> runTask(3, executionOrder, activeTasks, maximumActiveTasks, null, null, completed)); + + assertEquals(3, queue.pendingTasks()); + releaseFirstTask.countDown(); + org.junit.jupiter.api.Assertions.assertTimeoutPreemptively(Duration.ofSeconds(2), () -> completed.await()); + } + + assertEquals(List.of(1, 2, 3), executionOrder); + assertEquals(1, maximumActiveTasks.get()); + } + + private static void runTask( + int id, + List executionOrder, + AtomicInteger activeTasks, + AtomicInteger maximumActiveTasks, + CountDownLatch started, + CountDownLatch release, + CountDownLatch completed) { + int active = activeTasks.incrementAndGet(); + maximumActiveTasks.accumulateAndGet(active, Math::max); + executionOrder.add(id); + if (started != null) { + started.countDown(); + } + try { + if (release != null) { + release.await(); + } + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } finally { + activeTasks.decrementAndGet(); + completed.countDown(); + } + } +} diff --git a/code/src/test/java/org/segocode/webdl/bot/service/VideoServiceTest.java b/code/src/test/java/org/segocode/webdl/bot/service/VideoServiceTest.java new file mode 100644 index 0000000..531169b --- /dev/null +++ b/code/src/test/java/org/segocode/webdl/bot/service/VideoServiceTest.java @@ -0,0 +1,25 @@ +package org.segocode.webdl.bot.service; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +import java.nio.file.Files; +import java.nio.file.Path; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; +import org.telegram.telegrambots.meta.api.methods.send.SendVideo; + +class VideoServiceTest { + @TempDir + Path temporaryDirectory; + + @Test + void createsTelegramRequestForDownloadedVideo() throws Exception { + Path video = Files.createFile(temporaryDirectory.resolve("42.mp4")); + + SendVideo request = VideoService.sendVideo(123L, 42, temporaryDirectory); + + assertEquals("123", request.getChatId()); + assertEquals(42, request.getReplyToMessageId()); + assertEquals(video.toFile(), request.getVideo().getNewMediaFile()); + } +} diff --git a/code/src/test/java/org/segocode/webdl/bot/util/MessageUtilTest.java b/code/src/test/java/org/segocode/webdl/bot/util/MessageUtilTest.java new file mode 100644 index 0000000..dc17e40 --- /dev/null +++ b/code/src/test/java/org/segocode/webdl/bot/util/MessageUtilTest.java @@ -0,0 +1,21 @@ +package org.segocode.webdl.bot.util; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNull; + +import org.junit.jupiter.api.Test; + +class MessageUtilTest { + @Test + void extractsFirstHttpUrlFromMessage() { + assertEquals( + "https://example.com/video?id=1", + MessageUtil.extractUrlFromMessage("download https://example.com/video?id=1 now")); + } + + @Test + void rejectsTextWithoutHttpUrl() { + assertNull(MessageUtil.extractUrlFromMessage("example.com/video")); + assertNull(MessageUtil.extractUrlFromMessage("not a link")); + } +} diff --git a/code/src/test/java/org/segocode/webdl/system/util/FileUtilTest.java b/code/src/test/java/org/segocode/webdl/system/util/FileUtilTest.java new file mode 100644 index 0000000..978640b --- /dev/null +++ b/code/src/test/java/org/segocode/webdl/system/util/FileUtilTest.java @@ -0,0 +1,47 @@ +package org.segocode.webdl.system.util; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.nio.file.Files; +import java.nio.file.Path; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +class FileUtilTest { + @TempDir + Path temporaryDirectory; + + @Test + void locatesMp4Download() throws Exception { + Path video = Files.createFile(temporaryDirectory.resolve("42.mp4")); + + assertEquals( + video.toFile(), + FileUtil.locateVideoFile(temporaryDirectory.resolve("42").toString())); + } + + @Test + void ignoresNonMp4AndPartialDownloads() throws Exception { + Files.createFile(temporaryDirectory.resolve("42.webm")); + Files.createFile(temporaryDirectory.resolve("42.webm.part")); + Files.createFile(temporaryDirectory.resolve("42.info.json")); + + assertNull(FileUtil.locateVideoFile(temporaryDirectory.resolve("42").toString())); + } + + @Test + void removesAllContentsButKeepsDownloadDirectory() throws Exception { + Path nestedDirectory = Files.createDirectory(temporaryDirectory.resolve("nested")); + Files.createFile(nestedDirectory.resolve("video.mp4")); + Files.createFile(temporaryDirectory.resolve("other.webm")); + + FileUtil.cleanDirectory(temporaryDirectory); + + assertTrue(Files.isDirectory(temporaryDirectory)); + try (var files = Files.list(temporaryDirectory)) { + assertEquals(0, files.count()); + } + } +} diff --git a/media/demoPanel.png b/media/demoPanel.png deleted file mode 100644 index e84ee6f..0000000 Binary files a/media/demoPanel.png and /dev/null differ diff --git a/media/logo.png b/media/logo.png new file mode 100644 index 0000000..6c10d80 Binary files /dev/null and b/media/logo.png differ