From 18aeeaafe81ae73c0d87ae59afd221455945627d Mon Sep 17 00:00:00 2001 From: Matthew Etre Date: Mon, 12 Oct 2026 16:05:34 +0100 Subject: [PATCH] Keep Twitter posting when Discord login fails. OpenSea listeners now start without waiting for Discord, and a down Discord or Twitter destination logs an alert instead of exiting the process. Co-authored-by: Cursor --- .cursor/rules/discord-bot/RULE.md | 4 +- AGENTS.md | 10 +- src/Classes/APIBots/OpenSeaSaleBot.ts | 39 +++-- src/Classes/MintBot.ts | 32 +++- src/Classes/TwitterBot.ts | 165 +++++++++++++---- src/Utils/activityTriager.ts | 23 ++- src/Utils/destinationHealth.ts | 72 ++++++++ src/index.ts | 243 ++++++++++++++++++-------- 8 files changed, 450 insertions(+), 138 deletions(-) create mode 100644 src/Utils/destinationHealth.ts diff --git a/.cursor/rules/discord-bot/RULE.md b/.cursor/rules/discord-bot/RULE.md index 41f4e37f..4282759f 100644 --- a/.cursor/rules/discord-bot/RULE.md +++ b/.cursor/rules/discord-bot/RULE.md @@ -79,8 +79,8 @@ Non-`#` → `smartBotResponse.ts`. Channel must be present in active `channels*. ## OpenSea Dual Path -1. **Stream** (`@opensea/stream-js`): primary listings + sales; started only after Discord login succeeds (`startProductionBots` in `index.ts`). Handlers also no-op when `PRODUCTION_MODE` is false. -2. **REST poll** (`OpenSeaEventsPollBot`): sales backfill; starts after Discord ready; registers stream sale IDs to dedupe +1. **Stream** (`@opensea/stream-js`): primary listings + sales. Starts with the process, not after Discord login (`startProductionBots` in `index.ts`). Handlers also no-op when `PRODUCTION_MODE` is false. Discord posts are skipped when the client is not ready; Twitter posts still go out. +2. **REST poll** (`OpenSeaEventsPollBot`): sales backfill; starts alongside the stream; registers stream sale IDs to dedupe. Activity posts go through `activityTriager.ts` (hardcoded artist/platform → channel mapping + ban list). New sale destinations usually need code there, not only JSON. diff --git a/AGENTS.md b/AGENTS.md index 57d3bf9d..35411364 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -46,12 +46,12 @@ src/index.ts │ POST /new-mint ← Hasura mint webhook (header: webhook_secret) │ GET /callback ← Twitter OAuth │ GET|POST /update ← stub OK responses (not a real health check) - ├── Discord client (if PRODUCTION_MODE) — must login before feeds start + ├── Discord client (if PRODUCTION_MODE) — login retries in the background │ MessageCreate → # commands OR smartBotResponse ├── OpenSea WebSocket stream → OpenSeaListBot / OpenSeaSaleBot → activityTriager - ├── OpenSeaEventsPollBot (sales backfill, 30s) after Discord ready - ├── MintBot (queue + media-proxy poll → Discord; posts only when client is ready) - ├── ScheduleBot (birthdays + optional trivia; starts after Discord ready) + ├── OpenSeaEventsPollBot (sales backfill, 30s) — starts with the stream, not after Discord + ├── MintBot (queue + media-proxy poll → Discord if connected, Twitter if connected) + ├── ScheduleBot (birthdays + optional trivia; starts once Discord is ready) └── TwitterBot (optional, TWITTER_ENABLED) ``` @@ -137,7 +137,7 @@ Token/mint data can include chain IDs (Ethereum, Arbitrum, Base). **OpenSea stre 10. **CODEOWNERS**: `@ArtBlocks/Eng-Approvers-Product`. Do not assign personal Discord handles from the old README. 11. **OpenSea stream reconnect must `disconnect()` the previous Phoenix client** before constructing a new one. Do not reset `reconnectAttempts` until a stream event confirms the socket is live. Keep wildcard `*` subscriptions (OpenSea topics are collection slugs, not contracts; ArtBot filters by contract address). 12. **Do not add urql `cacheExchange`** — this process is long-lived; the document cache never expires. App-level caches (project dicts, wallet TTL) are the right layer. -13. **OpenSea stream, sales poll, and ScheduleBot start only after Discord login succeeds.** Express binds PORT immediately (Render). Mint webhooks may queue, but MintBot does not post until `client.isReady()`. Do not start feed handlers in parallel with `login()`. +13. **OpenSea stream and sales poll start even if Discord login fails.** Express binds PORT immediately (Render). A down Discord or Twitter destination must not exit the process or stop the other one. Posts go to whichever destination is connected. Log sustained failures with `alert:destination_down` and recovery with `alert:destination_recovered` (`destination` is `discord` or `twitter`) so Datadog can page on those fields. ScheduleBot still starts only after Discord is ready. Do not `process.exit` on login failure. ## Common tasks diff --git a/src/Classes/APIBots/OpenSeaSaleBot.ts b/src/Classes/APIBots/OpenSeaSaleBot.ts index 271144f7..4c41211a 100644 --- a/src/Classes/APIBots/OpenSeaSaleBot.ts +++ b/src/Classes/APIBots/OpenSeaSaleBot.ts @@ -78,7 +78,10 @@ export class OpenSeaSaleBot { try { const nftInfo = parseNftId(event.payload.item.nft_id) if (!nftInfo) { - logger.warn({ nftId: event.payload.item.nft_id }, 'Could not parse NFT ID') + logger.warn( + { nftId: event.payload.item.nft_id }, + 'Could not parse NFT ID' + ) return } @@ -151,7 +154,10 @@ export class OpenSeaSaleBot { const now = Date.now() const timeDiff = now - timestamp if (timeDiff > 1000 * 60 * 60 * 4) { - logger.info({ timestamp }, 'Skipping OpenSea sale from more than 4 hours ago') + logger.info( + { timestamp }, + 'Skipping OpenSea sale from more than 4 hours ago' + ) return } @@ -197,7 +203,11 @@ export class OpenSeaSaleBot { let sellerText = await ensOrAddress(seller) let buyerText = await ensOrAddress(buyer) - const platformUrl = buildArtBlocksTokenURL(sale.chainId, contractAddress, tokenId) + const platformUrl = buildArtBlocksTokenURL( + sale.chainId, + contractAddress, + tokenId + ) // Add OpenSea usernames if available (same logic as ReservoirSaleBot) if (!sellerText.includes('.eth')) { @@ -263,15 +273,19 @@ export class OpenSeaSaleBot { if (artBlocksData.collection_name) { logger.info({ name: artBlocksData.name }, 'OpenSea SALE') - sendEmbedToSaleChannels( - this.bot, - embed, - artBlocksData, - await getCollectionType(contractAddress) - ) + try { + sendEmbedToSaleChannels( + this.bot, + embed, + artBlocksData, + await getCollectionType(contractAddress) + ) + } catch (error) { + logger.error({ err: error }, 'Error posting OpenSea sale to Discord') + } } - // Post to Twitter if TwitterBot is available and sale meets criteria + // Twitter is independent of whether the Discord post succeeded. if ( this.twitterBot && this.shouldTweetNormalizedSale(sale, artBlocksData) @@ -318,7 +332,10 @@ export class OpenSeaSaleBot { } if (sale.price < 0.1 && sale.currency.includes('ETH')) { - logger.info({ price: sale.price, currency: sale.currency }, 'Skipping twitter sale for low-value OpenSea sale') + logger.info( + { price: sale.price, currency: sale.currency }, + 'Skipping twitter sale for low-value OpenSea sale' + ) return false } diff --git a/src/Classes/MintBot.ts b/src/Classes/MintBot.ts index 3de5e26f..7655eff3 100644 --- a/src/Classes/MintBot.ts +++ b/src/Classes/MintBot.ts @@ -100,12 +100,10 @@ export class MintBot { // Go through all mints in the queue and make sure the media image exists. // Uses event data only (no token API call). Polls media URL until available, then posts. async checkAndPostMints() { - if (!this.bot.isReady()) { - return - } if (Object.keys(this.mintsToPost).length === 0) { return } + const discordReady = this.bot.isReady() try { await Promise.all( @@ -164,8 +162,29 @@ export class MintBot { mint.contractAddress, mint.tokenId ) - await mint.postToDiscord() - await this.postToTwitter(mint) + if (discordReady) { + try { + await mint.postToDiscord() + } catch (e) { + logger.error( + { err: e, id, destination: 'discord' }, + 'Discord mint post failed' + ) + } + } else { + logger.info( + { id, destination: 'discord' }, + 'Skipping Discord mint post; client is not connected' + ) + } + try { + await this.postToTwitter(mint) + } catch (e) { + logger.error( + { err: e, id, destination: 'twitter' }, + 'Twitter mint post failed' + ) + } } catch (e) { logger.info({ err: e, id }, 'Error fetching image for mint') // Keep in queue to retry later @@ -225,9 +244,6 @@ export class MintBot { // tries to report any new mints to the discord! startRoutine() { this.intervalId = setInterval(async () => { - if (!this.bot.isReady()) { - return - } if (Object.keys(this.mintsToPost).length > 0) { logger.info( { count: Object.keys(this.mintsToPost).length }, diff --git a/src/Classes/TwitterBot.ts b/src/Classes/TwitterBot.ts index 89416b6c..90502a49 100644 --- a/src/Classes/TwitterBot.ts +++ b/src/Classes/TwitterBot.ts @@ -27,6 +27,10 @@ import { updateStatusRefreshToken, } from '../Data/supabase' import { logger } from '../logger' +import { + reportDestinationDown, + reportDestinationUp, +} from '../Utils/destinationHealth' // Twitter Bot Feature Toggles - Modify these to enable/disable functionality const TWITTER_LISTENING_ENABLED = false // Enables listening and responding to tweet mentions @@ -87,7 +91,7 @@ export class TwitterBot { } if (!appKey || !appSecret || !accessToken || !accessSecret) { - logger.warn('Twitter credentials are missing - not initializing TwitterBot') + reportDestinationDown('twitter', 'Twitter credentials are missing') return } @@ -104,14 +108,18 @@ export class TwitterBot { logger.info('Starting Twitter listener') this.startSearchAndReplyRoutine() } else { - logger.info('Twitter listener disabled via TWITTER_LISTENING_ENABLED constant') + logger.info( + 'Twitter listener disabled via TWITTER_LISTENING_ENABLED constant' + ) } } async startSearchAndReplyRoutine() { // Check if Twitter is globally disabled if (!isTwitterGloballyEnabled) { - logger.info('Twitter listener disabled via TWITTER_ENABLED environment variable') + logger.info( + 'Twitter listener disabled via TWITTER_ENABLED environment variable' + ) return } @@ -159,7 +167,10 @@ export class TwitterBot { error.rateLimitError && error.rateLimit ) { - logger.info({ resetTimestamp: error.rateLimit.reset }, 'Search rate limit hit') + logger.info( + { resetTimestamp: error.rateLimit.reset }, + 'Search rate limit hit' + ) } else if ( error instanceof ApiResponseError && error.code === 400 && @@ -173,7 +184,10 @@ export class TwitterBot { ).message.split(' ') const lastId = BigInt(messageSplit[messageSplit.length - 1]) const adjustedLastId = (lastId + BigInt('100000000000')).toString() - logger.info({ adjustedLastId }, 'TwitterBot since_id is invalid - setting to new value') + logger.info( + { adjustedLastId }, + 'TwitterBot since_id is invalid - setting to new value' + ) this.lastTweetId = adjustedLastId } else { logger.error({ err: error }, 'Error searching Twitter') @@ -186,7 +200,10 @@ export class TwitterBot { return } - logger.info({ resultCount: artbotTweets?.meta?.result_count }, 'artbotTweets search result') + logger.info( + { resultCount: artbotTweets?.meta?.result_count }, + 'artbotTweets search result' + ) if (artbotTweets.meta.result_count === 0) { logger.info('No new tweets found') @@ -197,7 +214,10 @@ export class TwitterBot { try { this.replyToTweet(tweet) } catch (e) { - logger.error({ err: e, tweetText: tweet.text }, 'Error responding to tweet') + logger.error( + { err: e, tweetText: tweet.text }, + 'Error responding to tweet' + ) } } if (artbotTweets.meta.newest_id) { @@ -236,9 +256,9 @@ export class TwitterBot { return } - const artBlocksData = await fetch( + const artBlocksData = (await fetch( getTokenApiUrl(projectBot.chainId, projectBot.coreContract, tokenId) - ).then((r) => r.json()) as ArtBlocksTokenData + ).then((r) => r.json())) as ArtBlocksTokenData const assetUrl = artBlocksData.preview_asset_url ?? '' @@ -274,7 +294,10 @@ export class TwitterBot { const tweetMessage = `${artBlocksData.name} by ${artBlocksData.artist} ${platform}\n\n${tokenUrl}` - logger.info({ tweetId: tweet.id, name: artBlocksData.name }, 'Replying to tweet') + logger.info( + { tweetId: tweet.id, name: artBlocksData.name }, + 'Replying to tweet' + ) for (let i = 0; i < NUM_RETRIES; i++) { try { await this.twitterClient?.v2.reply(tweetMessage, tweet.id, { @@ -286,7 +309,11 @@ export class TwitterBot { } catch (error) { if (error instanceof ApiResponseError && error.rateLimit) { logger.info( - { limit: error.rateLimit.limit, remaining: error.rateLimit.remaining, reset: error.rateLimit.reset }, + { + limit: error.rateLimit.limit, + remaining: error.rateLimit.remaining, + reset: error.rateLimit.reset, + }, 'Rate limit hit on tweet sending' ) const reset = new Date(error.rateLimit.reset * 1000) @@ -303,8 +330,14 @@ export class TwitterBot { } return } else { - logger.info({ tweetId: tweet.id, err: error }, 'Error replying to tweet') - logger.info({ attempt: i, numRetries: NUM_RETRIES }, 'Retrying tweet reply') + logger.info( + { tweetId: tweet.id, err: error }, + 'Error replying to tweet' + ) + logger.info( + { attempt: i, numRetries: NUM_RETRIES }, + 'Retrying tweet reply' + ) await delay(RETRY_DELAY_MS) } } @@ -314,7 +347,7 @@ export class TwitterBot { async uploadMedia(assetUrl: string): Promise { const res = await fetch(assetUrl) - let buff = Buffer.from(await res.arrayBuffer() as ArrayBuffer) + let buff = Buffer.from((await res.arrayBuffer()) as ArrayBuffer) while (buff.length > TWITTER_MEDIA_BYTE_LIMIT) { if (assetUrl.includes('.mp4')) { @@ -341,7 +374,10 @@ export class TwitterBot { return mediaId } catch (err) { logger.info({ err, assetUrl }, 'Error uploading media') - logger.info({ attempt: i, numRetries: NUM_RETRIES }, 'Retrying media upload') + logger.info( + { attempt: i, numRetries: NUM_RETRIES }, + 'Retrying media upload' + ) await delay(RETRY_DELAY_MS) } } @@ -361,26 +397,36 @@ export class TwitterBot { async _tweetMint(artBlock: Mint) { // Check if Twitter is globally disabled if (!isTwitterGloballyEnabled) { - logger.info('Twitter mint posting disabled via TWITTER_ENABLED environment variable') + logger.info( + 'Twitter mint posting disabled via TWITTER_ENABLED environment variable' + ) return } // Check if Twitter mint posting is enabled if (!TWITTER_MINT_POSTING_ENABLED) { - logger.info('Twitter mint posting disabled via TWITTER_MINT_POSTING_ENABLED constant') + logger.info( + 'Twitter mint posting disabled via TWITTER_MINT_POSTING_ENABLED constant' + ) return } const assetUrl = artBlock.image if (!artBlock.image) { - logger.error({ artBlock: JSON.stringify(artBlock) }, 'No artblock image defined') + logger.error( + { artBlock: JSON.stringify(artBlock) }, + 'No artblock image defined' + ) return } const mediaId = await this.uploadMedia(assetUrl) if (!mediaId) { - logger.error('no media id returned, not tweeting') + reportDestinationDown( + 'twitter', + 'Twitter mint media upload returned no media id' + ) return } @@ -396,6 +442,7 @@ export class TwitterBot { media: { media_ids: [mediaId] }, }) + reportDestinationUp('twitter') return { tweetRes, } @@ -405,7 +452,7 @@ export class TwitterBot { try { await this._tweetMint(mint) } catch (e) { - logger.error({ err: e }, 'Error posting to Twitter') + reportDestinationDown('twitter', 'Twitter mint post failed', { err: e }) } } @@ -430,13 +477,17 @@ export class TwitterBot { async sendStatusMessage(message: string, replyId?: string) { // Check if Twitter is globally disabled if (!isTwitterGloballyEnabled) { - logger.info('Twitter status messaging disabled via TWITTER_ENABLED environment variable') + logger.info( + 'Twitter status messaging disabled via TWITTER_ENABLED environment variable' + ) return } // Check if Twitter listening is enabled (status messages are mainly for user interactions) if (!TWITTER_LISTENING_ENABLED) { - logger.info('Twitter status messaging disabled via TWITTER_LISTENING_ENABLED constant') + logger.info( + 'Twitter status messaging disabled via TWITTER_LISTENING_ENABLED constant' + ) return } @@ -459,9 +510,9 @@ export class TwitterBot { * Throws an error if the API call fails */ private async getEthToUsdRate(): Promise { - const data = await fetch( + const data = (await fetch( 'https://api.coingecko.com/api/v3/simple/price?ids=ethereum&vs_currencies=usd' - ).then((r) => r.json()) as { ethereum: { usd: number } } + ).then((r) => r.json())) as { ethereum: { usd: number } } return data.ethereum.usd } @@ -561,13 +612,22 @@ export class TwitterBot { }) { // Check if Twitter is globally disabled if (!isTwitterGloballyEnabled) { - logger.info('Twitter sale posting disabled via TWITTER_ENABLED environment variable') + logger.info( + 'Twitter sale posting disabled via TWITTER_ENABLED environment variable' + ) return } // Check if Twitter sale posting is enabled if (!TWITTER_SALE_POSTING_ENABLED) { - logger.info('Twitter sale posting disabled via TWITTER_SALE_POSTING_ENABLED constant') + logger.info( + 'Twitter sale posting disabled via TWITTER_SALE_POSTING_ENABLED constant' + ) + return + } + + if (!this.twitterClient) { + reportDestinationDown('twitter', 'Twitter client is not initialized') return } @@ -579,7 +639,10 @@ export class TwitterBot { try { media_id = await this.uploadMedia(saleData.assetUrl) } catch (error) { - logger.error({ err: error }, 'Error uploading media for sale tweet') + reportDestinationDown('twitter', 'Twitter sale media upload failed', { + err: error, + tokenName: saleData.tokenName, + }) return } @@ -602,7 +665,11 @@ export class TwitterBot { try { usdPrice = await this.convertEthToUsd(saleData.salePrice) logger.info( - { salePrice: saleData.salePrice, currency: saleData.currency, usdPrice: usdPrice.toFixed(2) }, + { + salePrice: saleData.salePrice, + currency: saleData.currency, + usdPrice: usdPrice.toFixed(2), + }, 'Converted ETH to USD using CoinGecko rate' ) } catch (error) { @@ -642,7 +709,7 @@ export class TwitterBot { // Add view link tweetMessage += `\n\nview: ${saleData.tokenUrl + TWITTER_PROJECTBOT_UTM}` - logger.info({ tokenName: saleData.tokenName }, 'Posting sale tweet') + logger.info({ tokenName: saleData.tokenName }, 'Posting sale tweet') // Post the tweet with retry logic for (let i = 0; i < NUM_RETRIES; i++) { @@ -652,13 +719,23 @@ export class TwitterBot { media_ids: [media_id], }, }) - logger.info({ tokenName: saleData.tokenName }, 'Successfully tweeted sale') + logger.info( + { tokenName: saleData.tokenName }, + 'Successfully tweeted sale' + ) + reportDestinationUp('twitter') return } catch (error) { if (error instanceof ApiResponseError && error.rateLimit) { - logger.info( - { limit: error.rateLimit.limit, remaining: error.rateLimit.remaining, reset: error.rateLimit.reset }, - 'Rate limit hit on sale tweet' + reportDestinationDown( + 'twitter', + 'Twitter sale tweet rate limited', + { + limit: error.rateLimit.limit, + remaining: error.rateLimit.remaining, + reset: error.rateLimit.reset, + tokenName: saleData.tokenName, + } ) const reset = new Date(error.rateLimit.reset * 1000) const now = new Date() @@ -669,19 +746,33 @@ export class TwitterBot { `Sale tweet rate limited :( Will retry in ${diffMinutes} minutes` ) } catch (e) { - logger.error({ err: e }, 'Error sending rate limit status message') + logger.error( + { err: e }, + 'Error sending rate limit status message' + ) } return } else { logger.info({ err: error }, 'Error posting sale tweet') - logger.info({ attempt: i + 1, numRetries: NUM_RETRIES }, 'Retrying sale tweet') + logger.info( + { attempt: i + 1, numRetries: NUM_RETRIES }, + 'Retrying sale tweet' + ) await delay(RETRY_DELAY_MS) } } } - logger.error('Failed to post sale tweet after all retry attempts') + reportDestinationDown( + 'twitter', + 'Twitter sale tweet failed after retries', + { + tokenName: saleData.tokenName, + } + ) } catch (error) { - logger.error({ err: error }, 'Error in tweetSale') + reportDestinationDown('twitter', 'Twitter sale tweet failed', { + err: error, + }) } } } diff --git a/src/Utils/activityTriager.ts b/src/Utils/activityTriager.ts index 7a4ad94d..a5a5fd37 100644 --- a/src/Utils/activityTriager.ts +++ b/src/Utils/activityTriager.ts @@ -114,6 +114,17 @@ export const BAN_ADDRESSES = new Set([ '0xa7d8d9ef8d8ce8992df33d8b8cf4aebabd5bd270', ]) +function discordIsConnected(bot: Client): boolean { + if (bot.isReady()) { + return true + } + logger.info( + { destination: 'discord' }, + 'Skipping Discord post; client is not connected' + ) + return false +} + function sendEmbedToChannel( bot: Client, embed: EmbedBuilder, @@ -130,7 +141,11 @@ function sendEmbedToChannel( }) .catch((err) => { logger.info( - { channelName: projectConfig.channels[channelId].name, channelId, errMessage: err.message }, + { + channelName: projectConfig.channels[channelId].name, + channelId, + errMessage: err.message, + }, 'Error posting message in channel' ) }) @@ -145,6 +160,9 @@ export function sendEmbedToSaleChannels( artBlocksData: any, collectionType: CollectionType ) { + if (!discordIsConnected(bot)) { + return + } try { switch (collectionType) { case CollectionType.ENGINE: @@ -237,6 +255,9 @@ export function sendEmbedToListChannels( artBlocksData: any, collectionType: CollectionType ) { + if (!discordIsConnected(bot)) { + return + } try { switch (collectionType) { case CollectionType.ENGINE: diff --git a/src/Utils/destinationHealth.ts b/src/Utils/destinationHealth.ts new file mode 100644 index 00000000..66f95fe0 --- /dev/null +++ b/src/Utils/destinationHealth.ts @@ -0,0 +1,72 @@ +import { logger } from '../logger' + +export type Destination = 'discord' | 'twitter' + +/** + * Stable log fields for Datadog monitors: + * - alert:destination_down — page while a destination is unavailable + * - alert:destination_recovered — clear the page + * + * The error is emitted on the transition to down and again every + * ALERT_HEARTBEAT_MS while it stays down. Retries in between are warnings + * so a sustained outage does not emit an error on every attempt. + */ +const ALERT_HEARTBEAT_MS = 15 * 60 * 1000 + +type DownState = { + downSince: number + lastAlertAt: number +} + +const downState: Record = { + discord: null, + twitter: null, +} + +export function reportDestinationDown( + destination: Destination, + reason: string, + fields: Record = {} +) { + const now = Date.now() + const current = downState[destination] + const shouldAlert = + !current || now - current.lastAlertAt >= ALERT_HEARTBEAT_MS + + if (!current) { + downState[destination] = { downSince: now, lastAlertAt: now } + } else if (shouldAlert) { + current.lastAlertAt = now + } + + const downSince = downState[destination]?.downSince ?? now + const payload = { + destination, + reason, + downSince: new Date(downSince).toISOString(), + ...fields, + } + + if (shouldAlert) { + logger.error( + { alert: 'destination_down', ...payload }, + 'Destination unavailable' + ) + } else { + logger.warn(payload, 'Destination still unavailable') + } +} + +export function reportDestinationUp(destination: Destination) { + const current = downState[destination] + if (!current) { + return + } + + const downMs = Date.now() - current.downSince + downState[destination] = null + logger.info( + { alert: 'destination_recovered', destination, downMs }, + 'Destination recovered' + ) +} diff --git a/src/index.ts b/src/index.ts index cee54102..b81fd813 100644 --- a/src/index.ts +++ b/src/index.ts @@ -1,6 +1,13 @@ import * as dotenv from 'dotenv' dotenv.config() -import { Client, Events, GatewayIntentBits, Options, Status, User } from 'discord.js' +import { + Client, + Events, + GatewayIntentBits, + Options, + Status, + User, +} from 'discord.js' import * as dns from 'dns' import * as https from 'https' const express = require('express') @@ -42,6 +49,10 @@ import { smartBotResponse } from './Utils/smartBotResponse' import { logContext } from './Utils/logger' import { randomUUID } from 'crypto' import { logger } from './logger' +import { + reportDestinationDown, + reportDestinationUp, +} from './Utils/destinationHealth' // Misc. server configuration info. const DISCORD_TOKEN = process.env.DISCORD_TOKEN @@ -158,7 +169,10 @@ type MintEventPayload = { app.post( '/new-mint', function ( - req: { body: MintEventPayload; headers: Record }, + req: { + body: MintEventPayload + headers: Record + }, res: { setHeader: (key: string, value: string) => void json: (data: unknown) => void @@ -237,8 +251,51 @@ logger.info('Discord client created') logger.info({ PRODUCTION_MODE }, 'PRODUCTION_MODE') logger.info({ discordTokenExists: !!DISCORD_TOKEN }, 'DISCORD_TOKEN exists') -const MAX_LOGIN_RETRIES = 5 let loginRetryCount = 0 +let scheduleBotStarted = false +let discordDownTimer: NodeJS.Timeout | undefined + +function clearDiscordDownTimer() { + if (discordDownTimer) { + clearTimeout(discordDownTimer) + discordDownTimer = undefined + } +} + +/** Alert if Discord is still gone after a short reconnect window. */ +function scheduleDiscordDownAlert(reason: string) { + if (discordDownTimer) { + return + } + discordDownTimer = setTimeout(() => { + discordDownTimer = undefined + if (!discordClient.isReady()) { + reportDestinationDown('discord', reason) + } + }, 60_000) +} + +/** Clear a down alert if the client is still connected after a noisy error. */ +function scheduleDiscordRecoveryCheck() { + setTimeout(() => { + if (discordClient.isReady()) { + clearDiscordDownTimer() + reportDestinationUp('discord') + } + }, 60_000) +} + +function startScheduleBot() { + if (scheduleBotStarted) { + return + } + scheduleBotStarted = true + const scheduleBot = new ScheduleBot( + discordClient.channels.cache, + projectConfig + ) + botsToCleanup.push(scheduleBot) +} /** Resolve a hostname via DNS and log the result for connectivity diagnostics. */ async function checkDnsResolution(hostname: string): Promise { @@ -274,7 +331,9 @@ async function waitForDiscordGateway(): Promise { (res) => { const latencyMs = Date.now() - startMs const retryAfterHeader = res.headers['retry-after'] - const retryAfterSec = retryAfterHeader ? parseFloat(String(retryAfterHeader)) : null + const retryAfterSec = retryAfterHeader + ? parseFloat(String(retryAfterHeader)) + : null if (res.statusCode === 429) { const waitMs = retryAfterSec != null ? retryAfterSec * 1000 : 60_000 @@ -309,7 +368,10 @@ async function waitForDiscordGateway(): Promise { resolve(true) // proceed anyway; the error may be transient }) req.on('timeout', () => { - logger.warn({ url: DISCORD_GATEWAY_URL }, 'Discord gateway connectivity check timed out') + logger.warn( + { url: DISCORD_GATEWAY_URL }, + 'Discord gateway connectivity check timed out' + ) req.destroy() resolve(true) }) @@ -342,13 +404,13 @@ function sleep(ms: number): Promise { return new Promise((resolve) => setTimeout(resolve, ms)) } -/** Resolves only after Discord login succeeds. Exits the process after max retries. */ -async function attemptDiscordLogin(): Promise { - while (true) { - logger.info( - { attempt: loginRetryCount + 1, MAX_LOGIN_RETRIES }, - 'Attempting Discord login' - ) +/** + * Keeps trying Discord login in the background. A failed login does not + * exit the process; OpenSea and Twitter keep running. + */ +async function maintainDiscordLogin(): Promise { + while (!discordClient.isReady()) { + logger.info({ attempt: loginRetryCount + 1 }, 'Attempting Discord login') await runPreLoginDiagnostics() @@ -393,28 +455,24 @@ async function attemptDiscordLogin(): Promise { return } catch (error) { clearTimeout(timeoutHandle) - logger.error({ err: error }, 'Discord login failed') - - if (loginRetryCount < MAX_LOGIN_RETRIES) { - loginRetryCount++ - // Exponential backoff: 5s, 10s, 20s, 40s, 80s - const backoffTime = Math.min( - 5000 * Math.pow(2, loginRetryCount - 1), - 80000 - ) - logger.info( - { - loginRetryCount, - MAX_LOGIN_RETRIES, - backoffSeconds: backoffTime / 1000, - }, - 'Retrying Discord login attempt' - ) - await sleep(backoffTime) - } else { - logger.error('Max login retries reached. Giving up.') - process.exit(1) - } + const reason = + error instanceof Error ? error.message : 'Discord login failed' + reportDestinationDown('discord', reason, { + err: error, + attempt: loginRetryCount + 1, + }) + + loginRetryCount++ + // Exponential backoff: 5s, 10s, 20s, 40s, 80s, then stay at 80s. + const backoffTime = Math.min( + 5000 * Math.pow(2, Math.min(loginRetryCount - 1, 4)), + 80000 + ) + logger.info( + { attempt: loginRetryCount, backoffSeconds: backoffTime / 1000 }, + 'Retrying Discord login attempt' + ) + await sleep(backoffTime) } finally { discordClient.off('debug', debugListener) } @@ -422,22 +480,45 @@ async function attemptDiscordLogin(): Promise { } discordClient.on('ready', () => { - logger.info({ tag: discordClient.user?.tag }, 'Logged in to Discord') + clearDiscordDownTimer() + reportDestinationUp('discord') + logger.info( + { + tag: discordClient.user?.tag, + guildCount: discordClient.guilds.cache.size, + channelCount: discordClient.channels.cache.size, + blockTalkCached: discordClient.channels.cache.has(CHANNEL_BLOCK_TALK), + }, + 'Logged in to Discord' + ) + if (PRODUCTION_MODE) { + startScheduleBot() + } }) discordClient.on('error', (error) => { - logger.error({ err: error }, 'Discord client error') + reportDestinationDown('discord', error.message, { err: error }) + scheduleDiscordRecoveryCheck() }) discordClient.on('shardError', (error, shardId) => { - logger.error({ err: error, shardId }, 'Discord shard error') + reportDestinationDown('discord', error.message, { err: error, shardId }) + scheduleDiscordRecoveryCheck() }) discordClient.on('shardDisconnect', (closeEvent, shardId) => { logger.warn( - { shardId, code: closeEvent.code, reason: closeEvent.reason, wasClean: closeEvent.wasClean }, + { + shardId, + code: closeEvent.code, + reason: closeEvent.reason, + wasClean: closeEvent.wasClean, + }, 'Discord shard disconnected' ) + scheduleDiscordDownAlert( + `Discord shard disconnected (code ${closeEvent.code})` + ) }) discordClient.on('shardReconnecting', (shardId) => { @@ -445,9 +526,8 @@ discordClient.on('shardReconnecting', (shardId) => { }) discordClient.on('disconnect', () => { - logger.info('Discord client disconnected') - // Cleanup all bots - botsToCleanup.forEach((bot) => bot.cleanup()) + logger.warn('Discord client disconnected') + reportDestinationDown('discord', 'Discord client disconnected') }) export const triviaBot = new TriviaBot(discordClient) @@ -655,7 +735,9 @@ function initializeOpenSeaStreamClient(): OpenSeaStreamClient { errorMessage.includes('502') || errorMessage.includes('Bad Gateway') ) { - logger.warn('Detected 502 Bad Gateway error - OpenSea server issue. Automatic reconnection will be attempted with exponential backoff.') + logger.warn( + 'Detected 502 Bad Gateway error - OpenSea server issue. Automatic reconnection will be attempted with exponential backoff.' + ) } // Schedule reconnection for connection errors @@ -688,7 +770,10 @@ function scheduleReconnection() { } if (connectionState.reconnectAttempts >= MAX_RECONNECT_ATTEMPTS) { - logger.error({ MAX_RECONNECT_ATTEMPTS }, 'Max reconnection attempts reached. Stopping reconnection attempts. Falling back to API polling only for OpenSea events.') + logger.error( + { MAX_RECONNECT_ATTEMPTS }, + 'Max reconnection attempts reached. Stopping reconnection attempts. Falling back to API polling only for OpenSea events.' + ) return } @@ -696,20 +781,30 @@ function scheduleReconnection() { connectionState.reconnectAttempts++ logger.info( - { attempt: connectionState.reconnectAttempts, maxAttempts: MAX_RECONNECT_ATTEMPTS, delaySeconds: delay / 1000 }, + { + attempt: connectionState.reconnectAttempts, + maxAttempts: MAX_RECONNECT_ATTEMPTS, + delaySeconds: delay / 1000, + }, 'Scheduling OpenSea WebSocket reconnection attempt' ) reconnectTimeout = setTimeout(() => { logger.info( - { attempt: connectionState.reconnectAttempts, maxAttempts: MAX_RECONNECT_ATTEMPTS }, + { + attempt: connectionState.reconnectAttempts, + maxAttempts: MAX_RECONNECT_ATTEMPTS, + }, 'Attempting OpenSea WebSocket reconnection' ) try { openseaStreamClient = initializeOpenSeaStreamClient() setupStreamEventHandlers(openseaStreamClient) } catch (error) { - logger.error({ err: error }, 'Failed to reinitialize OpenSea Stream Client') + logger.error( + { err: error }, + 'Failed to reinitialize OpenSea Stream Client' + ) scheduleReconnection() } }, delay) @@ -725,9 +820,13 @@ function startHealthCheckMonitoring() { if (!connectionState.isConnected) { if (connectionState.reconnectAttempts >= MAX_RECONNECT_ATTEMPTS) { - logger.warn('OpenSea WebSocket health check: max reconnects reached, relying on API polling') + logger.warn( + 'OpenSea WebSocket health check: max reconnects reached, relying on API polling' + ) } else { - logger.warn('OpenSea WebSocket health check: Connection lost, reconnection should be in progress') + logger.warn( + 'OpenSea WebSocket health check: Connection lost, reconnection should be in progress' + ) } } else if (timeSinceLastConnect > 600000) { // 10 minutes @@ -740,7 +839,10 @@ function startHealthCheckMonitoring() { // Log connection stats if (connectionState.reconnectAttempts > 0) { logger.info( - { reconnectAttempts: connectionState.reconnectAttempts, isConnected: connectionState.isConnected }, + { + reconnectAttempts: connectionState.reconnectAttempts, + isConnected: connectionState.isConnected, + }, 'OpenSea WebSocket stats' ) } @@ -854,7 +956,7 @@ if (typeof global !== 'undefined') { ).getOpenSeaConnectionStats = getOpenSeaConnectionStats } -// OpenSea Stream Bots - constructed here; stream/poll start after Discord login. +// OpenSea Stream Bots - constructed here; stream and poll start in startProductionBots. const openSeaListBot = new OpenSeaListBot(discordClient) const openSeaSaleBot = new OpenSeaSaleBot(discordClient, abTwitterBot) // Primary sales handler botsToCleanup.push(openSeaListBot) @@ -878,7 +980,10 @@ const initOpenSeaApiPollingBots = async () => { .concat(studioContracts) .concat(engineContracts) - logger.info({ count: allContracts.length }, 'Tracking contracts for OpenSea API polling') + logger.info( + { count: allContracts.length }, + 'Tracking contracts for OpenSea API polling' + ) // OpenSea API configuration const OPENSEA_API_BASE = 'https://api.opensea.io/api/v2/events' @@ -900,7 +1005,9 @@ const initOpenSeaApiPollingBots = async () => { ) botsToCleanup.push(openSeaEventsBot) - logger.info('OpenSea API polling bot initialized successfully (sales backfill only)') + logger.info( + 'OpenSea API polling bot initialized successfully (sales backfill only)' + ) } // Helper function to check if an OpenSea NFT ID contains any of our tracked contracts @@ -924,24 +1031,13 @@ const isTrackedContract = ( } /** - * Express stays up so Render can bind PORT. Discord-dependent work - * (OpenSea stream, sales poll, scheduler) starts only after login succeeds. + * Express binds PORT immediately. OpenSea listeners start whether or not + * Discord or Twitter is up. Each post goes to the destinations that are + * connected. Discord login retries in the background and never exits the + * process. ScheduleBot starts from the Discord ready handler. */ async function startProductionBots() { - logger.info('Waiting for Discord login before starting OpenSea and scheduled bots') - await attemptDiscordLogin() - - logger.info( - { - tag: discordClient.user?.tag, - guildCount: discordClient.guilds.cache.size, - channelCount: discordClient.channels.cache.size, - blockTalkCached: discordClient.channels.cache.has(CHANNEL_BLOCK_TALK), - }, - 'Discord ready — starting OpenSea and scheduled bots' - ) - - logger.info('Starting OpenSea WebSocket Stream integration') + logger.info('Starting OpenSea listeners') openseaStreamClient = initializeOpenSeaStreamClient() setupStreamEventHandlers(openseaStreamClient) logger.info( @@ -949,15 +1045,14 @@ async function startProductionBots() { 'OpenSea Stream Config' ) - const scheduleBot = new ScheduleBot( - discordClient.channels.cache, - projectConfig - ) - botsToCleanup.push(scheduleBot) - initOpenSeaApiPollingBots().catch((err) => { logger.error({ err }, 'Error initializing OpenSea API polling bots') }) + + logger.info('Starting Discord login in the background') + void maintainDiscordLogin().catch((err) => { + reportDestinationDown('discord', 'Discord login loop stopped', { err }) + }) } if (PRODUCTION_MODE) {