diff --git a/dev/payload.config.ts b/dev/payload.config.ts index f53fc40..11eb050 100644 --- a/dev/payload.config.ts +++ b/dev/payload.config.ts @@ -123,123 +123,6 @@ export default buildConfig({ retryDelay: 60000, // 1 minute for dev queue: 'default', - // Example: Collection overrides for customization - // Uncomment and modify as needed for your use case - /* - collections: { - templates: { - // Custom access controls - restrict who can manage templates - access: { - read: ({ req: { user } }) => { - if (!user) return false - return user.role === 'admin' || user.permissions?.includes('mailing:read') - }, - create: ({ req: { user } }) => { - if (!user) return false - return user.role === 'admin' || user.permissions?.includes('mailing:create') - }, - update: ({ req: { user } }) => { - if (!user) return false - return user.role === 'admin' || user.permissions?.includes('mailing:update') - }, - delete: ({ req: { user } }) => { - if (!user) return false - return user.role === 'admin' - }, - }, - // Custom admin UI settings - admin: { - group: 'Marketing', - description: 'Email templates with enhanced security and categorization' - }, - // Add custom fields to templates - fields: [ - // Default plugin fields are automatically included - { - name: 'category', - type: 'select', - options: [ - { label: 'Marketing', value: 'marketing' }, - { label: 'Transactional', value: 'transactional' }, - { label: 'System Notifications', value: 'system' } - ], - defaultValue: 'transactional', - admin: { - position: 'sidebar', - description: 'Template category for organization' - } - }, - { - name: 'tags', - type: 'text', - hasMany: true, - admin: { - position: 'sidebar', - description: 'Tags for easy template filtering' - } - }, - { - name: 'isActive', - type: 'checkbox', - defaultValue: true, - admin: { - position: 'sidebar', - description: 'Only active templates can be used' - } - } - ], - // Custom validation hooks - hooks: { - beforeChange: [ - ({ data, req }) => { - // Example: Only admins can create system templates - if (data.category === 'system' && req.user?.role !== 'admin') { - throw new Error('Only administrators can create system notification templates') - } - - // Example: Auto-generate slug if not provided - if (!data.slug && data.name) { - data.slug = data.name.toLowerCase() - .replace(/[^a-z0-9]/g, '-') - .replace(/-+/g, '-') - .replace(/^-|-$/g, '') - } - - return data - } - ] - } - }, - emails: { - // Restrict access to emails collection - access: { - read: ({ req: { user } }) => { - if (!user) return false - return user.role === 'admin' || user.permissions?.includes('mailing:read') - }, - create: ({ req: { user } }) => { - if (!user) return false - return user.role === 'admin' || user.permissions?.includes('mailing:create') - }, - update: ({ req: { user } }) => { - if (!user) return false - return user.role === 'admin' || user.permissions?.includes('mailing:update') - }, - delete: ({ req: { user } }) => { - if (!user) return false - return user.role === 'admin' - }, - }, - // Custom admin configuration for emails - admin: { - group: 'Marketing', - description: 'Email delivery tracking and management', - defaultColumns: ['subject', 'to', 'status', 'priority', 'scheduledAt'], - } - } - }, - */ - // Optional: Custom rich text editor configuration // Comment out to use default lexical editor richTextEditor: lexicalEditor({ @@ -256,12 +139,6 @@ export default buildConfig({ // etc. ], }), - - - // Called after mailing plugin is fully initialized - onReady: async (payload) => { - await seedUser(payload) - }, }), ], secret: process.env.PAYLOAD_SECRET || 'test-secret_key', diff --git a/package.json b/package.json index 5c52251..87d3cba 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@xtr-dev/payload-mailing", - "version": "0.4.13", + "version": "0.4.15", "description": "Template-based email system with scheduling and job processing for PayloadCMS", "type": "module", "main": "dist/index.js", diff --git a/src/collections/Emails.ts b/src/collections/Emails.ts index 9bff505..fcd2a5c 100644 --- a/src/collections/Emails.ts +++ b/src/collections/Emails.ts @@ -1,6 +1,7 @@ import type { CollectionConfig } from 'payload' import { findExistingJobs, ensureEmailJob, updateEmailJobRelationship } from '../utils/jobScheduler.js' import { createContextLogger } from '../utils/logger.js' +import { resolveID } from '../utils/helpers.js' const Emails: CollectionConfig = { slug: 'emails', @@ -10,6 +11,26 @@ const Emails: CollectionConfig = { group: 'Mailing', description: 'Email delivery and status tracking', }, + defaultPopulate: { + template: true, + to: true, + cc: true, + bcc: true, + from: true, + replyTo: true, + jobs: true, + status: true, + attempts: true, + lastAttemptAt: true, + error: true, + priority: true, + scheduledAt: true, + sentAt: true, + variables: true, + html: true, + text: true, + createdAt: true, + }, fields: [ { name: 'template', @@ -177,9 +198,10 @@ const Emails: CollectionConfig = { readOnly: true, }, filterOptions: ({ id }) => { + const emailId = resolveID({ id }) return { 'input.emailId': { - equals: id, + equals: emailId ? String(emailId) : '', }, } }, diff --git a/src/sendEmail.ts b/src/sendEmail.ts index 710b809..3b6d624 100644 --- a/src/sendEmail.ts +++ b/src/sendEmail.ts @@ -3,6 +3,7 @@ import { getMailing, renderTemplate, parseAndValidateEmails, sanitizeFromName } import { BaseEmailDocument } from './types/index.js' import { processJobById } from './utils/emailProcessor.js' import { createContextLogger } from './utils/logger.js' +import { pollForJobId } from './utils/jobPolling.js' // Options for sending emails export interface SendEmailOptions { @@ -48,7 +49,6 @@ export const sendEmail = async = { ...options.data } as Partial - // If using a template, render it first if (options.template) { const { html, text, subject } = await renderTemplate( payload, @@ -56,7 +56,6 @@ export const sendEmail = async 0 ? validated[0] : undefined } if (emailData.from) { const validated = parseAndValidateEmails(emailData.from as string | string[]) - // from should be a single email, so take the first one if array emailData.from = validated && validated.length > 0 ? validated[0] : undefined } - // Sanitize fromName to prevent header injection emailData.fromName = sanitizeFromName(emailData.fromName as string) - // Normalize Date objects to ISO strings for consistent database storage if (emailData.scheduledAt instanceof Date) { emailData.scheduledAt = emailData.scheduledAt.toISOString() } @@ -124,19 +115,15 @@ export const sendEmail = async maxTotalTime) { - throw new Error( - `Job polling timed out after ${maxTotalTime}ms for email ${email.id}. ` + - `The auto-scheduling may have failed or is taking longer than expected.` - ) - } - - // Calculate delay with exponential backoff (25ms, 50ms, 100ms, 200ms, 400ms) - // Cap at 400ms per attempt for better responsiveness - const delay = Math.min(initialDelay * Math.pow(2, attempt), 400) - - if (attempt > 0) { - await new Promise(resolve => setTimeout(resolve, delay)) - } - - // Refetch the email to check for jobs - const emailWithJobs = await payload.findByID({ - collection: collectionSlug, - id: email.id, - }) - - - if (emailWithJobs.jobs && emailWithJobs.jobs.length > 0) { - // Job found! Get the first job ID (should only be one for a new email) - const firstJob = Array.isArray(emailWithJobs.jobs) ? emailWithJobs.jobs[0] : emailWithJobs.jobs - jobId = typeof firstJob === 'string' ? firstJob : String(firstJob.id || firstJob) - break - } - - // Log on later attempts to help with debugging (reduced threshold) - if (attempt >= 1) { - if (attempt >= 2) { - logger.debug(`Waiting for job creation for email ${email.id}, attempt ${attempt + 1}/${maxAttempts}`) - } - } - } - - if (!jobId) { - // Distinguish between different failure scenarios for better error handling - const timeoutMsg = Date.now() - startTime >= maxTotalTime - const errorType = timeoutMsg ? 'POLLING_TIMEOUT' : 'JOB_NOT_FOUND' - - const baseMessage = timeoutMsg - ? `Job polling timed out after ${maxTotalTime}ms for email ${email.id}` - : `No processing job found for email ${email.id} after ${maxAttempts} attempts (${Date.now() - startTime}ms)` - - throw new Error( - `${errorType}: ${baseMessage}. ` + - `This indicates the email was created but job auto-scheduling failed. ` + - `The email exists in the database but immediate processing cannot proceed. ` + - `You may need to: 1) Check job queue configuration, 2) Verify database hooks are working, ` + - `3) Process the email later using processEmailById('${email.id}').` - ) - } + // Poll for the job ID using configurable polling mechanism + const { jobId } = await pollForJobId({ + payload, + collectionSlug, + emailId: email.id, + config: mailingConfig.jobPolling, + logger, + }) try { await processJobById(payload, jobId) diff --git a/src/types/index.ts b/src/types/index.ts index e74ed24..4826286 100644 --- a/src/types/index.ts +++ b/src/types/index.ts @@ -1,6 +1,12 @@ import { Payload } from 'payload' import type { CollectionConfig, RichTextField } from 'payload' +// Payload ID type (string or number) +export type PayloadID = string | number + +// Payload relation type - can be populated (object with id) or unpopulated (just the ID) +export type PayloadRelation = T | PayloadID + // JSON value type that matches Payload's JSON field type export type JSONValue = string | number | boolean | { [k: string]: unknown } | unknown[] | null | undefined @@ -62,6 +68,13 @@ export interface BeforeSendMailOptions { export type BeforeSendHook = (options: BeforeSendMailOptions, email: BaseEmailDocument) => BeforeSendMailOptions | Promise +export interface JobPollingConfig { + maxAttempts?: number // Maximum number of polling attempts (default: 5) + initialDelay?: number // Initial delay in milliseconds (default: 25) + maxTotalTime?: number // Maximum total polling time in milliseconds (default: 3000) + maxBackoffDelay?: number // Maximum delay between attempts in milliseconds (default: 400) +} + export interface MailingPluginConfig { collections?: { templates?: string | Partial @@ -77,6 +90,7 @@ export interface MailingPluginConfig { richTextEditor?: RichTextField['editor'] beforeSend?: BeforeSendHook initOrder?: 'before' | 'after' + jobPolling?: JobPollingConfig } export interface QueuedEmail { diff --git a/src/utils/helpers.ts b/src/utils/helpers.ts index c477c89..6e7de20 100644 --- a/src/utils/helpers.ts +++ b/src/utils/helpers.ts @@ -1,5 +1,5 @@ import { Payload } from 'payload' -import { TemplateVariables } from '../types/index.js' +import { TemplateVariables, PayloadID, PayloadRelation } from '../types/index.js' /** * Parse and validate email addresses @@ -74,6 +74,49 @@ export const sanitizeFromName = (fromName: string | null | undefined): string | return sanitized.length > 0 ? sanitized : undefined } +/** + * Type guard to check if a Payload relation is populated (object) or unpopulated (ID) + */ +export const isPopulated = ( + value: PayloadRelation | null | undefined +): value is T => { + return value !== null && value !== undefined && typeof value === 'object' && 'id' in value +} + +/** + * Resolves a Payload relation to just the ID + * Handles both populated (object with id) and unpopulated (string/number) values + */ +export const resolveID = ( + value: PayloadRelation | null | undefined +): PayloadID | undefined => { + if (value === null || value === undefined) return undefined + + if (typeof value === 'string' || typeof value === 'number') { + return value + } + + if (typeof value === 'object' && 'id' in value) { + return value.id + } + + return undefined +} + +/** + * Resolves an array of Payload relations to an array of IDs + * Handles mixed arrays of populated and unpopulated values + */ +export const resolveIDs = ( + values: (PayloadRelation | null | undefined)[] | null | undefined +): PayloadID[] => { + if (!values || !Array.isArray(values)) return [] + + return values + .map(value => resolveID(value)) + .filter((id): id is PayloadID => id !== undefined) +} + export const getMailing = (payload: Payload) => { const mailing = (payload as any).mailing if (!mailing) { diff --git a/src/utils/jobPolling.ts b/src/utils/jobPolling.ts new file mode 100644 index 0000000..14be8dd --- /dev/null +++ b/src/utils/jobPolling.ts @@ -0,0 +1,115 @@ +import { Payload } from 'payload' +import { JobPollingConfig } from '../types/index.js' + +export interface PollForJobIdOptions { + payload: Payload + collectionSlug: string + emailId: string | number + config?: JobPollingConfig + logger?: { + debug: (message: string, ...args: any[]) => void + info: (message: string, ...args: any[]) => void + warn: (message: string, ...args: any[]) => void + error: (message: string, ...args: any[]) => void + } +} + +export interface PollForJobIdResult { + jobId: string + attempts: number + elapsedTime: number +} + +// Default job polling configuration values +const DEFAULT_JOB_POLLING_CONFIG: Required = { + maxAttempts: 5, + initialDelay: 25, + maxTotalTime: 3000, + maxBackoffDelay: 400, +} + +/** + * Polls for a job ID associated with an email document using exponential backoff. + * This utility handles the complexity of waiting for auto-scheduled jobs to be created. + * + * The polling mechanism uses exponential backoff with configurable parameters: + * - Starts with an initial delay and doubles on each retry + * - Caps individual delays at maxBackoffDelay + * - Enforces a maximum total polling time + * + * @param options - Polling options including payload, collection, email ID, and config + * @returns Promise resolving to job ID and timing information + * @throws Error if job is not found within the configured limits + */ +export const pollForJobId = async (options: PollForJobIdOptions): Promise => { + const { payload, collectionSlug, emailId, logger } = options + + // Merge user config with defaults + const config: Required = { + ...DEFAULT_JOB_POLLING_CONFIG, + ...options.config, + } + + const { maxAttempts, initialDelay, maxTotalTime, maxBackoffDelay } = config + const startTime = Date.now() + let jobId: string | undefined + + for (let attempt = 0; attempt < maxAttempts; attempt++) { + const elapsedTime = Date.now() - startTime + + // Check if we've exceeded the maximum total polling time + if (elapsedTime > maxTotalTime) { + throw new Error( + `Job polling timed out after ${maxTotalTime}ms for email ${emailId}. ` + + `The auto-scheduling may have failed or is taking longer than expected.` + ) + } + + // Calculate exponential backoff delay, capped at maxBackoffDelay + const delay = Math.min(initialDelay * Math.pow(2, attempt), maxBackoffDelay) + + // Wait before checking (skip on first attempt) + if (attempt > 0) { + await new Promise(resolve => setTimeout(resolve, delay)) + } + + // Fetch the email document to check for associated jobs + const emailWithJobs = await payload.findByID({ + collection: collectionSlug, + id: emailId, + }) + + // Check if jobs array exists and has entries + if (emailWithJobs.jobs && emailWithJobs.jobs.length > 0) { + const firstJob = Array.isArray(emailWithJobs.jobs) ? emailWithJobs.jobs[0] : emailWithJobs.jobs + jobId = typeof firstJob === 'string' ? firstJob : String(firstJob.id || firstJob) + + return { + jobId, + attempts: attempt + 1, + elapsedTime: Date.now() - startTime, + } + } + + // Log progress for attempts after the second try + if (attempt >= 2 && logger) { + logger.debug(`Waiting for job creation for email ${emailId}, attempt ${attempt + 1}/${maxAttempts}`) + } + } + + // If we reach here, job was not found + const elapsedTime = Date.now() - startTime + const timeoutMsg = elapsedTime >= maxTotalTime + const errorType = timeoutMsg ? 'POLLING_TIMEOUT' : 'JOB_NOT_FOUND' + const baseMessage = timeoutMsg + ? `Job polling timed out after ${maxTotalTime}ms for email ${emailId}` + : `No processing job found for email ${emailId} after ${maxAttempts} attempts (${elapsedTime}ms)` + + throw new Error( + `${errorType}: ${baseMessage}. ` + + `This indicates the email was created but job auto-scheduling failed. ` + + `The email exists in the database but immediate processing cannot proceed. ` + + `You may need to: 1) Check job queue configuration, 2) Verify database hooks are working, ` + + `3) Process the email later using processEmailById('${emailId}').` + ) +}