Merge pull request #62 from xtr-dev/dev

Dev

dev/payload.config.ts

@@ -123,123 +123,6 @@ export default buildConfig({
         retryDelay: 60000, // 1 minute for dev
         queue: 'default',
 
-        // Example: Collection overrides for customization
-        // Uncomment and modify as needed for your use case
-        /*
-        collections: {
-          templates: {
-            // Custom access controls - restrict who can manage templates
-            access: {
-              read: ({ req: { user } }) => {
-                if (!user) return false
-                return user.role === 'admin' || user.permissions?.includes('mailing:read')
-              },
-              create: ({ req: { user } }) => {
-                if (!user) return false
-                return user.role === 'admin' || user.permissions?.includes('mailing:create')
-              },
-              update: ({ req: { user } }) => {
-                if (!user) return false
-                return user.role === 'admin' || user.permissions?.includes('mailing:update')
-              },
-              delete: ({ req: { user } }) => {
-                if (!user) return false
-                return user.role === 'admin'
-              },
-            },
-            // Custom admin UI settings
-            admin: {
-              group: 'Marketing',
-              description: 'Email templates with enhanced security and categorization'
-            },
-            // Add custom fields to templates
-            fields: [
-              // Default plugin fields are automatically included
-              {
-                name: 'category',
-                type: 'select',
-                options: [
-                  { label: 'Marketing', value: 'marketing' },
-                  { label: 'Transactional', value: 'transactional' },
-                  { label: 'System Notifications', value: 'system' }
-                ],
-                defaultValue: 'transactional',
-                admin: {
-                  position: 'sidebar',
-                  description: 'Template category for organization'
-                }
-              },
-              {
-                name: 'tags',
-                type: 'text',
-                hasMany: true,
-                admin: {
-                  position: 'sidebar',
-                  description: 'Tags for easy template filtering'
-                }
-              },
-              {
-                name: 'isActive',
-                type: 'checkbox',
-                defaultValue: true,
-                admin: {
-                  position: 'sidebar',
-                  description: 'Only active templates can be used'
-                }
-              }
-            ],
-            // Custom validation hooks
-            hooks: {
-              beforeChange: [
-                ({ data, req }) => {
-                  // Example: Only admins can create system templates
-                  if (data.category === 'system' && req.user?.role !== 'admin') {
-                    throw new Error('Only administrators can create system notification templates')
-                  }
-
-                  // Example: Auto-generate slug if not provided
-                  if (!data.slug && data.name) {
-                    data.slug = data.name.toLowerCase()
-                      .replace(/[^a-z0-9]/g, '-')
-                      .replace(/-+/g, '-')
-                      .replace(/^-|-$/g, '')
-                  }
-
-                  return data
-                }
-              ]
-            }
-          },
-          emails: {
-            // Restrict access to emails collection
-            access: {
-              read: ({ req: { user } }) => {
-                if (!user) return false
-                return user.role === 'admin' || user.permissions?.includes('mailing:read')
-              },
-              create: ({ req: { user } }) => {
-                if (!user) return false
-                return user.role === 'admin' || user.permissions?.includes('mailing:create')
-              },
-              update: ({ req: { user } }) => {
-                if (!user) return false
-                return user.role === 'admin' || user.permissions?.includes('mailing:update')
-              },
-              delete: ({ req: { user } }) => {
-                if (!user) return false
-                return user.role === 'admin'
-              },
-            },
-            // Custom admin configuration for emails
-            admin: {
-              group: 'Marketing',
-              description: 'Email delivery tracking and management',
-              defaultColumns: ['subject', 'to', 'status', 'priority', 'scheduledAt'],
-            }
-          }
-        },
-        */
-
         // Optional: Custom rich text editor configuration
         // Comment out to use default lexical editor
         richTextEditor: lexicalEditor({
@@ -256,12 +139,6 @@ export default buildConfig({
             // etc.
           ],
         }),
-
-
-        // Called after mailing plugin is fully initialized
-        onReady: async (payload) => {
-          await seedUser(payload)
-        },
       }),
     ],
     secret: process.env.PAYLOAD_SECRET || 'test-secret_key',

package.json

@@ -1,6 +1,6 @@
 {
   "name": "@xtr-dev/payload-mailing",
-  "version": "0.4.13",
+  "version": "0.4.15",
   "description": "Template-based email system with scheduling and job processing for PayloadCMS",
   "type": "module",
   "main": "dist/index.js",

src/collections/Emails.ts

@@ -1,6 +1,7 @@
 import type { CollectionConfig } from 'payload'
 import { findExistingJobs, ensureEmailJob, updateEmailJobRelationship } from '../utils/jobScheduler.js'
 import { createContextLogger } from '../utils/logger.js'
+import { resolveID } from '../utils/helpers.js'
 
 const Emails: CollectionConfig = {
   slug: 'emails',
@@ -10,6 +11,26 @@ const Emails: CollectionConfig = {
     group: 'Mailing',
     description: 'Email delivery and status tracking',
   },
+  defaultPopulate: {
+    template: true,
+    to: true,
+    cc: true,
+    bcc: true,
+    from: true,
+    replyTo: true,
+    jobs: true,
+    status: true,
+    attempts: true,
+    lastAttemptAt: true,
+    error: true,
+    priority: true,
+    scheduledAt: true,
+    sentAt: true,
+    variables: true,
+    html: true,
+    text: true,
+    createdAt: true,
+  },
   fields: [
     {
       name: 'template',
@@ -177,9 +198,10 @@ const Emails: CollectionConfig = {
         readOnly: true,
       },
       filterOptions: ({ id }) => {
+        const emailId = resolveID({ id })
         return {
           'input.emailId': {
-            equals: id,
+            equals: emailId ? String(emailId) : '',
           },
         }
       },

src/sendEmail.ts

@@ -3,6 +3,7 @@ import { getMailing, renderTemplate, parseAndValidateEmails, sanitizeFromName }
 import { BaseEmailDocument } from './types/index.js'
 import { processJobById } from './utils/emailProcessor.js'
 import { createContextLogger } from './utils/logger.js'
+import { pollForJobId } from './utils/jobPolling.js'
 
 // Options for sending emails
 export interface SendEmailOptions<T extends BaseEmailDocument = BaseEmailDocument> {
@@ -48,7 +49,6 @@ export const sendEmail = async <TEmail extends BaseEmailDocument = BaseEmailDocu
 
   let emailData: Partial<TEmail> = { ...options.data } as Partial<TEmail>
 
-  // If using a template, render it first
   if (options.template) {
     const { html, text, subject } = await renderTemplate(
       payload,
@@ -56,7 +56,6 @@ export const sendEmail = async <TEmail extends BaseEmailDocument = BaseEmailDocu
       options.template.variables || {}
     )
 
-    // Template values take precedence over data values
     emailData = {
       ...emailData,
       subject,
@@ -70,20 +69,16 @@ export const sendEmail = async <TEmail extends BaseEmailDocument = BaseEmailDocu
     throw new Error('Field "to" is required for sending emails')
   }
 
-  // Validate required fields based on whether template was used
   if (options.template) {
-    // When using template, subject and html should have been set by renderTemplate
     if (!emailData.subject || !emailData.html) {
       throw new Error(`Template rendering failed: template "${options.template.slug}" did not provide required subject and html content`)
     }
   } else {
-    // When not using template, user must provide subject and html directly
     if (!emailData.subject || !emailData.html) {
       throw new Error('Fields "subject" and "html" are required when sending direct emails without a template')
     }
   }
 
-  // Process email addresses using shared validation (handle null values)
   if (emailData.to) {
     emailData.to = parseAndValidateEmails(emailData.to as string | string[])
   }
@@ -95,19 +90,15 @@ export const sendEmail = async <TEmail extends BaseEmailDocument = BaseEmailDocu
   }
   if (emailData.replyTo) {
     const validated = parseAndValidateEmails(emailData.replyTo as string | string[])
-    // replyTo should be a single email, so take the first one if array
     emailData.replyTo = validated && validated.length > 0 ? validated[0] : undefined
   }
   if (emailData.from) {
     const validated = parseAndValidateEmails(emailData.from as string | string[])
-    // from should be a single email, so take the first one if array
     emailData.from = validated && validated.length > 0 ? validated[0] : undefined
   }
 
-  // Sanitize fromName to prevent header injection
   emailData.fromName = sanitizeFromName(emailData.fromName as string)
 
-  // Normalize Date objects to ISO strings for consistent database storage
   if (emailData.scheduledAt instanceof Date) {
     emailData.scheduledAt = emailData.scheduledAt.toISOString()
   }
@@ -124,19 +115,15 @@ export const sendEmail = async <TEmail extends BaseEmailDocument = BaseEmailDocu
     emailData.updatedAt = emailData.updatedAt.toISOString()
   }
 
-  // Create the email in the collection with proper typing
-  // The hooks will automatically create and populate the job relationship
   const email = await payload.create({
     collection: collectionSlug,
     data: emailData
   })
 
-  // Validate that the created email has the expected structure
   if (!email || typeof email !== 'object' || !email.id) {
     throw new Error('Failed to create email: invalid response from database')
   }
 
-  // If processImmediately is true, get the job from the relationship and process it now
   if (options.processImmediately) {
     const logger = createContextLogger(payload, 'IMMEDIATE')
 
@@ -144,72 +131,15 @@ export const sendEmail = async <TEmail extends BaseEmailDocument = BaseEmailDocu
       throw new Error('PayloadCMS jobs not configured - cannot process email immediately')
     }
 
-    // Poll for the job with optimized backoff and timeout protection
+    // Poll for the job ID using configurable polling mechanism
-    // This handles the async nature of hooks and ensures we wait for job creation
+    const { jobId } = await pollForJobId({
-    const maxAttempts = 5 // Reduced from 10 to minimize delay
+      payload,
-    const initialDelay = 25 // Reduced from 50ms for faster response
+      collectionSlug,
-    const maxTotalTime = 3000 // 3 second total timeout
+      emailId: email.id,
-    const startTime = Date.now()
+      config: mailingConfig.jobPolling,
-    let jobId: string | undefined
+      logger,
-
-
-    for (let attempt = 0; attempt < maxAttempts; attempt++) {
-      // Check total timeout before continuing
-      if (Date.now() - startTime > maxTotalTime) {
-        throw new Error(
-          `Job polling timed out after ${maxTotalTime}ms for email ${email.id}. ` +
-          `The auto-scheduling may have failed or is taking longer than expected.`
-        )
-      }
-
-      // Calculate delay with exponential backoff (25ms, 50ms, 100ms, 200ms, 400ms)
-      // Cap at 400ms per attempt for better responsiveness
-      const delay = Math.min(initialDelay * Math.pow(2, attempt), 400)
-
-      if (attempt > 0) {
-        await new Promise(resolve => setTimeout(resolve, delay))
-      }
-
-      // Refetch the email to check for jobs
-      const emailWithJobs = await payload.findByID({
-        collection: collectionSlug,
-        id: email.id,
     })
 
-
-      if (emailWithJobs.jobs && emailWithJobs.jobs.length > 0) {
-        // Job found! Get the first job ID (should only be one for a new email)
-        const firstJob = Array.isArray(emailWithJobs.jobs) ? emailWithJobs.jobs[0] : emailWithJobs.jobs
-        jobId = typeof firstJob === 'string' ? firstJob : String(firstJob.id || firstJob)
-        break
-      }
-
-      // Log on later attempts to help with debugging (reduced threshold)
-      if (attempt >= 1) {
-        if (attempt >= 2) {
-          logger.debug(`Waiting for job creation for email ${email.id}, attempt ${attempt + 1}/${maxAttempts}`)
-        }
-      }
-    }
-
-    if (!jobId) {
-      // Distinguish between different failure scenarios for better error handling
-      const timeoutMsg = Date.now() - startTime >= maxTotalTime
-      const errorType = timeoutMsg ? 'POLLING_TIMEOUT' : 'JOB_NOT_FOUND'
-
-      const baseMessage = timeoutMsg
-        ? `Job polling timed out after ${maxTotalTime}ms for email ${email.id}`
-        : `No processing job found for email ${email.id} after ${maxAttempts} attempts (${Date.now() - startTime}ms)`
-
-      throw new Error(
-        `${errorType}: ${baseMessage}. ` +
-        `This indicates the email was created but job auto-scheduling failed. ` +
-        `The email exists in the database but immediate processing cannot proceed. ` +
-        `You may need to: 1) Check job queue configuration, 2) Verify database hooks are working, ` +
-        `3) Process the email later using processEmailById('${email.id}').`
-      )
-    }
-
     try {
       await processJobById(payload, jobId)
       logger.debug(`Successfully processed email ${email.id} immediately`)

src/types/index.ts

@@ -1,6 +1,12 @@
 import { Payload } from 'payload'
 import type { CollectionConfig, RichTextField } from 'payload'
 
+// Payload ID type (string or number)
+export type PayloadID = string | number
+
+// Payload relation type - can be populated (object with id) or unpopulated (just the ID)
+export type PayloadRelation<T extends { id: PayloadID }> = T | PayloadID
+
 // JSON value type that matches Payload's JSON field type
 export type JSONValue = string | number | boolean | { [k: string]: unknown } | unknown[] | null | undefined
 
@@ -62,6 +68,13 @@ export interface BeforeSendMailOptions {
 
 export type BeforeSendHook = (options: BeforeSendMailOptions, email: BaseEmailDocument) => BeforeSendMailOptions | Promise<BeforeSendMailOptions>
 
+export interface JobPollingConfig {
+  maxAttempts?: number        // Maximum number of polling attempts (default: 5)
+  initialDelay?: number       // Initial delay in milliseconds (default: 25)
+  maxTotalTime?: number       // Maximum total polling time in milliseconds (default: 3000)
+  maxBackoffDelay?: number    // Maximum delay between attempts in milliseconds (default: 400)
+}
+
 export interface MailingPluginConfig {
   collections?: {
     templates?: string | Partial<CollectionConfig>
@@ -77,6 +90,7 @@ export interface MailingPluginConfig {
   richTextEditor?: RichTextField['editor']
   beforeSend?: BeforeSendHook
   initOrder?: 'before' | 'after'
+  jobPolling?: JobPollingConfig
 }
 
 export interface QueuedEmail {

src/utils/helpers.ts

@@ -1,5 +1,5 @@
 import { Payload } from 'payload'
-import { TemplateVariables } from '../types/index.js'
+import { TemplateVariables, PayloadID, PayloadRelation } from '../types/index.js'
 
 /**
  * Parse and validate email addresses
@@ -74,6 +74,49 @@ export const sanitizeFromName = (fromName: string | null | undefined): string |
   return sanitized.length > 0 ? sanitized : undefined
 }
 
+/**
+ * Type guard to check if a Payload relation is populated (object) or unpopulated (ID)
+ */
+export const isPopulated = <T extends { id: PayloadID }>(
+  value: PayloadRelation<T> | null | undefined
+): value is T => {
+  return value !== null && value !== undefined && typeof value === 'object' && 'id' in value
+}
+
+/**
+ * Resolves a Payload relation to just the ID
+ * Handles both populated (object with id) and unpopulated (string/number) values
+ */
+export const resolveID = <T extends { id: PayloadID }>(
+  value: PayloadRelation<T> | null | undefined
+): PayloadID | undefined => {
+  if (value === null || value === undefined) return undefined
+
+  if (typeof value === 'string' || typeof value === 'number') {
+    return value
+  }
+
+  if (typeof value === 'object' && 'id' in value) {
+    return value.id
+  }
+
+  return undefined
+}
+
+/**
+ * Resolves an array of Payload relations to an array of IDs
+ * Handles mixed arrays of populated and unpopulated values
+ */
+export const resolveIDs = <T extends { id: PayloadID }>(
+  values: (PayloadRelation<T> | null | undefined)[] | null | undefined
+): PayloadID[] => {
+  if (!values || !Array.isArray(values)) return []
+
+  return values
+    .map(value => resolveID(value))
+    .filter((id): id is PayloadID => id !== undefined)
+}
+
 export const getMailing = (payload: Payload) => {
   const mailing = (payload as any).mailing
   if (!mailing) {

src/utils/jobPolling.ts

@@ -0,0 +1,115 @@
+import { Payload } from 'payload'
+import { JobPollingConfig } from '../types/index.js'
+
+export interface PollForJobIdOptions {
+  payload: Payload
+  collectionSlug: string
+  emailId: string | number
+  config?: JobPollingConfig
+  logger?: {
+    debug: (message: string, ...args: any[]) => void
+    info: (message: string, ...args: any[]) => void
+    warn: (message: string, ...args: any[]) => void
+    error: (message: string, ...args: any[]) => void
+  }
+}
+
+export interface PollForJobIdResult {
+  jobId: string
+  attempts: number
+  elapsedTime: number
+}
+
+// Default job polling configuration values
+const DEFAULT_JOB_POLLING_CONFIG: Required<JobPollingConfig> = {
+  maxAttempts: 5,
+  initialDelay: 25,
+  maxTotalTime: 3000,
+  maxBackoffDelay: 400,
+}
+
+/**
+ * Polls for a job ID associated with an email document using exponential backoff.
+ * This utility handles the complexity of waiting for auto-scheduled jobs to be created.
+ *
+ * The polling mechanism uses exponential backoff with configurable parameters:
+ * - Starts with an initial delay and doubles on each retry
+ * - Caps individual delays at maxBackoffDelay
+ * - Enforces a maximum total polling time
+ *
+ * @param options - Polling options including payload, collection, email ID, and config
+ * @returns Promise resolving to job ID and timing information
+ * @throws Error if job is not found within the configured limits
+ */
+export const pollForJobId = async (options: PollForJobIdOptions): Promise<PollForJobIdResult> => {
+  const { payload, collectionSlug, emailId, logger } = options
+
+  // Merge user config with defaults
+  const config: Required<JobPollingConfig> = {
+    ...DEFAULT_JOB_POLLING_CONFIG,
+    ...options.config,
+  }
+
+  const { maxAttempts, initialDelay, maxTotalTime, maxBackoffDelay } = config
+  const startTime = Date.now()
+  let jobId: string | undefined
+
+  for (let attempt = 0; attempt < maxAttempts; attempt++) {
+    const elapsedTime = Date.now() - startTime
+
+    // Check if we've exceeded the maximum total polling time
+    if (elapsedTime > maxTotalTime) {
+      throw new Error(
+        `Job polling timed out after ${maxTotalTime}ms for email ${emailId}. ` +
+        `The auto-scheduling may have failed or is taking longer than expected.`
+      )
+    }
+
+    // Calculate exponential backoff delay, capped at maxBackoffDelay
+    const delay = Math.min(initialDelay * Math.pow(2, attempt), maxBackoffDelay)
+
+    // Wait before checking (skip on first attempt)
+    if (attempt > 0) {
+      await new Promise(resolve => setTimeout(resolve, delay))
+    }
+
+    // Fetch the email document to check for associated jobs
+    const emailWithJobs = await payload.findByID({
+      collection: collectionSlug,
+      id: emailId,
+    })
+
+    // Check if jobs array exists and has entries
+    if (emailWithJobs.jobs && emailWithJobs.jobs.length > 0) {
+      const firstJob = Array.isArray(emailWithJobs.jobs) ? emailWithJobs.jobs[0] : emailWithJobs.jobs
+      jobId = typeof firstJob === 'string' ? firstJob : String(firstJob.id || firstJob)
+
+      return {
+        jobId,
+        attempts: attempt + 1,
+        elapsedTime: Date.now() - startTime,
+      }
+    }
+
+    // Log progress for attempts after the second try
+    if (attempt >= 2 && logger) {
+      logger.debug(`Waiting for job creation for email ${emailId}, attempt ${attempt + 1}/${maxAttempts}`)
+    }
+  }
+
+  // If we reach here, job was not found
+  const elapsedTime = Date.now() - startTime
+  const timeoutMsg = elapsedTime >= maxTotalTime
+  const errorType = timeoutMsg ? 'POLLING_TIMEOUT' : 'JOB_NOT_FOUND'
+  const baseMessage = timeoutMsg
+    ? `Job polling timed out after ${maxTotalTime}ms for email ${emailId}`
+    : `No processing job found for email ${emailId} after ${maxAttempts} attempts (${elapsedTime}ms)`
+
+  throw new Error(
+    `${errorType}: ${baseMessage}. ` +
+    `This indicates the email was created but job auto-scheduling failed. ` +
+    `The email exists in the database but immediate processing cannot proceed. ` +
+    `You may need to: 1) Check job queue configuration, 2) Verify database hooks are working, ` +
+    `3) Process the email later using processEmailById('${emailId}').`
+  )
+}