From 04c49956fe15eb371671ae3c3f95ad241de2d516 Mon Sep 17 00:00:00 2001 From: Ryahn Date: Mon, 3 Feb 2025 18:54:33 -0600 Subject: [PATCH] fixed cron --- .gitignore | 5 +- app.js | 37 +---- src/db/seeds/test_cleanup.js | 164 ++++++++++++++++++++ src/email_server/services/MessageService.js | 48 ++++-- src/routes/email.js | 2 +- src/utils/functions.js | 61 ++++++-- 6 files changed, 258 insertions(+), 59 deletions(-) create mode 100644 src/db/seeds/test_cleanup.js diff --git a/.gitignore b/.gitignore index 842a5aa..2bf6d47 100644 --- a/.gitignore +++ b/.gitignore @@ -4,4 +4,7 @@ haraka.js main.js *.log tls_key.pem -tls_cert.pem \ No newline at end of file +tls_cert.pem +.DS_Store +._* +deploy.sh \ No newline at end of file diff --git a/app.js b/app.js index e7881b0..3f9b77b 100644 --- a/app.js +++ b/app.js @@ -10,7 +10,6 @@ const config = require('./src/config/main'); const path = require('path'); const cors = require('cors'); - const swaggerOptions = { definition: { openapi: '3.1.0', @@ -68,35 +67,13 @@ app.get('/twitter-card.svg', (req, res) => { }); // Schedule cleanup job -// cron.schedule('*/10 * * * *', async () => { -// try { -// await MessageService.cleanup(); -// console.log('Daily cleanup completed'); -// } catch (error) { -// console.error('Cleanup failed:', error); -// } -// }); - - -app.get('/api/stats/total', (req, res) => { - // Replace this with your actual database query - res.json({ - emailsCreated: 152847, - messagesReceived: 892365 - }); -}); - -app.get('/api/stats/timeseries', (req, res) => { - // Replace this with your actual database query - res.json([ - { name: 'Mon', emails: 1240, messages: 5430 }, - { name: 'Tue', emails: 1580, messages: 6210 }, - { name: 'Wed', emails: 1890, messages: 7840 }, - { name: 'Thu', emails: 2090, messages: 8120 }, - { name: 'Fri', emails: 1870, messages: 6980 }, - { name: 'Sat', emails: 1450, messages: 5640 }, - { name: 'Sun', emails: 1320, messages: 4980 } - ]); +cron.schedule('*/10 * * * *', async () => { + try { + await MessageService.cleanup(); + console.log('Daily cleanup completed'); + } catch (error) { + console.error('Cleanup failed:', error); + } }); app.use(express.static(path.join(__dirname, 'client/build'))); diff --git a/src/db/seeds/test_cleanup.js b/src/db/seeds/test_cleanup.js new file mode 100644 index 0000000..68bca11 --- /dev/null +++ b/src/db/seeds/test_cleanup.js @@ -0,0 +1,164 @@ +const { mysqlSafeTimestamp, generateUniqueName } = require('../../utils/functions'); +const MessageService = require('../../email_server/services/MessageService'); + +/** + * @param { import("knex").Knex } knex + * @returns { Promise } + */ +exports.seed = async function(knex) { + try { + console.log('Starting cleanup test seeder...'); + + // Clean up existing test data first + await knex('messages').where('created_at', '<', mysqlSafeTimestamp()).delete(); + await knex('temp_emails').where('created_at', '<', mysqlSafeTimestamp()).delete(); + + // Create or get test user + await knex('users') + .insert({ + email: 'test@cleanup.com', + password: '$2b$10$abcdefghijklmnopqrstuvwxyz12345', + is_admin: false + }) + .onConflict('email') + .merge(); + + const testUser = await knex('users') + .where('email', 'test@cleanup.com') + .first(); + + // Get a domain for test emails + const domain = await knex('domains').where('active', true).first(); + if (!domain) throw new Error('No active domains found'); + + // Create temp emails that should be cleaned up (already expired) + const tempEmails = []; + for (let i = 0; i < 5; i++) { + // Created 15 days ago, expired yesterday + const createdAt = mysqlSafeTimestamp(false, -15); + const expiresAt = mysqlSafeTimestamp(false, -1); // Set to yesterday + + console.log(`Creating expired email with created_at: ${createdAt}, expires_at: ${expiresAt}`); + + const email = `${generateUniqueName()}@${domain.name}`; + await knex('temp_emails') + .insert({ + email: email, + user_id: testUser.id, + expires_at: expiresAt, + created_at: createdAt, + updated_at: createdAt + }); + + const tempEmail = await knex('temp_emails') + .where('email', email) + .first(); + + tempEmails.push(tempEmail); + } + + // Create messages for expired emails + for (const tempEmail of tempEmails) { + const messageCount = Math.floor(Math.random() * 3) + 1; + for (let i = 0; i < messageCount; i++) { + const messageDate = mysqlSafeTimestamp(false, -15); + await knex('messages').insert({ + temp_email_id: tempEmail.id, + from: 'sender@example.com', + to: tempEmail.email, + subject: 'Old Test Message', + body: 'This is an old test message that should be cleaned up.', + headers: JSON.stringify({ + 'message-id': ``, + 'date': new Date().toISOString() + }), + created_at: messageDate, + updated_at: messageDate, + received_at: messageDate + }); + } + } + + // Create control group - emails that should NOT be cleaned up + for (let i = 0; i < 2; i++) { + const createdAt = mysqlSafeTimestamp(false, -7); // 7 days ago + const expiresAt = mysqlSafeTimestamp(false, 7); // expires in 7 days + + console.log(`Creating active email with created_at: ${createdAt}, expires_at: ${expiresAt}`); + + const email = `${generateUniqueName()}@${domain.name}`; + await knex('temp_emails') + .insert({ + email: email, + user_id: testUser.id, + expires_at: expiresAt, + created_at: createdAt, + updated_at: createdAt + }); + + const tempEmail = await knex('temp_emails') + .where('email', email) + .first(); + + // Create recent messages + const messageCount = Math.floor(Math.random() * 2) + 1; + for (let j = 0; j < messageCount; j++) { + await knex('messages').insert({ + temp_email_id: tempEmail.id, + from: 'sender@example.com', + to: tempEmail.email, + subject: 'Recent Test Message', + body: 'This is a recent test message that should NOT be cleaned up.', + headers: JSON.stringify({ + 'message-id': ``, + 'date': new Date().toISOString() + }), + created_at: mysqlSafeTimestamp(false, -7), + updated_at: mysqlSafeTimestamp(false, -7), + received_at: mysqlSafeTimestamp(false, -7) + }); + } + } + + // Query and display all temp emails + const emailDebug = await knex('temp_emails') + .select('id', 'email', 'created_at', 'expires_at') + .orderBy('created_at'); + + console.log('\nCurrent temp_emails in database:'); + emailDebug.forEach(email => { + console.log(`ID: ${email.id}, Email: ${email.email}`); + console.log(`Created: ${email.created_at}, Expires: ${email.expires_at}\n`); + }); + + // Log counts and run cleanup + const beforeCounts = await getRecordCounts(knex); + console.log('\nBefore cleanup:', beforeCounts); + + const cleanupResults = await MessageService.cleanup(); + console.log('Cleanup results:', cleanupResults); + + const afterCounts = await getRecordCounts(knex); + console.log('After cleanup:', afterCounts); + console.log('Records cleaned up:', { + tempEmails: beforeCounts.tempEmails - afterCounts.tempEmails, + messages: beforeCounts.messages - afterCounts.messages + }); + + } catch (error) { + console.error('Seeder failed:', error); + throw error; + } +}; + +async function getRecordCounts(knex) { + const [tempEmails, messages] = await Promise.all([ + knex('temp_emails').count('* as count').first(), + knex('messages').count('* as count').first() + ]); + + return { + tempEmails: parseInt(tempEmails.count), + messages: parseInt(messages.count) + }; +} \ No newline at end of file diff --git a/src/email_server/services/MessageService.js b/src/email_server/services/MessageService.js index 0f136b8..7de2c02 100644 --- a/src/email_server/services/MessageService.js +++ b/src/email_server/services/MessageService.js @@ -8,6 +8,7 @@ const Knex = require("knex"); const knexConfig = require("../../config/database"); const knex = Knex(knexConfig.development); Model.knex(knex); +const { mysqlTimestampCompare, mysqlSafeTimestamp } = require("../../utils/functions"); class MessageService { static async isValidDomain(email) { @@ -46,39 +47,52 @@ class MessageService { } static async cleanup() { - const now = new Date().toISOString(); - const cutoffDate = new Date(); - cutoffDate.setDate(cutoffDate.getDate() - config.storage.retention); + const now = mysqlSafeTimestamp(); + console.log('Current time:', now); + // Find all expired temp emails const expiredEmails = await TempEmail.query() - .where("expires_at", "<", now) - .select("id"); + .select('id', 'created_at', 'expires_at') + .where('expires_at', '<', now); + + console.log('Found expired emails:', expiredEmails.length); + console.log('Expired emails:', expiredEmails); - const expiredEmailIds = expiredEmails.map((email) => email.id); - - if (expiredEmailIds.length > 0) { - console.log("Deleting messages for expired emails"); + if (expiredEmails.length > 0) { + // Delete associated messages first const deletedMessages = await Message.query() - .whereIn("temp_email_id", expiredEmailIds) + .whereIn('temp_email_id', expiredEmails.map(email => email.id)) .delete(); + console.log('Deleted associated messages:', deletedMessages); - console.log("Deleting expired temp emails"); + // Delete the expired temp emails const deletedTempEmails = await TempEmail.query() - .whereIn("id", expiredEmailIds) + .whereIn('id', expiredEmails.map(email => email.id)) .delete(); + console.log('Deleted expired emails:', deletedTempEmails); + // Delete old messages (using created_at) + const cutoffDate = mysqlSafeTimestamp(false, -14); // 14 days ago const expiredMessages = await Message.query() - .where("created_at", "<", cutoffDate) + .where('created_at', '<', cutoffDate) .delete(); + console.log('Deleted old messages:', expiredMessages); - console.log("Deleted messages:", deletedMessages + expiredMessages); - console.log("Deleted emails:", deletedTempEmails); - return { deletedMessages, deletedTempEmails }; + return { + deletedMessages: deletedMessages + expiredMessages, + deletedTempEmails, + expiredEmailsFound: expiredEmails.length + }; } - return { deletedMessages: 0, deletedTempEmails: 0 }; + return { + deletedMessages: 0, + deletedTempEmails: 0, + expiredEmailsFound: 0 + }; } + static async search(params) { let query = Message.query() .joinRelated("temp_email") diff --git a/src/routes/email.js b/src/routes/email.js index 9dae088..5bd0d97 100644 --- a/src/routes/email.js +++ b/src/routes/email.js @@ -158,7 +158,7 @@ router.post('/generate', async (req, res) => { const tempEmail = await TempEmail.query().insert({ email: randomDomainName, user_id: req.user.id, - expires_at: mysqlSafeTimestamp(14) + expires_at: mysqlSafeTimestamp(false, 14) }); await DailyStats.incrementEmailCount(); diff --git a/src/utils/functions.js b/src/utils/functions.js index 5839ea4..fe66761 100644 --- a/src/utils/functions.js +++ b/src/utils/functions.js @@ -2,23 +2,63 @@ const { uniqueNamesGenerator, adjectives, animals, colors, names, languages, sta const Domain = require('../db/models/Domain'); /** + * + * @param {boolean} dateOnly - If true, return the date only * @param {number} days - Number of days to add to the current date * @returns {string} - MySQL safe timestamp */ -function mysqlSafeTimestamp(dateOnly = false, days) { - if (dateOnly && days) { - return new Date(new Date().setDate(new Date().getDate() + days)).toISOString().split('T')[0]; - } - +function mysqlSafeTimestamp(dateOnly = false, days = 0) { + const date = new Date(); if (days) { - return new Date(new Date().setDate(new Date().getDate() + days)).toISOString().replace('T', ' ').replace(/\.\d{3}Z$/, ''); + date.setDate(date.getDate() + days); } - + if (dateOnly) { - return new Date().toISOString().split('T')[0]; + // Return YYYY-MM-DD format + return date.toISOString().split('T')[0]; } + + // Return YYYY-MM-DD HH:mm:ss format + return date.toISOString() + .replace('T', ' ') + .replace(/\.\d{3}Z$/, ''); +} - return new Date().toISOString().replace('T', ' ').replace(/\.\d{3}Z$/, ''); +/** + * + * @param {string} created_at - The created_at timestamp + * @param {string} expires_at - The expires_at timestamp + * @returns {boolean} - True if the expires_at is today and the difference between created_at and expires_at is 14 days + */ +function mysqlTimestampCompare(created_at, expires_at) { + if (!created_at || !expires_at) return false; + + const parseDate = (mysqlDateTime) => { + if (mysqlDateTime instanceof Date) return mysqlDateTime; + if (typeof mysqlDateTime === 'string') { + // Handle MySQL datetime format: YYYY-MM-DD HH:mm:ss + const [datePart, timePart] = mysqlDateTime.split(' '); + return new Date(`${datePart}T${timePart}.000Z`); + } + return new Date(mysqlDateTime); + }; + + const createdDate = parseDate(created_at); + const expiresDate = parseDate(expires_at); + const today = new Date(); + + // Check if expires_at is today + const isExpiresToday = ( + expiresDate.getFullYear() === today.getFullYear() && + expiresDate.getMonth() === today.getMonth() && + expiresDate.getDate() === today.getDate() + ); + + // Calculate days difference + const diffTime = expiresDate - createdDate; + const diffDays = diffTime / (1000 * 60 * 60 * 24); + + return isExpiresToday && Math.floor(diffDays) === 14; } function generateUniqueName() { @@ -70,5 +110,6 @@ async function getRandomDomain(safeTLD = false) { module.exports = { mysqlSafeTimestamp, generateUniqueName, - getRandomDomain + getRandomDomain, + mysqlTimestampCompare } \ No newline at end of file