From f2e1414c61da04b1d6bfc594328480b8f91a30dc Mon Sep 17 00:00:00 2001 From: Ryann Date: Sat, 22 Mar 2025 02:50:26 +0000 Subject: [PATCH] Fixes and cleanup script --- api/api_server.js | 14 ++ ...322014115_add_ignore_to_mailboxes_alias.js | 23 ++ api/db/models/Alias.js | 3 +- api/db/models/Mailbox.js | 3 +- api/scripts/cleanup.js | 200 ++++++++++++++++++ api/scripts/export_2_cloudflare.js | 20 +- api/test.js | 4 +- api/web_server.js | 26 +-- ...te_mysql_admin.sh => create_mysql_admin.sh | 3 +- docker-compose.yml | 21 +- 10 files changed, 281 insertions(+), 36 deletions(-) create mode 100644 api/db/migrations/20250322014115_add_ignore_to_mailboxes_alias.js create mode 100644 api/scripts/cleanup.js rename api/create_mysql_admin.sh => create_mysql_admin.sh (91%) diff --git a/api/api_server.js b/api/api_server.js index c9d8ffc..3547af7 100644 --- a/api/api_server.js +++ b/api/api_server.js @@ -102,6 +102,20 @@ app.use((err, req, res, next) => { res.status(500).json({ error: 'Something went wrong!' }); }); +const cron = require('node-cron'); +const { cleanupOrphanedMailboxes, cleanupUnmatchedAndExpired, cleanupInactiveMailboxes } = require('./scripts/cleanup'); +cron.schedule('0 0 * * *', async () => { + console.log('Running mailbox cleanup job'); + try { + await cleanupOrphanedMailboxes(); + await cleanupUnmatchedAndExpired(); + await cleanupInactiveMailboxes(); + console.log('Mailbox cleanup completed successfully'); + } catch (error) { + console.error('Mailbox cleanup failed:', error); + } +}); + // Start server app.listen(port, ip, () => { console.log(`API server listening on port ${port}`); diff --git a/api/db/migrations/20250322014115_add_ignore_to_mailboxes_alias.js b/api/db/migrations/20250322014115_add_ignore_to_mailboxes_alias.js new file mode 100644 index 0000000..178a31a --- /dev/null +++ b/api/db/migrations/20250322014115_add_ignore_to_mailboxes_alias.js @@ -0,0 +1,23 @@ +/** + * @param { import("knex").Knex } knex + * @returns { Promise } + */ +exports.up = function(knex) { + return knex.schema.alterTable('mailbox', function (table) { + table.tinyint('ignore').defaultTo(0); + }).alterTable('alias', function (table) { + table.tinyint('ignore').defaultTo(0); + }); +}; + +/** + * @param { import("knex").Knex } knex + * @returns { Promise } + */ +exports.down = function(knex) { + return knex.schema.alterTable('mailbox', function (table) { + table.dropColumn('ignore'); + }).alterTable('alias', function (table) { + table.dropColumn('ignore'); + }); +}; diff --git a/api/db/models/Alias.js b/api/db/models/Alias.js index 89c52c8..af0992e 100644 --- a/api/db/models/Alias.js +++ b/api/db/models/Alias.js @@ -22,7 +22,8 @@ class Alias extends BaseModel { domain: { type: 'string', minLength: 1, maxLength: 255 }, created: { type: 'string', format: 'date-time' }, modified: { type: 'string', format: 'date-time' }, - active: { type: 'integer', default: 1 } + active: { type: 'integer', default: 1 }, + ignore: { type: 'integer', default: 0 } } }; } diff --git a/api/db/models/Mailbox.js b/api/db/models/Mailbox.js index f09c0ab..bcab6b8 100644 --- a/api/db/models/Mailbox.js +++ b/api/db/models/Mailbox.js @@ -33,7 +33,8 @@ class Mailbox extends BaseModel { token: { type: 'string', maxLength: 255 }, token_validity: { type: 'string', format: 'date-time' }, password_expiry: { type: 'string', format: 'date-time' }, - expires: { type: 'string', format: 'date-time' } + expires: { type: 'string', format: 'date-time' }, + ignore: { type: 'integer', default: 0 } } }; } diff --git a/api/scripts/cleanup.js b/api/scripts/cleanup.js new file mode 100644 index 0000000..90cbc32 --- /dev/null +++ b/api/scripts/cleanup.js @@ -0,0 +1,200 @@ +const fs = require('fs').promises; +const path = require('path'); +const { models } = require('../db/db'); +const { Mailbox, Alias } = models; +const { format, isBefore } = require('date-fns'); + +async function scanMailboxDirectories(callback) { + const mailPath = '/var/mail'; + + // Verify mail directory exists + try { + await fs.access(mailPath); + } catch (err) { + console.error(`Mail directory ${mailPath} does not exist or is not accessible`); + return; + } + + const domains = await fs.readdir(mailPath); + console.log(`Found ${domains.length} domains in ${mailPath}`); + + for (const domain of domains) { + const domainPath = path.join(mailPath, domain); + const stat = await fs.stat(domainPath); + + if (stat.isDirectory()) { + const users = await fs.readdir(domainPath); + console.log(`Processing domain ${domain} with ${users.length} users`); + + for (const user of users) { + const userPath = path.join(domainPath, user); + const userStat = await fs.stat(userPath); + + if (userStat.isDirectory()) { + await callback({ user, domain, userPath }); + } + } + } + } +} + +async function cleanupOrphanedMailboxes() { + try { + // Get all valid mailboxes from database - include ignore flag + const validMailboxes = await Mailbox.query() + .select('username', 'ignore'); // Add ignore to selection + + // Create a lookup map for quick checking + const validMailboxMap = {}; + validMailboxes.forEach(mailbox => { + const [user, domain] = mailbox.username.split('@'); + if (!validMailboxMap[domain]) validMailboxMap[domain] = new Map(); + validMailboxMap[domain].set(user, mailbox.ignore); // Store ignore status + }); + + await scanMailboxDirectories(async ({ user, domain, userPath }) => { + // Check if mailbox exists and is not ignored + const hasMailbox = validMailboxMap[domain] && validMailboxMap[domain].has(user); + const isIgnored = hasMailbox && validMailboxMap[domain].get(user) === 1; + + // Only remove if mailbox doesn't exist or is not ignored + if (!hasMailbox || !isIgnored) { + console.log(`Checking mailbox ${user}@${domain} - exists: ${hasMailbox}, ignored: ${isIgnored}`); + console.log(`Removing orphaned mailbox: ${user}@${domain}`); + await fs.rm(userPath, { recursive: true, force: true }); + } + }); + + console.log('Mailbox cleanup completed'); + } catch (error) { + console.error('Error during mailbox cleanup:', error); + } +} + +async function cleanupUnmatchedAndExpired() { + try { + let expired = 0; + let removed = 0; + const mailboxes = await Mailbox.query() + .select('username', 'domain', 'expires', 'ignore') + .where('ignore', 0); + + const aliases = await Alias.query() + .select('address', 'domain', 'ignore') + .where('ignore', 0); + + // Create lookup maps for efficient matching + const mailboxMap = new Map( + mailboxes.map(m => [m.username, m]) + ); + + const aliasMap = new Map( + aliases.map(a => [a.address, a]) + ); + + // Check mailboxes without matching aliases + for (const [mailboxAddress, mailbox] of mailboxMap) { + if (!aliasMap.has(mailboxAddress)) { + console.log(`Removing mailbox without alias: ${mailboxAddress}`); + await Mailbox.query() + .where('username', mailbox.username) + .patch({ active: 0 }); + removed++; + } + } + + // Check aliases without matching mailboxes + for (const [aliasAddress, alias] of aliasMap) { + if (!mailboxMap.has(aliasAddress)) { + console.log(`Removing alias without mailbox: ${aliasAddress}`); + await Alias.query() + .where('address', alias.address) + .patch({ active: 0 }); + removed++; + } + } + + // Check expiration for matched pairs + const currentTime = format(new Date(), 'yyyy-MM-dd HH:mm:ss'); + for (const [mailboxAddress, mailbox] of mailboxMap) { + console.log(`Checking ${mailboxAddress}: expires ${mailbox.expires} vs current ${currentTime}`); + + if (aliasMap.has(mailboxAddress) && isBefore(new Date(mailbox.expires), new Date())) { + console.log(`Removing expired mailbox and alias: ${mailboxAddress}`); + + await Mailbox.query() + .where('username', mailbox.username) + .where('ignore', 0) + .delete(); + + await Alias.query() + .where('address', mailboxAddress) + .where('ignore', 0) + .delete(); + + expired++; + } + } + + await scanMailboxDirectories(async ({ user, domain, userPath }) => { + const mailboxAddress = `${user}@${domain}`; + const mailbox = mailboxMap.get(mailboxAddress); + const alias = aliasMap.get(mailboxAddress); + + // Check if this mailbox exists in database with ignore=1 + const ignoredMailbox = await Mailbox.query() + .where('username', mailboxAddress) + .where('ignore', 1) + .first(); + + // Only remove if mailbox doesn't exist OR (exists, not ignored, and expired) + if (!ignoredMailbox && (!mailbox || !alias || (mailbox && mailbox.expires < currentTime))) { + console.log(`Removing directory for invalid/expired mailbox: ${mailboxAddress}`); + await fs.rm(userPath, { recursive: true, force: true }); + } + }); + + console.log('Cleanup of unmatched and expired items completed'); + } catch (error) { + console.error('Error during cleanup:', error); + } +} + +async function cleanupInactiveMailboxes() { + try { + const mailboxes = await Mailbox.query() + .select('username', 'domain', 'active') + .where('active', 0) + .where('ignore', 0); + + const aliases = await Alias.query() + .select('address', 'domain', 'active') + .where('active', 0) + .where('ignore', 0); + + for (const mailbox of mailboxes) { + await Mailbox.query() + .where('username', mailbox.username) + .where('ignore', 0) + .delete(); + } + + for (const alias of aliases) { + await Alias.query() + .where('address', alias.address) + .where('ignore', 0) + .delete(); + } + + console.log('Cleanup of inactive mailboxes completed'); + } catch (error) { + console.error('Error during cleanup:', error); + } +} + + +module.exports = { + cleanupOrphanedMailboxes, + cleanupUnmatchedAndExpired, + cleanupInactiveMailboxes +}; \ No newline at end of file diff --git a/api/scripts/export_2_cloudflare.js b/api/scripts/export_2_cloudflare.js index cd05e72..066d42e 100644 --- a/api/scripts/export_2_cloudflare.js +++ b/api/scripts/export_2_cloudflare.js @@ -125,7 +125,7 @@ async function configureDNS(domain) { await cf.post(`/zones/${zoneId}/dns_records`, { type: 'MX', name: '@', - content: `mail.${domain}`, + content: `mail.2weekmail.fyi`, priority: 10, ttl: 3600 }); @@ -161,18 +161,18 @@ async function configureDNS(domain) { await cf.post(`/zones/${zoneId}/dns_records`, { type: 'TXT', name: '_dmarc', - content: `"v=DMARC1; p=reject; rua=mailto:postmaster@${domain}"`, + content: `"v=DMARC1; p=reject; rua=mailto:postmaster@2weekmail.fyi"`, ttl: 3600 }); - // Step 8: Add A record for mail subdomain - console.log(`Adding A record for mail.${domain}`); - await cf.post(`/zones/${zoneId}/dns_records`, { - type: 'A', - name: 'mail', - content: process.env.SERVER_IP, - ttl: 3600 - }); + // // Step 8: Add A record for mail subdomain + // console.log(`Adding A record for mail.${domain}`); + // await cf.post(`/zones/${zoneId}/dns_records`, { + // type: 'A', + // name: 'mail', + // content: process.env.SERVER_IP, + // ttl: 3600 + // }); console.log(`DNS configuration completed for ${domain}`); return true; diff --git a/api/test.js b/api/test.js index bd2bae8..d3a4289 100644 --- a/api/test.js +++ b/api/test.js @@ -1,3 +1,3 @@ -const uuid = require('uuid'); +const { format } = require('date-fns'); -console.log(uuid.v4()); \ No newline at end of file +console.log(format(new Date(), 'yyyy-MM-dd HH:mm:ss')); \ No newline at end of file diff --git a/api/web_server.js b/api/web_server.js index 2e155bd..f9431ce 100644 --- a/api/web_server.js +++ b/api/web_server.js @@ -73,21 +73,21 @@ app.get('/terms', (req, res) => { const setupSwagger = require('./swagger'); setupSwagger(app); -const { deleteExpiredMailboxes } = require('./utils/mailbox_cleanup'); -const { cleanupOrphanedMailboxes } = require('./utils/helpers'); +// const { deleteExpiredMailboxes } = require('./utils/mailbox_cleanup'); +// const { cleanupOrphanedMailboxes } = require('./utils/helpers'); -const cron = require('node-cron'); +// const cron = require('node-cron'); -cron.schedule('0 0 * * *', async () => { - console.log('Running mailbox cleanup job'); - try { - await deleteExpiredMailboxes(); - await cleanupOrphanedMailboxes(); - console.log('Mailbox cleanup completed successfully'); - } catch (error) { - console.error('Mailbox cleanup failed:', error); - } -}); +// cron.schedule('0 0 * * *', async () => { +// console.log('Running mailbox cleanup job'); +// try { +// await deleteExpiredMailboxes(); +// await cleanupOrphanedMailboxes(); +// console.log('Mailbox cleanup completed successfully'); +// } catch (error) { +// console.error('Mailbox cleanup failed:', error); +// } +// }); // Start server app.listen(PORT, () => { diff --git a/api/create_mysql_admin.sh b/create_mysql_admin.sh similarity index 91% rename from api/create_mysql_admin.sh rename to create_mysql_admin.sh index b686386..29a8651 100755 --- a/api/create_mysql_admin.sh +++ b/create_mysql_admin.sh @@ -2,7 +2,8 @@ # Variables - replace these with your desired values ADMIN_USER="admin" -ADMIN_PASSWORD="CHANGEME" +ADMIN_PASSWORD=$(openssl rand -base64 32) +DB_PASS="ZSI8R1LFJVOqX65EmBsBA1OQPO2arA==" CONTAINER_NAME="mailserver_db" echo "Creating MySQL admin user with root-like privileges..." diff --git a/docker-compose.yml b/docker-compose.yml index f5e64c9..5fee9c1 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,5 +1,3 @@ -version: '3.8' - services: # Database db: @@ -45,9 +43,12 @@ services: - ./certs:/tmp/docker-mailserver/certs - /etc/letsencrypt:/etc/letsencrypt environment: + - OVERRIDE_HOSTNAME=mail.2weekmail.fyi + - SPOOF_PROTECTION=1 + - LOGROTATE_INTERVAL=daily - ENABLE_SPAMASSASSIN=0 - ENABLE_CLAMAV=0 - - ENABLE_POSTGREY=1 + - ENABLE_POSTGREY=0 - POSTMASTER_ADDRESS=admin@2weekmail.fyi - POSTFIX_MYSQL_HOST=db - POSTFIX_MYSQL_USER=${DB_USER} @@ -60,9 +61,12 @@ services: - ENABLE_POSTFIX_VIRTUAL_TRANSPORT=1 - POSTFIX_DAGENT=lmtp:unix:/var/run/dovecot/lmtp - SSL_TYPE=letsencrypt - - SSL_CERT_PATH=/tmp/docker-mailserver/certs/2weekmail.test-cert.pem - - SSL_KEY_PATH=/tmp/docker-mailserver/certs/2weekmail.test-key.pem + # - SSL_CERT_PATH=/tmp/docker-mailserver/certs/2weekmail.test-cert.pem + # - SSL_KEY_PATH=/tmp/docker-mailserver/certs/2weekmail.test-key.pem - TZ=UTC + cap_add: + - NET_ADMIN + - SYS_PTRACE networks: - mail_network depends_on: @@ -86,7 +90,7 @@ services: - POSTFIXADMIN_SMTP_PORT=25 - POSTFIXADMIN_CONFIGURED=true volumes: - - postfixadmin_data:/var/www/html/templates_c + - postfixadmin_data:/var/www/html - ./config/postfixadmin:/var/www/html - ./logs/postfixadmin:/var/log/apache2 networks: @@ -110,9 +114,9 @@ services: - ROUNDCUBEMAIL_DB_USER=${DB_USER} - ROUNDCUBEMAIL_DB_PASSWORD=${DB_PASS} - ROUNDCUBEMAIL_DB_NAME=${DB_NAME} - - ROUNDCUBEMAIL_DEFAULT_HOST=ssl://mailserver + - ROUNDCUBEMAIL_DEFAULT_HOST=ssl://mail.2weekmail.fyi - ROUNDCUBEMAIL_DEFAULT_PORT=993 - - ROUNDCUBEMAIL_SMTP_SERVER=tls://mailserver + - ROUNDCUBEMAIL_SMTP_SERVER=tls://mail.2weekmail.fyi - ROUNDCUBEMAIL_SMTP_PORT=587 volumes: - roundcube_data:/var/www/html @@ -155,6 +159,7 @@ services: - ./api:/app - /app/node_modules - opendkim_data:/etc/opendkim + - mailserver_data:/var/mail:rw environment: - PORT=${PORT} - WEB_PORT=${WEB_PORT}