VPS 4 Deploy Agent
297 lines
13 KiB
TypeScript
297 lines
13 KiB
TypeScript
import { S3Client, PutObjectCommand, GetObjectCommand, HeadBucketCommand, ListObjectsV2Command } from '@aws-sdk/client-s3';
|
|
import { Upload } from '@aws-sdk/lib-storage';
|
|
import { PluginInstance, PluginContext } from '../../backend/src/core/types';
|
|
import { storageProvider, StorageUploadPayload, StorageProviderInterface } from '../../backend/src/core/StorageProvider';
|
|
import { pluginConfig } from '../../backend/src/core/plugin-config';
|
|
import sharp from 'sharp';
|
|
import fs from 'fs';
|
|
import path from 'path';
|
|
|
|
const VALID_CATEGORIES = ['cdi', 'xrays', 'documents', 'exports', 'posts', 'cards', 'partners', 'whatsapp', 'media'];
|
|
|
|
class UnifiedStorageProviderPlugin implements PluginInstance, StorageProviderInterface {
|
|
manifest: any;
|
|
private s3: S3Client | null = null;
|
|
private bucket: string = '';
|
|
private prefix: string = '';
|
|
private localFallbackDir: string = path.resolve(process.cwd(), 'storage', 'local_fallback');
|
|
private ctx!: PluginContext;
|
|
|
|
/** Activation logic */
|
|
async activate(ctx: PluginContext): Promise<void> {
|
|
this.ctx = ctx;
|
|
const storedConfig = pluginConfig.get('UnifiedStorageProvider') || {};
|
|
|
|
// Ensure local fallback directory exists
|
|
if (!fs.existsSync(this.localFallbackDir)) {
|
|
fs.mkdirSync(this.localFallbackDir, { recursive: true });
|
|
}
|
|
|
|
const accessKey = storedConfig.wasabiAccessKey || process.env.WASABI_ACCESS_KEY;
|
|
const secretKey = storedConfig.wasabiSecretKey || process.env.WASABI_SECRET_KEY;
|
|
const region = storedConfig.wasabiRegion || process.env.WASABI_REGION || 'us-east-1';
|
|
this.bucket = storedConfig.wasabiBucket || process.env.WASABI_BUCKET || '';
|
|
this.prefix = storedConfig.wasabiBucketPrefix || process.env.WASABI_BUCKET_PREFIX || '';
|
|
|
|
const rawEndpoint = storedConfig.wasabiEndpoint || process.env.WASABI_ENDPOINT || 's3.wasabisys.com';
|
|
const endpoint = rawEndpoint.startsWith('http') ? rawEndpoint : `https://${rawEndpoint}`;
|
|
|
|
if (!storedConfig['category_bucket_whatsapp']) {
|
|
storedConfig['category_bucket_whatsapp'] = this.bucket || 'clube67-storage';
|
|
pluginConfig.set('UnifiedStorageProvider', storedConfig);
|
|
}
|
|
|
|
if (!accessKey || !secretKey || !this.bucket) {
|
|
ctx.logger.error('Wasabi credentials or bucket missing in .env or plugin config');
|
|
} else {
|
|
this.s3 = new S3Client({
|
|
region,
|
|
endpoint,
|
|
credentials: {
|
|
accessKeyId: accessKey,
|
|
secretAccessKey: secretKey,
|
|
},
|
|
forcePathStyle: true,
|
|
});
|
|
|
|
storageProvider.register(this);
|
|
ctx.logger.info(`UnifiedStorageProvider activated (Endpoint: ${endpoint})`);
|
|
|
|
// start the background worker
|
|
this.startSyncWorker();
|
|
}
|
|
|
|
// --- Proxy Route ---
|
|
ctx.app.get('/api/storage/view/*', async (req, res) => {
|
|
const filePath = req.params[0];
|
|
if (!filePath) return res.status(400).send('Path missing');
|
|
|
|
try {
|
|
// 1. Check local fallback
|
|
const localPath = path.join(this.localFallbackDir, filePath);
|
|
if (fs.existsSync(localPath)) {
|
|
const data = fs.readFileSync(localPath);
|
|
const ext = filePath.split('.').pop()?.toLowerCase();
|
|
const mimeTypes: any = { 'jpg': 'image/jpeg', 'jpeg': 'image/jpeg', 'png': 'image/png', 'webp': 'image/webp', 'pdf': 'application/pdf' };
|
|
res.setHeader('Content-Type', mimeTypes[ext] || 'application/octet-stream');
|
|
res.setHeader('X-Storage-Source', 'local_fallback');
|
|
res.setHeader('Cache-Control', 'public, max-age=31536000');
|
|
return res.send(data);
|
|
}
|
|
|
|
// 2. Wasabi Logic
|
|
if (!this.s3 || !this.bucket) return res.status(500).send('Storage not initialized');
|
|
|
|
const category = filePath.split('/')[0];
|
|
const storedConfig = pluginConfig.get('UnifiedStorageProvider') || {};
|
|
const mappedBucket = storedConfig[`category_bucket_${category}`] || this.bucket;
|
|
const legacyBucket = (category === 'whatsapp') ? 'clube67-whatsapp-v1' : null;
|
|
|
|
const tryFetch = async (bucket: string) => {
|
|
return this.s3!.send(new GetObjectCommand({ Bucket: bucket, Key: filePath }));
|
|
};
|
|
|
|
let data;
|
|
try {
|
|
data = await tryFetch(mappedBucket);
|
|
} catch (e: any) {
|
|
if (legacyBucket && legacyBucket !== mappedBucket) {
|
|
try { data = await tryFetch(legacyBucket); } catch (e2) { throw e; }
|
|
} else { throw e; }
|
|
}
|
|
|
|
const ext = filePath.split('.').pop()?.toLowerCase();
|
|
const mimeTypes: any = { 'jpg': 'image/jpeg', 'jpeg': 'image/jpeg', 'png': 'image/png', 'gif': 'image/gif', 'webp': 'image/webp', 'svg': 'image/svg+xml', 'pdf': 'application/pdf' };
|
|
res.setHeader('Content-Type', data.ContentType || mimeTypes[ext] || 'application/octet-stream');
|
|
res.setHeader('X-Storage-Source', 'wasabi');
|
|
res.setHeader('Cache-Control', 'public, max-age=31536000');
|
|
|
|
// In v3, Body has utility methods to convert the stream
|
|
if (data.Body) {
|
|
const bytes = await (data.Body as any).transformToByteArray();
|
|
res.send(Buffer.from(bytes));
|
|
} else {
|
|
res.status(404).send('Body empty');
|
|
}
|
|
} catch (err: any) {
|
|
res.status(404).send('File not found');
|
|
}
|
|
});
|
|
|
|
// --- Admin/Fetch Routes ---
|
|
ctx.app.post('/api/storage/list-files', async (req, res) => {
|
|
const { accessKey, secretKey, bucket, prefix, region, endpoint } = req.body;
|
|
if (!accessKey || !secretKey || !bucket) return res.status(400).json({ error: 'Params missing' });
|
|
try {
|
|
const s3 = new S3Client({
|
|
region: region || 'us-east-1',
|
|
endpoint: endpoint || 'https://s3.wasabisys.com',
|
|
credentials: { accessKeyId: accessKey, secretAccessKey: secretKey },
|
|
forcePathStyle: true
|
|
});
|
|
const data = await s3.send(new ListObjectsV2Command({ Bucket: bucket, Prefix: prefix || '', MaxKeys: 20 }));
|
|
const files = data.Contents?.map(item => ({ key: item.Key, size: item.Size, lastModified: item.LastModified })) || [];
|
|
res.json({ files });
|
|
} catch (err: any) { res.status(500).json({ error: err.message }); }
|
|
});
|
|
|
|
setInterval(() => { storageProvider.updateHealth(); }, 30000);
|
|
}
|
|
|
|
async deactivate(ctx: PluginContext): Promise<void> { ctx.logger.info('UnifiedStorageProvider deactivated'); }
|
|
|
|
async checkHealth(): Promise<boolean> {
|
|
if (!this.s3 || !this.bucket) return false;
|
|
try { await this.s3.send(new HeadBucketCommand({ Bucket: this.bucket })); return true; }
|
|
catch (err) { return false; }
|
|
}
|
|
|
|
async upload(payload: StorageUploadPayload): Promise<{ provider: string; path: string }> {
|
|
const storedConfig = pluginConfig.get('UnifiedStorageProvider') || {};
|
|
const { category, file } = payload;
|
|
if (!VALID_CATEGORIES.includes(category)) throw new Error(`Invalid category: ${category}`);
|
|
|
|
let buffer = file.buffer;
|
|
let originalName = file.originalname;
|
|
let mimetype = file.mimetype;
|
|
|
|
if (mimetype.startsWith('image/') && buffer.length > 150 * 1024) {
|
|
try {
|
|
this.ctx.logger.info(`[Optimizer] Processing ${originalName}`);
|
|
const optimized = await sharp(buffer)
|
|
.resize(1200, 1200, { fit: 'inside', withoutEnlargement: true })
|
|
.webp({ quality: 75 })
|
|
.toBuffer();
|
|
buffer = optimized;
|
|
mimetype = 'image/webp';
|
|
originalName = originalName.replace(/\.[^.]+$/, '.webp');
|
|
} catch (err: any) { this.ctx.logger.warn(`[Optimizer] Failed: ${err.message}`); }
|
|
}
|
|
|
|
const fileName = `${Date.now()}-${originalName.replace(/[^a-zA-Z0-9.-]/g, '_')}`;
|
|
const relativePath = `${category}/${fileName}`;
|
|
|
|
try {
|
|
if (!this.s3 || !this.bucket) throw new Error('S3 not initialized');
|
|
const targetBucket = storedConfig[`category_bucket_${category}`] || this.bucket;
|
|
|
|
const parallelUploads3 = new Upload({
|
|
client: this.s3,
|
|
params: {
|
|
Bucket: targetBucket,
|
|
Key: relativePath,
|
|
Body: buffer,
|
|
ContentType: mimetype,
|
|
ACL: 'public-read'
|
|
},
|
|
});
|
|
|
|
await parallelUploads3.done();
|
|
return { provider: 'wasabi', path: relativePath };
|
|
} catch (err: any) {
|
|
this.ctx.logger.error(`[Wasabi] Fallback to LOCAL: ${err.message}`);
|
|
|
|
// Log fallback attempt
|
|
try {
|
|
await this.ctx.db('janitor_logs').insert({
|
|
filename: originalName,
|
|
category: category,
|
|
status: 'fallback',
|
|
message: `Erro Wasabi: ${err.message.substring(0, 100)}`
|
|
});
|
|
} catch (dbErr) { }
|
|
|
|
const localDest = path.join(this.localFallbackDir, category);
|
|
if (!fs.existsSync(localDest)) fs.mkdirSync(localDest, { recursive: true });
|
|
fs.writeFileSync(path.join(this.localFallbackDir, relativePath), buffer);
|
|
return { provider: 'local_fallback', path: relativePath };
|
|
}
|
|
}
|
|
|
|
private startSyncWorker() {
|
|
setInterval(async () => {
|
|
if (!this.s3 || !this.bucket) return;
|
|
const scanFolders = [
|
|
{ path: this.localFallbackDir, isCategorized: true },
|
|
{ path: path.resolve(process.cwd(), 'uploads'), isCategorized: false, category: 'media' }
|
|
];
|
|
for (const folder of scanFolders) {
|
|
try {
|
|
if (!fs.existsSync(folder.path)) continue;
|
|
if (folder.isCategorized) {
|
|
const categories = fs.readdirSync(folder.path);
|
|
for (const cat of categories) {
|
|
const catDir = path.join(folder.path, cat);
|
|
if (!fs.statSync(catDir).isDirectory()) continue;
|
|
await this.syncFolder(catDir, cat);
|
|
}
|
|
} else if (folder.category) {
|
|
await this.syncFolder(folder.path, folder.category);
|
|
}
|
|
} catch (err: any) { }
|
|
}
|
|
}, 120000);
|
|
}
|
|
|
|
private async syncFolder(dir: string, category: string) {
|
|
if (!this.s3 || !this.bucket) return;
|
|
try {
|
|
const files = fs.readdirSync(dir);
|
|
if (files.length === 0) return;
|
|
for (const file of files) {
|
|
const localPath = path.join(dir, file);
|
|
if (!fs.existsSync(localPath) || fs.statSync(localPath).isDirectory()) continue;
|
|
|
|
try {
|
|
// Log step 1: Wasabi
|
|
const logId = await this.ctx.db('janitor_logs').insert({
|
|
filename: file,
|
|
category: category,
|
|
status: 'wasabi',
|
|
message: 'Sincronizando com Wasabi...'
|
|
});
|
|
|
|
const buffer = fs.readFileSync(localPath);
|
|
const targetBucket = pluginConfig.get('UnifiedStorageProvider')?.[`category_bucket_${category}`] || this.bucket;
|
|
|
|
const parallelUploads3 = new Upload({
|
|
client: this.s3,
|
|
params: {
|
|
Bucket: targetBucket,
|
|
Key: `${category}/${file}`,
|
|
Body: buffer
|
|
},
|
|
});
|
|
|
|
await parallelUploads3.done();
|
|
|
|
// Log step 2: Deleting VPS
|
|
await this.ctx.db('janitor_logs').where('id', logId[0]).update({
|
|
status: 'vps_delete',
|
|
message: 'Wasabi OK, excluindo do servidor...'
|
|
});
|
|
|
|
fs.unlinkSync(localPath);
|
|
|
|
// Log step 3: Fim
|
|
await this.ctx.db('janitor_logs').where('id', logId[0]).update({
|
|
status: 'fim',
|
|
message: 'Limpeza concluída com sucesso.'
|
|
});
|
|
} catch (err: any) {
|
|
this.ctx.logger.error(`[Janitor] Sync failed for ${file}: ${err.message}`);
|
|
await this.ctx.db('janitor_logs').insert({
|
|
filename: file,
|
|
category: category,
|
|
status: 'erro',
|
|
message: `Falha: ${err.message.substring(0, 100)}`
|
|
});
|
|
}
|
|
}
|
|
} catch (err: any) { }
|
|
}
|
|
}
|
|
|
|
const instance = new UnifiedStorageProviderPlugin();
|
|
export default instance;
|