Initial commit: LogMaster with Viewer Collector Monitor modes
This commit is contained in:
448
dist-server/server.js
Normal file
448
dist-server/server.js
Normal file
@@ -0,0 +1,448 @@
|
||||
"use strict";
|
||||
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
|
||||
if (k2 === undefined) k2 = k;
|
||||
var desc = Object.getOwnPropertyDescriptor(m, k);
|
||||
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
|
||||
desc = { enumerable: true, get: function() { return m[k]; } };
|
||||
}
|
||||
Object.defineProperty(o, k2, desc);
|
||||
}) : (function(o, m, k, k2) {
|
||||
if (k2 === undefined) k2 = k;
|
||||
o[k2] = m[k];
|
||||
}));
|
||||
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
|
||||
Object.defineProperty(o, "default", { enumerable: true, value: v });
|
||||
}) : function(o, v) {
|
||||
o["default"] = v;
|
||||
});
|
||||
var __importStar = (this && this.__importStar) || (function () {
|
||||
var ownKeys = function(o) {
|
||||
ownKeys = Object.getOwnPropertyNames || function (o) {
|
||||
var ar = [];
|
||||
for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
|
||||
return ar;
|
||||
};
|
||||
return ownKeys(o);
|
||||
};
|
||||
return function (mod) {
|
||||
if (mod && mod.__esModule) return mod;
|
||||
var result = {};
|
||||
if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
|
||||
__setModuleDefault(result, mod);
|
||||
return result;
|
||||
};
|
||||
})();
|
||||
Object.defineProperty(exports, "__esModule", { value: true });
|
||||
const http = __importStar(require("http"));
|
||||
const fs = __importStar(require("fs"));
|
||||
const path = __importStar(require("path"));
|
||||
const url = __importStar(require("url"));
|
||||
const logSourceManager_1 = require("./logSourceManager");
|
||||
const configManager_1 = require("./configManager");
|
||||
const collector_1 = require("./collector");
|
||||
const s3Source_1 = require("./s3Source");
|
||||
const configFile_1 = require("./configFile");
|
||||
const monitor_1 = require("./monitor");
|
||||
const configManager = new configManager_1.ConfigManager();
|
||||
const logSourceManager = new logSourceManager_1.LogSourceManager(configManager);
|
||||
// Mode: "viewer" | "collector" | "monitor" | "both" | "all"
|
||||
const MODE = (process.env.LOG_MODE || 'both').toLowerCase();
|
||||
const PORT = parseInt(process.env.PORT || '8080', 10);
|
||||
const HOST = process.env.HOST || '0.0.0.0';
|
||||
// Collector setup
|
||||
let collector = null;
|
||||
if (MODE === 'collector' || MODE === 'both') {
|
||||
let forwardTargets = [];
|
||||
try {
|
||||
forwardTargets = JSON.parse(process.env.COLLECTOR_FORWARD_TARGETS || '[]');
|
||||
}
|
||||
catch { }
|
||||
collector = new collector_1.LogCollector({
|
||||
storageDir: process.env.COLLECTOR_STORAGE_DIR || path.join(process.cwd(), 'collector-data'),
|
||||
maxLinesPerStream: parseInt(process.env.COLLECTOR_MAX_LINES || '100000', 10),
|
||||
maxStreams: parseInt(process.env.COLLECTOR_MAX_STREAMS || '50', 10),
|
||||
udpEnabled: process.env.COLLECTOR_UDP_ENABLED !== 'false',
|
||||
udpPort: parseInt(process.env.COLLECTOR_UDP_PORT || '5140', 10),
|
||||
persistToDisk: process.env.COLLECTOR_PERSIST !== 'false',
|
||||
rotation: {
|
||||
maxSizeBytes: parseInt(process.env.COLLECTOR_ROTATION_MAX_SIZE || '5242880', 10),
|
||||
maxAgeMs: parseInt(process.env.COLLECTOR_ROTATION_MAX_AGE || '600000', 10),
|
||||
compression: process.env.COLLECTOR_ROTATION_COMPRESSION || 'gzip',
|
||||
fileExtension: process.env.COLLECTOR_ROTATION_EXTENSION || '.log',
|
||||
},
|
||||
forwardTargets,
|
||||
});
|
||||
}
|
||||
// S3 sources from env
|
||||
const s3Sources = new Map();
|
||||
function loadS3SourcesFromEnv() {
|
||||
// Format: S3_SOURCE_<name>=bucket:prefix
|
||||
// S3_ACCESS_KEY_ID, S3_SECRET_ACCESS_KEY, S3_REGION (global)
|
||||
// Or per-source: S3_SOURCE_<name>_ACCESS_KEY, S3_SOURCE_<name>_SECRET_KEY, etc.
|
||||
const globalAccessKey = process.env.S3_ACCESS_KEY_ID || process.env.AWS_ACCESS_KEY_ID || '';
|
||||
const globalSecretKey = process.env.S3_SECRET_ACCESS_KEY || process.env.AWS_SECRET_ACCESS_KEY || '';
|
||||
const globalRegion = process.env.S3_REGION || process.env.AWS_REGION || 'us-east-1';
|
||||
const globalEndpoint = process.env.S3_ENDPOINT || '';
|
||||
for (const [key, value] of Object.entries(process.env)) {
|
||||
if (key.startsWith('S3_SOURCE_') && value && !key.includes('_ACCESS_KEY') && !key.includes('_SECRET_KEY') && !key.includes('_REGION') && !key.includes('_ENDPOINT')) {
|
||||
const name = key.replace('S3_SOURCE_', '').toLowerCase();
|
||||
const parts = value.split(':');
|
||||
const bucket = parts[0];
|
||||
const prefix = parts.slice(1).join(':') || '';
|
||||
const accessKey = process.env[`S3_SOURCE_${name.toUpperCase()}_ACCESS_KEY`] || globalAccessKey;
|
||||
const secretKey = process.env[`S3_SOURCE_${name.toUpperCase()}_SECRET_KEY`] || globalSecretKey;
|
||||
const region = process.env[`S3_SOURCE_${name.toUpperCase()}_REGION`] || globalRegion;
|
||||
const endpoint = process.env[`S3_SOURCE_${name.toUpperCase()}_ENDPOINT`] || globalEndpoint;
|
||||
if (accessKey && secretKey && bucket) {
|
||||
const config = { accessKeyId: accessKey, secretAccessKey: secretKey, region, bucket, prefix: prefix || undefined, endpoint: endpoint || undefined };
|
||||
s3Sources.set(name, new s3Source_1.S3Source(config));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
loadS3SourcesFromEnv();
|
||||
// Monitor setup
|
||||
let monitor = null;
|
||||
if (MODE === 'monitor' || MODE === 'both' || MODE === 'all') {
|
||||
let monitorRules = [];
|
||||
try {
|
||||
monitorRules = JSON.parse(process.env.MONITOR_RULES || '[]');
|
||||
}
|
||||
catch { }
|
||||
monitor = new monitor_1.LogMonitor({
|
||||
enabled: process.env.MONITOR_ENABLED !== 'false',
|
||||
scanIntervalMs: parseInt(process.env.MONITOR_SCAN_INTERVAL || '60000', 10),
|
||||
rules: monitorRules,
|
||||
storageDir: process.env.MONITOR_STORAGE_DIR || path.join(process.cwd(), 'monitor-data'),
|
||||
});
|
||||
}
|
||||
// Remote collector URL (for viewer mode connecting to a remote collector)
|
||||
const REMOTE_COLLECTOR_URL = process.env.REMOTE_COLLECTOR_URL || '';
|
||||
const MIME_TYPES = {
|
||||
'.html': 'text/html; charset=utf-8',
|
||||
'.css': 'text/css; charset=utf-8',
|
||||
'.js': 'application/javascript; charset=utf-8',
|
||||
'.json': 'application/json; charset=utf-8',
|
||||
'.png': 'image/png',
|
||||
'.svg': 'image/svg+xml',
|
||||
'.ico': 'image/x-icon',
|
||||
};
|
||||
function sendJson(res, data, status = 200) {
|
||||
res.writeHead(status, { 'Content-Type': 'application/json; charset=utf-8' });
|
||||
res.end(JSON.stringify(data));
|
||||
}
|
||||
function sendError(res, message, status = 500) {
|
||||
sendJson(res, { error: message }, status);
|
||||
}
|
||||
async function readBody(req) {
|
||||
return new Promise((resolve, reject) => {
|
||||
const chunks = [];
|
||||
req.on('data', (chunk) => chunks.push(chunk));
|
||||
req.on('end', () => resolve(Buffer.concat(chunks).toString('utf-8')));
|
||||
req.on('error', reject);
|
||||
});
|
||||
}
|
||||
async function fetchRemote(remotePath) {
|
||||
if (!REMOTE_COLLECTOR_URL)
|
||||
throw new Error('No remote collector configured');
|
||||
const fullUrl = REMOTE_COLLECTOR_URL.replace(/\/$/, '') + remotePath;
|
||||
return new Promise((resolve, reject) => {
|
||||
const transport = fullUrl.startsWith('https') ? require('https') : require('http');
|
||||
transport.get(fullUrl, (res) => {
|
||||
const chunks = [];
|
||||
res.on('data', (c) => chunks.push(c));
|
||||
res.on('end', () => {
|
||||
try {
|
||||
resolve(JSON.parse(Buffer.concat(chunks).toString('utf-8')));
|
||||
}
|
||||
catch {
|
||||
resolve(Buffer.concat(chunks).toString('utf-8'));
|
||||
}
|
||||
});
|
||||
}).on('error', reject);
|
||||
});
|
||||
}
|
||||
const server = http.createServer(async (req, res) => {
|
||||
const parsedUrl = url.parse(req.url || '/', true);
|
||||
const pathname = parsedUrl.pathname || '/';
|
||||
const method = req.method || 'GET';
|
||||
// CORS
|
||||
res.setHeader('Access-Control-Allow-Origin', '*');
|
||||
res.setHeader('Access-Control-Allow-Methods', 'GET, POST, DELETE, OPTIONS');
|
||||
res.setHeader('Access-Control-Allow-Headers', 'Content-Type, X-Stream-Name');
|
||||
if (method === 'OPTIONS') {
|
||||
res.writeHead(204);
|
||||
res.end();
|
||||
return;
|
||||
}
|
||||
// ---- Collector endpoints ----
|
||||
if (collector && (pathname.startsWith('/collect/') || pathname.startsWith('/collector/'))) {
|
||||
const handled = collector.handleRequest(req, res, pathname);
|
||||
if (handled)
|
||||
return;
|
||||
}
|
||||
// ---- Monitor endpoints ----
|
||||
if (monitor && pathname.startsWith('/api/monitor/')) {
|
||||
const handled = monitor.handleRequest(req, res, pathname);
|
||||
if (handled)
|
||||
return;
|
||||
}
|
||||
// ---- API Routes ----
|
||||
if (pathname.startsWith('/api/')) {
|
||||
try {
|
||||
// GET /api/mode
|
||||
if (pathname === '/api/mode' && method === 'GET') {
|
||||
return sendJson(res, { mode: MODE, hasCollector: !!collector, hasRemoteCollector: !!REMOTE_COLLECTOR_URL, s3Sources: Array.from(s3Sources.keys()) });
|
||||
}
|
||||
// GET /api/config
|
||||
if (pathname === '/api/config' && method === 'GET') {
|
||||
return sendJson(res, { ...configManager.getConfig(), mode: MODE });
|
||||
}
|
||||
// GET /api/sources
|
||||
if (pathname === '/api/sources' && method === 'GET') {
|
||||
return sendJson(res, logSourceManager.getSources());
|
||||
}
|
||||
// POST /api/sources { path: "..." }
|
||||
if (pathname === '/api/sources' && method === 'POST') {
|
||||
const body = JSON.parse(await readBody(req));
|
||||
logSourceManager.addSource(body.path);
|
||||
return sendJson(res, logSourceManager.getSources());
|
||||
}
|
||||
// DELETE /api/sources?path=...
|
||||
if (pathname === '/api/sources' && method === 'DELETE') {
|
||||
const sourcePath = parsedUrl.query.path;
|
||||
if (!sourcePath)
|
||||
return sendError(res, 'Missing path parameter', 400);
|
||||
logSourceManager.removeSource(sourcePath);
|
||||
return sendJson(res, logSourceManager.getSources());
|
||||
}
|
||||
// GET /api/logs?source=...
|
||||
if (pathname === '/api/logs' && method === 'GET') {
|
||||
const source = parsedUrl.query.source;
|
||||
if (source && source !== 'ALL') {
|
||||
const entries = await logSourceManager.readLog(source);
|
||||
return sendJson(res, entries);
|
||||
}
|
||||
else {
|
||||
const entries = await logSourceManager.readAllLogs();
|
||||
return sendJson(res, entries);
|
||||
}
|
||||
}
|
||||
// ---- S3 endpoints ----
|
||||
// POST /api/s3/add { name, bucket, prefix, accessKeyId, secretAccessKey, region, endpoint? }
|
||||
if (pathname === '/api/s3/add' && method === 'POST') {
|
||||
const body = JSON.parse(await readBody(req));
|
||||
const { name, bucket, prefix, accessKeyId, secretAccessKey, region, endpoint } = body;
|
||||
if (!name || !bucket || !accessKeyId || !secretAccessKey || !region) {
|
||||
return sendError(res, 'Missing required fields: name, bucket, accessKeyId, secretAccessKey, region', 400);
|
||||
}
|
||||
const config = { accessKeyId, secretAccessKey, region, bucket, prefix, endpoint };
|
||||
s3Sources.set(name, new s3Source_1.S3Source(config));
|
||||
return sendJson(res, { ok: true, name });
|
||||
}
|
||||
// GET /api/s3/sources
|
||||
if (pathname === '/api/s3/sources' && method === 'GET') {
|
||||
return sendJson(res, Array.from(s3Sources.keys()));
|
||||
}
|
||||
// GET /api/s3/list?name=...&prefix=...
|
||||
if (pathname === '/api/s3/list' && method === 'GET') {
|
||||
const name = parsedUrl.query.name;
|
||||
const prefix = parsedUrl.query.prefix;
|
||||
const s3 = s3Sources.get(name);
|
||||
if (!s3)
|
||||
return sendError(res, `S3 source not found: ${name}`, 404);
|
||||
const objects = await s3.listObjects(prefix);
|
||||
return sendJson(res, objects);
|
||||
}
|
||||
// GET /api/s3/read?name=...&key=...
|
||||
if (pathname === '/api/s3/read' && method === 'GET') {
|
||||
const name = parsedUrl.query.name;
|
||||
const key = parsedUrl.query.key;
|
||||
const s3 = s3Sources.get(name);
|
||||
if (!s3)
|
||||
return sendError(res, `S3 source not found: ${name}`, 404);
|
||||
if (!key)
|
||||
return sendError(res, 'Missing key parameter', 400);
|
||||
const content = await s3.getObject(key);
|
||||
const lines = content.split('\n').filter(l => l.trim() !== '');
|
||||
const entries = lines.map((line, i) => ({
|
||||
line,
|
||||
lineNumber: i + 1,
|
||||
source: `s3://${name}/${key}`,
|
||||
timestamp: extractTimestamp(line),
|
||||
level: extractLevel(line),
|
||||
}));
|
||||
return sendJson(res, entries);
|
||||
}
|
||||
// DELETE /api/s3/remove?name=...
|
||||
if (pathname === '/api/s3/remove' && method === 'DELETE') {
|
||||
const name = parsedUrl.query.name;
|
||||
if (!name)
|
||||
return sendError(res, 'Missing name parameter', 400);
|
||||
s3Sources.delete(name);
|
||||
return sendJson(res, { ok: true });
|
||||
}
|
||||
// ---- Remote Collector endpoints (viewer reads from remote collector) ----
|
||||
// GET /api/remote/streams
|
||||
if (pathname === '/api/remote/streams' && method === 'GET') {
|
||||
if (!REMOTE_COLLECTOR_URL)
|
||||
return sendJson(res, []);
|
||||
try {
|
||||
const streams = await fetchRemote('/collector/streams');
|
||||
return sendJson(res, streams);
|
||||
}
|
||||
catch (err) {
|
||||
return sendError(res, `Cannot reach remote collector: ${err.message}`);
|
||||
}
|
||||
}
|
||||
// GET /api/remote/stream/:id?tail=N
|
||||
const remoteStreamMatch = pathname.match(/^\/api\/remote\/stream\/([a-zA-Z0-9_\-\.]+)$/);
|
||||
if (remoteStreamMatch && method === 'GET') {
|
||||
const streamId = remoteStreamMatch[1];
|
||||
const tail = parsedUrl.query.tail;
|
||||
const tailParam = tail ? `?tail=${tail}` : '';
|
||||
try {
|
||||
const data = await fetchRemote(`/collector/stream/${streamId}${tailParam}`);
|
||||
// Convert to LogEntry format
|
||||
const lines = data.lines || [];
|
||||
const entries = lines.map((line, i) => ({
|
||||
line,
|
||||
lineNumber: i + 1,
|
||||
source: `collector://${streamId}`,
|
||||
timestamp: extractTimestamp(line),
|
||||
level: extractLevel(line),
|
||||
}));
|
||||
return sendJson(res, entries);
|
||||
}
|
||||
catch (err) {
|
||||
return sendError(res, err.message);
|
||||
}
|
||||
}
|
||||
// ---- Collector streams as log source (local) ----
|
||||
// GET /api/collector/streams
|
||||
if (pathname === '/api/collector/streams' && method === 'GET') {
|
||||
if (!collector)
|
||||
return sendJson(res, []);
|
||||
return sendJson(res, collector.getStreams());
|
||||
}
|
||||
// GET /api/collector/stream/:id
|
||||
const localStreamMatch = pathname.match(/^\/api\/collector\/stream\/([a-zA-Z0-9_\-\.]+)$/);
|
||||
if (localStreamMatch && method === 'GET') {
|
||||
const streamId = localStreamMatch[1];
|
||||
if (!collector)
|
||||
return sendError(res, 'Collector not enabled', 400);
|
||||
try {
|
||||
const tail = parsedUrl.query.tail;
|
||||
const lines = collector.readStream(streamId, tail ? parseInt(tail, 10) : undefined);
|
||||
const entries = lines.map((line, i) => ({
|
||||
line,
|
||||
lineNumber: i + 1,
|
||||
source: `stream://${streamId}`,
|
||||
timestamp: extractTimestamp(line),
|
||||
level: extractLevel(line),
|
||||
}));
|
||||
return sendJson(res, entries);
|
||||
}
|
||||
catch (err) {
|
||||
return sendError(res, err.message, 404);
|
||||
}
|
||||
}
|
||||
// POST /api/config/save - Save config to file
|
||||
if (pathname === '/api/config/save' && method === 'POST') {
|
||||
const body = JSON.parse(await readBody(req));
|
||||
(0, configFile_1.saveConfigFile)(body);
|
||||
return sendJson(res, { ok: true });
|
||||
}
|
||||
return sendError(res, 'Not found', 404);
|
||||
}
|
||||
catch (err) {
|
||||
console.error('API error:', err);
|
||||
return sendError(res, err.message);
|
||||
}
|
||||
}
|
||||
// ---- Static Files ----
|
||||
const webDir = path.join(__dirname, '..', 'src', 'web');
|
||||
let filePath;
|
||||
// Serve collector UI when in collector mode, monitor UI when in monitor mode
|
||||
if (pathname === '/' || pathname === '/index.html') {
|
||||
if (MODE === 'collector') {
|
||||
filePath = path.join(webDir, 'collector.html');
|
||||
}
|
||||
else if (MODE === 'monitor') {
|
||||
filePath = path.join(webDir, 'monitor.html');
|
||||
}
|
||||
else {
|
||||
filePath = path.join(webDir, 'index.html');
|
||||
}
|
||||
}
|
||||
else {
|
||||
filePath = path.join(webDir, pathname);
|
||||
}
|
||||
const webRoot = path.resolve(webDir);
|
||||
const resolvedPath = path.resolve(filePath);
|
||||
if (!resolvedPath.startsWith(webRoot)) {
|
||||
res.writeHead(403);
|
||||
res.end('Forbidden');
|
||||
return;
|
||||
}
|
||||
try {
|
||||
if (!fs.existsSync(resolvedPath) || fs.statSync(resolvedPath).isDirectory()) {
|
||||
res.writeHead(404);
|
||||
res.end('Not found');
|
||||
return;
|
||||
}
|
||||
const ext = path.extname(resolvedPath);
|
||||
const contentType = MIME_TYPES[ext] || 'application/octet-stream';
|
||||
res.writeHead(200, { 'Content-Type': contentType });
|
||||
res.end(fs.readFileSync(resolvedPath));
|
||||
}
|
||||
catch {
|
||||
res.writeHead(500);
|
||||
res.end('Internal server error');
|
||||
}
|
||||
});
|
||||
// Helper functions
|
||||
function extractTimestamp(line) {
|
||||
const patterns = [
|
||||
/(\d{4}-\d{2}-\d{2}[T ]\d{2}:\d{2}:\d{2}(?:\.\d+)?(?:Z|[+-]\d{2}:?\d{2})?)/,
|
||||
/^([A-Z][a-z]{2}\s+\d{1,2}\s+\d{2}:\d{2}:\d{2})/,
|
||||
/\[(\d{2}\/[A-Z][a-z]{2}\/\d{4}:\d{2}:\d{2}:\d{2}\s[+-]\d{4})\]/,
|
||||
];
|
||||
for (const p of patterns) {
|
||||
const m = line.match(p);
|
||||
if (m)
|
||||
return m[1];
|
||||
}
|
||||
return null;
|
||||
}
|
||||
function extractLevel(line) {
|
||||
const upper = line.toUpperCase();
|
||||
for (const level of ['FATAL', 'ERROR', 'WARN', 'WARNING', 'INFO', 'DEBUG', 'TRACE']) {
|
||||
if (new RegExp(`\\b${level}\\b`).test(upper))
|
||||
return level === 'WARNING' ? 'WARN' : level;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
server.listen(PORT, HOST, () => {
|
||||
console.log('');
|
||||
console.log(' ╔══════════════════════════════════════════════════╗');
|
||||
console.log(` ║ 📋 Log Viewer - Mode: ${MODE.toUpperCase().padEnd(24)}║`);
|
||||
console.log(' ╠══════════════════════════════════════════════════╣');
|
||||
console.log(` ║ URL: http://${HOST}:${PORT} ║`);
|
||||
console.log(` ║ Sources: ${String(logSourceManager.getSources().length).padEnd(3)} file(s) ║`);
|
||||
console.log(` ║ S3 Sources: ${String(s3Sources.size).padEnd(3)} configured ║`);
|
||||
console.log(` ║ Collector: ${collector ? 'ACTIVE' : 'OFF'} ║`);
|
||||
console.log(` ║ Monitor: ${monitor ? 'ACTIVE' : 'OFF'} ║`);
|
||||
if (REMOTE_COLLECTOR_URL) {
|
||||
console.log(` ║ Remote: ${REMOTE_COLLECTOR_URL.substring(0, 30).padEnd(30)} ║`);
|
||||
}
|
||||
console.log(' ╚══════════════════════════════════════════════════╝');
|
||||
console.log('');
|
||||
if (collector) {
|
||||
console.log(' Collector endpoints:');
|
||||
console.log(` POST http://${HOST}:${PORT}/collect/<stream-id> (plain text, one line per line)`);
|
||||
console.log(` POST http://${HOST}:${PORT}/collect/<stream-id>/json (JSON array of entries)`);
|
||||
console.log('');
|
||||
}
|
||||
});
|
||||
//# sourceMappingURL=server.js.map
|
||||
Reference in New Issue
Block a user