How we collect metrics

[INPUT]
    Name          tcp
    Port          5170
    Chunk_Size    32
    Buffer_Size   64
    Tag           custom

[INPUT]
    Name          cpu
    Tag           cpu_usage
    Interval_Sec  60
    Interval_NSec 0

[INPUT]
    Name          mem
    Tag           memory_usage
    Interval_Sec  60
    Interval_NSec 0

[INPUT]
    Name          exec
    Tag           disk_space
    Command       df --output=pcent --type=ext4
    Interval_Sec  60
    Interval_NSec 0

[INPUT]
    Name          exec
    Tag           app_tag
    Command       docker inspect --format '{{json .Config.Labels}}' app
    Interval_Sec  300
    Interval_NSec 0

[INPUT]
    Name          health
    Host          127.0.0.1
    Port          80
    Interval_Sec  5
    Interval_NSec 0
    Tag           web_health

[FILTER]
    Name          record_modifier
    Match         *
    Record hostname ${PARENT_HOSTNAME}

[FILTER]
    Name          lua
    Match         *
    script        /etc/fluent-bit/scripts/append_tag.lua
    call          append_tag

[OUTPUT]
    Name          http
    Match         *
    Host          your-tracking-endpoint-goes-here.com
    Port          443
    Format        json
    URI           /track
    tls           On

'use strict';

const http = require('http');
const express = require('express');
const mysql = require('mysql');
const SSE = require('express-sse');
const sse = new SSE();
const constants = require('./constants');
const notifyEvent = require('./notifyEvent');

function start(allowedIpAddresses) {
	const connection = mysql.createConnection(constants.mysql);

	try {
		connection.connect();
	} catch (error) {
		console.log('MySQL Connection Eror: ', error.message || error);
	}

	const app = express();
	app.use(express.json({
		limit: '1024kb'
	}));

	/**
	 * POST /track
	 * Write-only route that receives fluentbit data from servers
	 */
	app.post('/track', (req, res) => {
		req.body.forEach(event => {
			const isDiskSpaceEventWithIrrelevantData = event.tag === 'disk_space' && !event.exec;

			if (!isDiskSpaceEventWithIrrelevantData) {
				if (event.tag === 'disk_space' && typeof event.exec !== 'undefined') {
					event.exec = parseInt(event.exec);
				}

				sse.send(event, 'result');
				notifyEvent(event);

				const parameters = {
					created_at: { toSqlString: () => 'FROM_UNIXTIME(' + parseFloat(event.date) + ')' },
					tag: event.tag,
					hostname: event.hostname,
					data: JSON.stringify(event)
				};

				connection.query('INSERT INTO events SET ?', parameters, (error, results, fields) => {
					if (error) {
						console.log('Insert error: ', error.message || error)
					}
				});
			}
		});

		res.send("OK\n");
	});

	/**
	 * GET /stream
	 * Read-only route that streams fluentbit data that were received from servers via the /track route
	 */
	app.options('/stream', middlewares);
	app.get('/stream', middlewares, sse.init);

	http.createServer(app).listen(constants.server.port, constants.server.host);

	console.log(`Running on https://${constants.server.host}:${constants.server.port}`);
}

'use strict';

const constants = require('./constants');
const Twilio = require('twilio');

const client = new Twilio(constants.twilio.accountSid, constants.twilio.authToken);

let notifications = {};

const renotifyThresholdMS = 1000 * 60 * 60 * 12; // 12 hours

function notifyEvent(event) {
	purgeNotifications();

	if (event.tag === 'disk_space' && typeof event.exec !== 'undefined') {
		if (event.exec > 95) {
			report(event, 'ERROR', `Disk space is ${event.exec}%`);
		} else if (event.exec > 90) {
			report(event, 'WARNING', `Disk space is ${event.exec}%`);
		}
	}

	// TODO: Add more rules here
}

function purgeNotifications() {
	const cutOffTimestamp = Date.now() - renotifyThresholdMS;

	for (var key in notifications) {
		if (notifications[key] < cutOffTimestamp) {
			delete notifications[key];
			console.log('Deleted key');
		}
	}
}

function report(event, severity, message) {
	const key = event.hostname + severity + message;

	if (!(key in notifications)) {
		notifications[key] = Date.now();

		const body = `${severity}: ${message} on ${event.hostname}`;
		console.log(`Sending "${body}"`)

		client.messages.create({
			body,
			from: constants.twilio.fromPhoneNumber,
			to: constants.twilio.toPhoneNumber,
		});
	}
}

module.exports = notifyEvent;

CREATE TABLE `events` (
  `id` int(11) unsigned NOT NULL AUTO_INCREMENT,
  `created_at` datetime DEFAULT NULL,
  `tag` varchar(255) DEFAULT NULL,
  `hostname` varchar(255) DEFAULT NULL,
  `data` longtext CHARACTER SET utf8mb4 COLLATE utf8mb4_bin DEFAULT NULL,
  PRIMARY KEY (`id`),
  KEY `created_at` (`created_at`),
  KEY `hostname` (`hostname`,`tag`,`created_at`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4