zabbix_export: version: '7.0' media_types: - name: 'OTRS CE' type: WEBHOOK parameters: - name: alert_message value: '{ALERT.MESSAGE}' - name: alert_subject value: '{ALERT.SUBJECT}' - name: event_id value: '{EVENT.ID}' - name: event_nseverity value: '{EVENT.NSEVERITY}' - name: event_severity value: '{EVENT.SEVERITY}' - name: event_source value: '{EVENT.SOURCE}' - name: event_update_nseverity value: '{EVENT.UPDATE.NSEVERITY}' - name: event_update_severity value: '{EVENT.UPDATE.SEVERITY}' - name: event_update_status value: '{EVENT.UPDATE.STATUS}' - name: event_value value: '{EVENT.VALUE}' - name: otrs_auth_password value: '' - name: otrs_auth_user value: '' - name: otrs_closed_state_id value: '0' - name: otrs_customer value: '' - name: otrs_default_priority_id value: '3' - name: otrs_queue value: '' - name: otrs_ticket_id value: '{EVENT.TAGS.__zbx_otrs_ticket_id}' - name: otrs_ticket_state value: new - name: otrs_ticket_type value: Unclassified - name: otrs_time_unit value: '0' - name: otrs_url value: '' - name: trigger_id value: '{TRIGGER.ID}' - name: zabbix_url value: '{$ZABBIX.URL}' status: DISABLED script: | const CLogger = function(serviceName) { this.serviceName = serviceName; this.INFO = 4 this.WARN = 3 this.ERROR = 2 this.log = function(level, msg) { Zabbix.log(level, '[' + this.serviceName + '] ' + msg); } } const CWebhook = function(value) { try { params = JSON.parse(value); if (['0', '1', '2', '3', '4'].indexOf(params.event_source) === -1) { throw 'Incorrect "event_source" parameter given: ' + params.event_source + '.\nMust be 0-4.'; } if (['0', '3', '4'].indexOf(params.event_source) !== -1 && ['0', '1'].indexOf(params.event_value) === -1) { throw 'Incorrect "event_value" parameter given: ' + params.event_value + '.\nMust be 0 or 1.'; } if (['0', '3', '4'].indexOf(params.event_source) !== -1) { if (params.event_source === '1' && ['0', '1', '2', '3'].indexOf(params.event_value) === -1) { throw 'Incorrect "event_value" parameter given: ' + params.event_value + '.\nMust be 0-3.'; } if (params.event_source === '0' && ['0', '1'].indexOf(params.event_update_status) === -1) { throw 'Incorrect "event_update_status" parameter given: ' + params.event_update_status + '.\nMust be 0 or 1.'; } if (params.event_source === '4') { if (['0', '1', '2', '3', '4', '5'].indexOf(params.event_update_nseverity) !== -1 && params.event_update_nseverity != params.event_nseverity) { params.event_nseverity = params.event_update_nseverity; params.event_severity = params.event_update_severity; params.event_update_status = '1'; } } } this.runCallback = function(name, params) { if (typeof this[name] === 'function') { return this[name].apply(this, [params]); } } this.handleEvent = function(source, event) { const alert = { source: source, event: event }; return [ this.runCallback('on' + source + event, alert), this.runCallback('on' + event, alert), this.runCallback('onEvent', alert) ]; } this.handleEventless = function(source) { const alert = { source: source, event: null }; return [ this.runCallback('on' + source, alert), this.runCallback('onEvent', alert) ]; } this.run = function() { var results = []; if (typeof this.httpProxy === 'string' && this.httpProxy.trim() !== '') { this.request.setProxy(this.httpProxy); } const types = { '0': 'Trigger', '1': 'Discovery', '2': 'Autoreg', '3': 'Internal', '4': 'Service' }; if (['0', '3', '4'].indexOf(this.params.event_source) !== -1) { var event = (this.params.event_update_status === '1') ? 'Update' : ((this.params.event_value === '1') ? 'Problem' : 'Resolve'); results = this.handleEvent(types[this.params.event_source], event); } else if (typeof types[this.params.event_source] !== 'undefined') { results = this.handleEventless(types[this.params.event_source]); } else { throw 'Unexpected "event_source": ' + this.params.event_source; } for (idx in results) { if (typeof results[idx] !== 'undefined') { return JSON.stringify(results[idx]); } } } this.httpProxy = params.http_proxy; this.params = params; this.runCallback('onCheckParams', {}); } catch (error) { throw 'Webhook processing failed: ' + error; } } const CWebhookHelper = { createProblemURL: function(event_source, zabbix_url, trigger_id, event_id) { if (event_source === '0') { return zabbix_url + '/tr_events.php?triggerid=' + trigger_id + '&eventid=' + event_id; } else if (event_source === '4') { return zabbix_url + '/zabbix.php?action=service.list'; } return zabbix_url; }, }; const CParamValidator = { isType: function(value, type) { if (type === 'array') { return Array.isArray(value); } if (type === 'integer') { return CParamValidator.isInteger(value); } if (type === 'float') { return CParamValidator.isFloat(value); } return (typeof value === type); }, isInteger: function(value) { if (!CParamValidator.ifMatch(value, /^-?\d+$/)) { return false; } return !isNaN(parseInt(value)); }, isFloat: function(value) { if (!CParamValidator.ifMatch(value, /^-?\d+\.\d+$/)) { return false; } return !isNaN(parseFloat(value)); }, isDefined: function(value) { return !CParamValidator.isType(value, 'undefined'); }, isEmpty: function(value) { if (!CParamValidator.isType(value, 'string')) { throw 'Value "' + value + '" must be a string to be checked for emptiness.'; } return (value.trim() === ''); }, isMacroSet: function(value, macro) { if (CParamValidator.isDefined(macro)) { return !(CParamValidator.ifMatch(value, '^\{' + macro + '\}$')) } return !(CParamValidator.ifMatch(value, '^\{[$#]{0,1}[A-Z_\.]+[\:]{0,1}["]{0,1}.*["]{0,1}\}$') || value === '*UNKNOWN*') }, withinRange: function(value, min, max) { if (!CParamValidator.isType(value, 'number')) { throw 'Value "' + value + '" must be a number to be checked for range.'; } if (value < ((CParamValidator.isDefined(min)) ? min : value) || value > ((CParamValidator.isDefined(max)) ? max : value)) { return false; } return true; }, inArray: function(value, array) { if (!CParamValidator.isType(array, 'array')) { throw 'The array must be an array to check the value for existing in it.'; } return (array.indexOf((typeof value === 'string') ? value.toLowerCase() : value) !== -1); }, ifMatch: function(value, regex) { return (new RegExp(regex)).test(value); }, match: function(value, regex) { if (!CParamValidator.isType(value, 'string')) { throw 'Value "' + value + '" must be a string to be matched with the regular expression.'; } return value.match(new RegExp(regex)); }, checkURL: function(value) { if (CParamValidator.isEmpty(value)) { throw 'URL value "' + value + '" must be a non-empty string.'; } if (!CParamValidator.ifMatch(value, '^(http|https):\/\/.+')) { throw 'URL value "' + value + '" must contain a schema.'; } return value.endsWith('/') ? value.slice(0, -1) : value; }, check: function(key, rule, params) { if (!CParamValidator.isDefined(rule.type)) { throw 'Mandatory attribute "type" has not been defined for parameter "' + key + '".'; } if (!CParamValidator.isDefined(params[key])) { throw 'Checked parameter "' + key + '" was not found in the list of input parameters.'; } var value = params[key], error_message = null; switch (rule.type) { case 'string': if (!CParamValidator.isType(value, 'string')) { throw 'Value "' + key + '" must be a string.'; } if (CParamValidator.isEmpty(value)) { error_message = 'Value "' + key + '" must be a non-empty string'; break; } if (CParamValidator.isDefined(rule.len) && value.length < rule.len) { error_message = 'Value "' + key + '" must be a string with a length > ' + rule.len; } if (CParamValidator.isDefined(rule.regex) && !CParamValidator.ifMatch(value, rule.regex)) { error_message = 'Value "' + key + '" must match the regular expression "' + rule.regex + '"'; } if (CParamValidator.isDefined(rule.url) && rule.url === true) { value = CParamValidator.checkURL(value); } break; case 'integer': if (!CParamValidator.isInteger(value)) { error_message = 'Value "' + key + '" must be an integer'; break; } value = parseInt(value); break; case 'float': if (!CParamValidator.isFloat(value)) { error_message = 'Value "' + key + '" must be a floating-point number'; break; } value = parseFloat(value); break; case 'boolean': if (CParamValidator.inArray(value, ['1', 'true', 'yes', 'on'])) { value = true; } else if (CParamValidator.inArray(value, ['0', 'false', 'no', 'off'])) { value = false; } else { error_message = 'Value "' + key + '" must be a boolean-like.'; } break; case 'array': try { value = JSON.parse(value); } catch (error) { throw 'Value "' + key + '" contains invalid JSON.'; } if (!CParamValidator.isType(value, 'array')) { error_message = 'Value "' + key + '" must be an array.'; } if (CParamValidator.isDefined(rule.tags) && rule.tags === true) { value = value.reduce(function(acc, obj) { acc[obj.tag] = obj.value || null; return acc; }, {}); } break; case 'object': value = JSON.parse(value); if (!CParamValidator.isType(value, 'object')) { error_message = 'Value "' + key + '" must be an object.'; } break; default: throw 'Unexpected attribute type "' + rule.type + '" for value "' + key + '". Available: ' + ['integer', 'float', 'string', 'boolean', 'array', 'object'].join(', '); } params[key] = value; if (CParamValidator.inArray(rule.type, ['integer', 'float']) && error_message === null && (CParamValidator.isDefined(rule.min) || CParamValidator.isDefined(rule.max)) && !CParamValidator.withinRange(value, rule.min, rule.max)) { error_message = 'Value "' + key + '" must be a number ' + ((CParamValidator.isDefined(rule.min) && CParamValidator.isDefined(rule.max)) ? (rule.min + '..' + rule.max) : ((CParamValidator.isDefined(rule.min)) ? '>' + rule.min : '<' + rule.max)); } else if (CParamValidator.isDefined(rule.array) && !CParamValidator.inArray(value, rule.array)) { error_message = 'Value "' + key + '" must be in the array ' + JSON.stringify(rule.array); } else if (CParamValidator.isDefined(rule.macro) && !CParamValidator.isMacroSet(value.toString(), rule.macro)) { error_message = 'The macro ' + ((CParamValidator.isDefined(rule.macro)) ? '{' + rule.macro + '} ' : ' ') + 'is not set'; } if (error_message !== null) { if (CParamValidator.isDefined(rule.default) && CParamValidator.isType(rule.default, rule.type)) { params[key] = rule.default; } else { Zabbix.log(4, 'Default value for "' + key + '" must be a ' + rule.type + '. Skipped.'); throw 'Incorrect value for variable "' + key + '". ' + error_message; } } return this; }, validate: function(rules, params) { if (!CParamValidator.isType(params, 'object') || CParamValidator.isType(params, 'array')) { throw 'Incorrect parameters value. The value must be an object.'; } for (var key in rules) { CParamValidator.check(key, rules[key], params); } } } const CHttpRequest = function(logger) { this.request = new HttpRequest(); if (typeof logger !== 'object' || logger === null) { this.logger = Zabbix; } else { this.logger = logger; } this.clearHeader = function() { this.request.clearHeader(); } this.addHeaders = function(value) { var headers = []; if (typeof value === 'object' && value !== null) { if (!Array.isArray(value)) { Object.keys(value).forEach(function(key) { headers.push(key + ': ' + value[key]); }); } else { headers = value; } } else if (typeof value === 'string') { value.split('\r\n').forEach(function(header) { headers.push(header); }); } for (var idx in headers) { this.request.addHeader(headers[idx]); } } this.setProxy = function(proxy) { this.request.setProxy(proxy); } this.plainRequest = function(method, url, data) { var resp = null; method = method.toLowerCase(); this.logger.log(4, 'Sending ' + method + ' request:' + JSON.stringify(data)); if (['get', 'post', 'put', 'patch', 'delete', 'trace'].indexOf(method) !== -1) { resp = this.request[method](url, data); } else if (['connect', 'head', 'options'].indexOf(method) !== -1) { resp = this.request[method](url); } else { throw 'Unexpected method. Method ' + method + ' is not supported.'; } this.logger.log(4, 'Response has been received: ' + resp); return resp; } this.jsonRequest = function(method, url, data) { this.addHeaders('Content-Type: application/json'); var resp = this.plainRequest(method, url, JSON.stringify(data)); try { resp = JSON.parse(resp); } catch (error) { throw 'Failed to parse response: not well-formed JSON was received'; } return resp; } this.getStatus = function() { return this.request.getStatus(); } } const SEVERITIES = ["not_classified", "information", "warning", "average", "high", "disaster"], serviceLogName = ' ((OTRS)) CE Webhook ', Logger = new CLogger(serviceLogName), OTRS = CWebhook; OTRS.prototype.onCheckParams = function () { CParamValidator.validate({alert_subject: {type: 'string'}, alert_message: {type: 'string'}, event_nseverity: {type: 'integer', default: -1}, otrs_url: {type: 'string', url: true}, otrs_auth_user: {type: 'string'}, otrs_auth_password: {type: 'string'}, otrs_customer: {type: 'string'}, otrs_default_priority_id: {type: 'integer', min: 1, max: 5}, otrs_queue: {type: 'string'}, otrs_ticket_type: {type: 'string'}, otrs_ticket_state: {type: 'string'}, otrs_time_unit: {type: 'integer'}, otrs_closed_state_id: {type: 'integer', default: 0}, zabbix_url: {type: 'string', url: true}}, this.params); this.params.entrypoint = '/nph-genericinterface.pl/Webservice/ZabbixTicketConnector/Ticket'; var priority; if (this.params.event_nseverity >= 0 && this.params.event_nseverity < SEVERITIES.length) { priority = this.params['severity_' + SEVERITIES[this.params.event_nseverity]]; } this.priority = (CParamValidator.isDefined(priority)) ? priority.trim() : this.params.otrs_default_priority_id; if (this.params.event_source === '0') { CParamValidator.validate({trigger_id: {type: 'integer'}, event_id: {type: 'integer'}}, this.params); this.params.zabbix_url = CWebhookHelper.createProblemURL(this.params.event_source, this.params.zabbix_url, this.params.trigger_id, this.params.event_id); this.params.alert_message = this.params.alert_subject + '\n' + this.params.alert_message + '\n' + this.params.zabbix_url + '\n'; } if (this.params.event_value != '0' && CParamValidator.isMacroSet(this.params.otrs_ticket_id)) { this.params.event_update_status = '1'; } this.dynamicFields = {} Object.keys(this.params).forEach(function (key) { if (key.startsWith('dynamicfield_')) { this.dynamicFields[key.substring(13)] = this.params[key]; } }); this.data = { Article: { Subject: this.params.alert_subject, Body: (CParamValidator.isDefined(this.params.alert_message)) ? this.params.alert_message : '', TimeUnit: this.params.otrs_time_unit.toString(), ContentType: 'text/plain; charset=utf8' } }; this.result = {tags: {}}; }; OTRS.prototype.sendRequest = function (method) { var url = this.params.otrs_url + this.params.entrypoint + '?UserLogin=' + encodeURIComponent(this.params.otrs_auth_user) + '&Password=' + encodeURIComponent(this.params.otrs_auth_password); var response = this.request.jsonRequest(method, url, this.data); if (!CParamValidator.isType(response, 'object')) { Logger.log(Logger.INFO, 'API response ERROR: ' + response); throw 'Unknown error. Check debug log for more information.'; } if (this.request.getStatus() < 200 || this.request.getStatus() >= 300) { var message = 'status code ' + this.request.getStatus(); Logger.log(Logger.INFO, 'API response ERROR with ' + message + ': ' + response); throw 'Request failed with ' + message + '. Check debug log for more information.'; } if (CParamValidator.isDefined(response.Error) && Object.keys(response.Error).length > 0) { Logger.log(Logger.INFO, 'API response ERROR: ' + JSON.stringify(response.Error)); throw 'Request failed: ' + JSON.stringify(response.Error); } return { status: this.request.getStatus(), response: response }; }; OTRS.prototype.createTicket = function () { this.data['Ticket'] = { Title: this.params.alert_subject, Queue: this.params.otrs_queue, Type: this.params.otrs_ticket_type, State: this.params.otrs_ticket_state, PriorityID: this.priority.toString(), CustomerUser: this.params.otrs_customer } var result = this.sendRequest('post'); if (!CParamValidator.isDefined(result.response.TicketID) || result.status != 200) { throw 'Cannot create ((OTRS)) CE ticket. Check debug log for more information.'; } return result.response.TicketID; } OTRS.prototype.updateTicket = function () { CParamValidator.validate({otrs_ticket_id: {type: 'string'}, entrypoint: {type: 'string'}}, this.params); this.params.entrypoint += '/' + encodeURIComponent(this.params.otrs_ticket_id); var result = this.sendRequest('put'); if (!CParamValidator.isDefined(result.response.TicketID) || result.status != 200) { throw 'Cannot update ((OTRS)) CE ticket. Check debug log for more information.'; } return result.response.TicketID; } OTRS.prototype.onProblem = function (alert) { Logger.log(Logger.INFO, 'Source: ' + alert.source + '; Event: ' + alert.event); if (CParamValidator.isDefined(alert.source) && CParamValidator.inArray(alert.source, ['trigger', 'service', 'internal'])) { if (Object.keys(this.dynamicFields).length > 0) { this.data.DynamicField = []; Object.keys(this.dynamicFields).forEach(function(field) { if (field !== undefined) { if (this.dynamicFields[field].match(/^\d{4}[.-]\d{2}[.-]\d{2}$/)) { this.dynamicFields[field] = this.dynamicFields[field].replace(/\./g, '-'); } this.data.DynamicField.push({Name: field, Value: this.dynamicFields[field]}); } }); } const ticket_id = this.createTicket(alert); this.result.tags.__zbx_otrs_ticket_id = ticket_id; this.result.tags.__zbx_otrs_ticketlink = this.params.otrs_url + 'index.pl?Action=AgentTicketZoom;TicketID=' + ticket_id; return this.result; } return this.createTicket(); } OTRS.prototype.onUpdate = function (alert) { Logger.log(Logger.INFO, 'Source: ' + alert.source + '; Event: ' + alert.event); this.updateTicket(); return this.result; } OTRS.prototype.onResolve = function (alert) { Logger.log(Logger.INFO, 'Source: ' + alert.source + '; Event: ' + alert.event); if (this.params.otrs_closed_state_id > 0) { this.data['Ticket'] = { StateID: this.params.otrs_closed_state_id } } this.updateTicket(); return this.result; } OTRS.prototype.onDiscovery = function (alert) { return this.onProblem(alert); } OTRS.prototype.onAutoreg = function (alert) { return this.onProblem(alert); } try { var hook = new OTRS(value); hook.request = new CHttpRequest(Logger); return hook.run(); } catch (error) { Logger.log(Logger.WARN, 'notification failed: ' + error); throw 'Sending failed: ' + error; } process_tags: 'YES' show_event_menu: 'YES' event_menu_url: '{EVENT.TAGS.__zbx_otrs_ticketlink}' event_menu_name: '((OTRS)) CE: ticket #{EVENT.TAGS.__zbx_otrs_ticket_id}' description: | This media type integrates your Zabbix installation with your Zammad installation using the Zabbix webhook feature. ((OTRS)) CE configuration: 1. Create a new web service. To do so, navigate to "Admin" → "Web services" and import the "ZabbixTicketConnector.yml" file (it can be found in the official Zabbix repository next to the media type file). 2. Create a new customer. 3. Create a new customer user. Select the ID of the customer that you created in the previous step. 4. Create a new agent. Depending on the ticket queue you want to use for tickets created by the webhook, make sure the group it belongs to has the `RW` permission. In the example below, if you want to use the `Misc` queue, you need to give the `users` group the `RW` permission. Zabbix configuration: 1. Before you can start using the ((OTRS)) CE webhook, set the global macro "{$ZABBIX.URL}": - In the Zabbix web interface, go to "Administration" → "Macros" in the top-left dropdown menu. - Set the global macro "{$ZABBIX.URL}" to the URL of the Zabbix frontend. The URL should be either an IP address, a fully qualified domain name, or localhost. - Specifying a protocol is mandatory, whereas the port is optional. Depending on the web server configuration, you might also need to append "/zabbix" to the end of URL. Good examples: - http://zabbix.com - https://zabbix.lan/zabbix - http://server.zabbix.lan/ - http://localhost - http://127.0.0.1:8080 - Bad examples: - zabbix.com - http://zabbix/ 2. Set the following webhook parameters: - otrs_auth_user - the username of the agent - otrs_auth_password - the password of the agent - otrs_customer - the email of the customer user - otrs_queue - the queue that will be used for tickets created by the webhook - otrs_url - the frontend URL of your ((OTRS)) CE installation (for example, "https://otrs.example.com/otrs") 3. If you want to prioritize issues according to the severity values in Zabbix, you can define mapping parameters (create them as additional webhook parameters): - severity_ - the ((OTRS)) CE priority ID ( in the parameter name can be one of the following values: "not_classified", "information", "warning", "average", "high", "disaster") 4. If you have dynamic fields in ((OTRS)) CE and want them to be filled with values from Zabbix, add webhook parameters in the format "dynamicfield_<((OTRS)) CE dynamic field name>", similarly to the previous step. Dynamic fields can only be of the types "text", "textarea", "checkbox", or "date". 5. If you want the webhook to close tickets related to **resolved** problems in Zabbix, you can change the following parameter value: - otrs_closed_state_id - ((OTRS)) CE state ID for closed tasks (possible values: 0 - Disable tickets closing, >0 - State ID from the State Management page). 6. If you use the ticket type feature, you can change the type of the created tickets: - otrs_ticket_type - ((OTRS)) CE ticket type (set to "Unclassified" by default; present on fresh installations). 7. Click the "Enabled" checkbox to enable the mediatype and click the "Update" button to save the webhook settings. 8. Create a Zabbix user and add media: - To create a new user, go to the "Users" → "Users" section, click the "Create user" button in the top right corner. In the "User" tab, fill in all required fields (marked with red asterisks). - In the "Media" tab, click "Add" and select the type "Zammad" from the drop-down list. Add any value in the "Send to" field: it is not used in the webhook, but is required. - Make sure this user has access to all hosts for which you would like problem notifications to be sent to Zammad. 9. Done! You can now start using this media type in actions and create tickets. You can find the latest version of this media and additional information in the official Zabbix repository: https://git.zabbix.com/projects/ZBX/repos/zabbix/browse/templates/media/otrs_ce message_templates: - event_source: TRIGGERS operation_mode: PROBLEM subject: 'Problem: {EVENT.NAME}' message: | Problem started at {EVENT.TIME} on {EVENT.DATE} Problem name: {EVENT.NAME} Host: {HOST.NAME} Severity: {EVENT.SEVERITY} Operational data: {EVENT.OPDATA} Original problem ID: {EVENT.ID} {TRIGGER.URL} - event_source: TRIGGERS operation_mode: RECOVERY subject: 'Resolved in {EVENT.DURATION}: {EVENT.NAME}' message: | Problem has been resolved in {EVENT.DURATION} at {EVENT.RECOVERY.TIME} on {EVENT.RECOVERY.DATE} Problem name: {EVENT.NAME} Host: {HOST.NAME} Severity: {EVENT.SEVERITY} Original problem ID: {EVENT.ID} {TRIGGER.URL} - event_source: TRIGGERS operation_mode: UPDATE subject: 'Updated problem in {EVENT.AGE}: {EVENT.NAME}' message: | {USER.FULLNAME} {EVENT.UPDATE.ACTION} problem at {EVENT.UPDATE.DATE} {EVENT.UPDATE.TIME}. {EVENT.UPDATE.MESSAGE} Current problem status is {EVENT.STATUS}, age is {EVENT.AGE}, acknowledged: {EVENT.ACK.STATUS}. - event_source: DISCOVERY operation_mode: PROBLEM subject: 'Discovery: {DISCOVERY.DEVICE.STATUS} {DISCOVERY.DEVICE.IPADDRESS}' message: | Discovery rule: {DISCOVERY.RULE.NAME} Device IP: {DISCOVERY.DEVICE.IPADDRESS} Device DNS: {DISCOVERY.DEVICE.DNS} Device status: {DISCOVERY.DEVICE.STATUS} Device uptime: {DISCOVERY.DEVICE.UPTIME} Device service name: {DISCOVERY.SERVICE.NAME} Device service port: {DISCOVERY.SERVICE.PORT} Device service status: {DISCOVERY.SERVICE.STATUS} Device service uptime: {DISCOVERY.SERVICE.UPTIME} - event_source: AUTOREGISTRATION operation_mode: PROBLEM subject: 'Autoregistration: {HOST.HOST}' message: | Host name: {HOST.HOST} Host IP: {HOST.IP} Agent port: {HOST.PORT} - event_source: INTERNAL operation_mode: PROBLEM subject: '[{EVENT.STATUS}] {EVENT.NAME}' message: | Problem started at {EVENT.TIME} on {EVENT.DATE} Problem name: {EVENT.NAME} Host: {HOST.NAME} Original problem ID: {EVENT.ID} - event_source: INTERNAL operation_mode: RECOVERY subject: '[{EVENT.STATUS}] {EVENT.NAME}' message: | Problem has been resolved in {EVENT.DURATION} at {EVENT.RECOVERY.TIME} on {EVENT.RECOVERY.DATE} Problem name: {EVENT.NAME} Host: {HOST.NAME} Original problem ID: {EVENT.ID} - event_source: SERVICE operation_mode: PROBLEM subject: 'Service "{SERVICE.NAME}" problem: {EVENT.NAME}' message: | Service problem started at {EVENT.TIME} on {EVENT.DATE} Service problem name: {EVENT.NAME} Service: {SERVICE.NAME} Severity: {EVENT.SEVERITY} Original problem ID: {EVENT.ID} Service description: {SERVICE.DESCRIPTION} {SERVICE.ROOTCAUSE} - event_source: SERVICE operation_mode: RECOVERY subject: 'Service "{SERVICE.NAME}" resolved in {EVENT.DURATION}: {EVENT.NAME}' message: | Service "{SERVICE.NAME}" has been resolved at {EVENT.RECOVERY.TIME} on {EVENT.RECOVERY.DATE} Problem name: {EVENT.NAME} Problem duration: {EVENT.DURATION} Severity: {EVENT.SEVERITY} Original problem ID: {EVENT.ID} Service description: {SERVICE.DESCRIPTION} - event_source: SERVICE operation_mode: UPDATE subject: 'Changed "{SERVICE.NAME}" service status to {EVENT.UPDATE.SEVERITY} in {EVENT.AGE}' message: | Changed "{SERVICE.NAME}" service status to {EVENT.UPDATE.SEVERITY} at {EVENT.UPDATE.DATE} {EVENT.UPDATE.TIME}. Current problem age is {EVENT.AGE}. Service description: {SERVICE.DESCRIPTION} {SERVICE.ROOTCAUSE}