Ask or search…
K

Examples

Examples of log patterns identified and optimized by Nimbus.

Logs with common message patterns

These are high volume log events that repeat most of their content. For most applications most of the time, this will be the primary driver of log volume. Examples include health checks and heart beat notifications.
Before
After
Transform
[
{
"ddsource": "nodejs",
"host": "itemrefresh-0",
"message": "refresh item catalogue for itemId: ITEM470",
"path": "/",
"service": "itemrefresh",
"status": "info",
"timestamp": "2023-11-23T00:16:09.970Z"
},
{
"ddsource": "nodejs",
"host": "itemrefresh-0",
"message": "refresh item catalogue for itemId: ITEM8185",
"path": "/",
"service": "itemrefresh",
"status": "info",
"timestamp": "2023-11-23T00:16:09.997Z"
},
{
"ddsource": "nodejs",
"host": "itemrefresh-0",
"message": "refresh item catalogue for itemId: ITEM7594",
"path": "/",
"service": "itemrefresh",
"status": "info",
"timestamp": "2023-11-23T00:16:10.010Z"
},
// 37 more messages
...
]
  • 97.5% volume reduction
{
"ddsource": "nodejs",
"host": "itemrefresh-0",
"path": "/",
"service": "itemrefresh",
"status": "info",
"message": [
"refresh item catalogue for itemId: ITEM470",
"refresh item catalogue for itemId: ITEM8185",
"refresh item catalogue for itemId: ITEM7594",
// 37 more messages
...
],
"nimsize": 40,
"timestamp": "2023-11-23T00:16:09.970Z"
}
- name: itemrefresh
type: reduce
rules:
process_when:
- key: service
op: EQUAL
val: itemrefresh
group_by:
- host
- path
pull_up:
- ddsource
- path
- status
- service
msg_field: message

Logs with common identifiers

These are logs that describe a sequence of related events. These sequences usually have some sort of common identifier like a transactionId or a jobId. Examples include a background job and business specific user flows.
Before
After
Transform
[
{
"ddsource": "nodejs",
"host": "i-02c254cac52b34698",
"message": {
"customerId": "CU6912663",
"ddsource": "nodejs",
"host": "i-02c254cac52b34698",
"itemId": "ITEM1525",
"itemName": "Product 40",
"itemPrice": 74.49291940739313,
"msg": "adding ITEM5151 to cart",
"quantity": 3,
"service": "checkout",
"status": "info",
"timestamp": "2023-11-23T00:15:21.446Z",
"transactionId": "TX60014718"
},
"service": "checkout",
"status": "info",
"timestamp": "2023-11-23T00:15:21.446Z"
},
{
"ddsource": "nodejs",
"host": "i-02c254cac52b34698",
"message": {
"customerId": "CU6912663",
"ddsource": "nodejs",
"discountAmount": 11.292290011912169,
"host": "i-02c254cac52b34698",
"msg": "applying discount DISC16",
"service": "checkout",
"status": "info",
"timestamp": "2023-11-23T00:15:21.446Z",
"transactionId": "TX60014718"
},
"service": "checkout",
"status": "info",
"timestamp": "2023-11-23T00:15:21.473Z"
},
{
"ddsource": "nodejs",
"host": "i-02c254cac52b34698",
"message": {
"customerId": "CU6912663",
"ddsource": "nodejs",
"estimatedDelivery": "2023-11-25",
"host": "i-02c254cac52b34698",
"msg": "calculating shipping info for ITEM5151",
"service": "checkout",
"shippingAddress": "524 Main St, Anytown, AN 87948",
"shippingMethod": "Express",
"status": "info",
"timestamp": "2023-11-23T00:15:21.446Z",
"transactionId": "TX60014718"
},
"service": "checkout",
"status": "info",
"timestamp": "2023-11-23T00:15:21.488Z"
},
{
"ddsource": "nodejs",
"host": "i-02c254cac52b34698",
"message": {
"customerId": "CU6912663",
"ddsource": "nodejs",
"host": "i-02c254cac52b34698",
"msg": "payment for ITEM5151 succeeded",
"paymentMethod": "PayPal",
"service": "checkout",
"status": "info",
"timestamp": "2023-11-23T00:15:21.446Z",
"totalAmount": 36.863789865682904,
"transactionId": "TX60014718"
},
"service": "checkout",
"status": "info",
"timestamp": "2023-11-23T00:15:21.501Z"
}
]
  • 75% reduction in log volume
{
"customerId": "CU6912663",
"ddsource": "nodejs",
"host": "some-host",
"message": [
"adding ITEM5151 to cart",
"applying discount DISC16",
"calculating shipping info for ITEM5151",
"payment for ITEM5151 succeeded"
],
"nimdata": [
{
"ddsource": "nodejs",
"host": "some-host",
"message": {
"customerId": "CU6912663",
"ddsource": "nodejs",
"host": "some-host",
"itemId": "ITEM1525",
"itemName": "Product 40",
"itemPrice": 74.49291940739313,
"msg": "adding ITEM5151 to cart",
"quantity": 3,
"service": "checkout",
"status": "info",
"timestamp": "2023-11-23T00:15:21.446Z",
"transactionId": "TX60014718"
},
"service": "checkout",
"status": "info",
"timestamp": "2023-11-23T00:15:21.446Z"
},
{
"ddsource": "nodejs",
"host": "some-host",
"message": {
"customerId": "CU6912663",
"ddsource": "nodejs",
"discountAmount": 11.292290011912169,
"host": "some-host",
"msg": "applying discount DISC16",
"service": "checkout",
"status": "info",
"timestamp": "2023-11-23T00:15:21.446Z",
"transactionId": "TX60014718"
},
"service": "checkout",
"status": "info",
"timestamp": "2023-11-23T00:15:21.473Z"
},
{
"ddsource": "nodejs",
"host": "some-host",
"message": {
"customerId": "CU6912663",
"ddsource": "nodejs",
"estimatedDelivery": "2023-11-25",
"host": "some-host",
"msg": "calculating shipping info for ITEM5151",
"service": "checkout",
"shippingAddress": "524 Main St, Anytown, AN 87948",
"shippingMethod": "Express",
"status": "info",
"timestamp": "2023-11-23T00:15:21.446Z",
"transactionId": "TX60014718"
},
"service": "checkout",
"status": "info",
"timestamp": "2023-11-23T00:15:21.488Z"
},
{
"ddsource": "nodejs",
"host": "some-host",
"message": {
"customerId": "CU6912663",
"ddsource": "nodejs",
"host": "some-host",
"msg": "payment for ITEM5151 succeeded",
"paymentMethod": "PayPal",
"service": "checkout",
"status": "info",
"timestamp": "2023-11-23T00:15:21.446Z",
"totalAmount": 36.863789865682904,
"transactionId": "TX60014718"
},
"service": "checkout",
"status": "info",
"timestamp": "2023-11-23T00:15:21.501Z"
}
],
"nimkind": "opt",
"nimmatch": "checkout",
"nimsize": 4,
"service": "checkout",
"status": "info",
"timestamp": "2023-11-23T00:15:21.446Z",
"transactionId": "TX60014718"
}
- name: checkout
type: reduce
rules:
# process when service is exactly equal to "checkout"
process_when:
- key: service
op: EQUAL
val: checkout
# make sure these fields are still available at the "top level" instead of being nested
pull_up:
- message.transactionId
- message.customerId
# group all logs by the following top level keys
group_by:
- customerId
- transactionId
# specify the message field, the highlighted body of the log
msg_field: message.msg
# details how custom top level keys will be merged
merge_strategies:
transactionId: discard
customerId: discard

Multi-Line Logs

Many times, an application will emit a single log across multiple lines such as the case with a JSON log. Unless you specifically account for this, most logging agents will consume each newline as a separate log event. Nimbus can identify when this happens and stitch these logs back together.
Before
After
Transform
[
{
"ddsource": "nimbus",
"host": "some-host",
"message": "{",
"path": "/",
"service": "healthcheck",
"source_type": "http_server",
"status": "info",
"timestamp": "2023-11-23T00:05:58.108Z"
},
{
"ddsource": "nimbus",
"host": "some-host",
"message": " \"id\": \"2460\",",
"path": "/",
"service": "healthcheck",
"source_type": "http_server",
"status": "info",
"timestamp": "2023-11-23T00:05:58.134Z"
},
{
"ddsource": "nimbus",
"host": "some-host",
"message": " \"method\": \"GET\",",
"path": "/",
"service": "healthcheck",
"source_type": "http_server",
"status": "info",
"timestamp": "2023-11-23T00:05:58.147Z"
},
{
"ddsource": "nimbus",
"host": "some-host",
"message": " \"url\": \"/health\",",
"path": "/",
"service": "healthcheck",
"source_type": "http_server",
"status": "info",
"timestamp": "2023-11-23T00:05:58.160Z"
},
{
"ddsource": "nimbus",
"host": "some-host",
"message": " \"query\": {},",
"path": "/",
"service": "healthcheck",
"source_type": "http_server",
"status": "info",
"timestamp": "2023-11-23T00:05:58.174Z"
},
{
"ddsource": "nimbus",
"host": "some-host",
"message": " \"params\": {},",
"path": "/",
"service": "healthcheck",
"source_type": "http_server",
"status": "info",
"timestamp": "2023-11-23T00:05:58.187Z"
},
{
"ddsource": "nimbus",
"host": "some-host",
"message": " \"headers\": {",
"path": "/",
"service": "healthcheck",
"source_type": "http_server",
"status": "info",
"timestamp": "2023-11-23T00:05:58.199Z"
},
{
"ddsource": "nimbus",
"host": "some-host",
"message": " \"host\": \"100.119.27.217:8080\",",
"path": "/",
"service": "healthcheck",
"source_type": "http_server",
"status": "info",
"timestamp": "2023-11-23T00:05:58.210Z"
},
{
"ddsource": "nimbus",
"host": "some-host",
"message": " \"user-agent\": \"kube-probe/1.18\",",
"path": "/",
"service": "healthcheck",
"source_type": "http_server",
"status": "info",
"timestamp": "2023-11-23T00:05:58.221Z"
},
{
"ddsource": "nimbus",
"host": "some-host",
"message": " \"accept-encoding\": \"gzip\",",
"path": "/",
"service": "healthcheck",
"source_type": "http_server",
"status": "info",
"timestamp": "2023-11-23T00:05:58.233Z"
},
{
"ddsource": "nimbus",
"host": "some-host",
"message": " \"connection\": \"close\"",
"path": "/",
"service": "healthcheck",
"source_type": "http_server",
"status": "info",
"timestamp": "2023-11-23T00:05:58.245Z"
},
{
"ddsource": "nimbus",
"host": "some-host",
"message": " },",
"path": "/",
"service": "healthcheck",
"source_type": "http_server",
"status": "info",
"timestamp": "2023-11-23T00:05:58.256Z"
},
{
"ddsource": "nimbus",
"host": "some-host",
"message": " \"remoteAddress\": \"::ffff:172.20.65.189\",",
"path": "/",
"service": "healthcheck",
"source_type": "http_server",
"status": "info",
"timestamp": "2023-11-23T00:05:58.269Z"
},
{
"ddsource": "nimbus",
"host": "some-host",
"message": " \"remotePort\": 60444",
"path": "/",
"service": "healthcheck",
"source_type": "http_server",
"status": "info",
"timestamp": "2023-11-23T00:05:58.280Z"
},
{
"ddsource": "nimbus",
"host": "some-host",
"message": "}",
"path": "/",
"service": "healthcheck",
"source_type": "http_server",
"status": "info",
"timestamp": "2023-11-23T00:05:58.292Z"
},
{
"ddsource": "nimbus",
"host": "some-host",
"message": "{",
"path": "/",
"service": "healthcheck",
"source_type": "http_server",
"status": "info",
"timestamp": "2023-11-23T00:05:58.304Z"
},
{
"ddsource": "nimbus",
"host": "some-host",
"message": " \"statusCode\": 200,",
"path": "/",
"service": "healthcheck",
"source_type": "http_server",
"status": "info",
"timestamp": "2023-11-23T00:05:58.316Z"
},
{
"ddsource": "nimbus",
"host": "some-host",
"message": " \"headers\": {",
"path": "/",
"service": "healthcheck",
"source_type": "http_server",
"status": "info",
"timestamp": "2023-11-23T00:05:58.327Z"
},
{
"ddsource": "nimbus",
"host": "some-host",
"message": " \"x-powered-by\": \"Express\"",
"path": "/",
"service": "healthcheck",
"source_type": "http_server",
"status": "info",
"timestamp": "2023-11-23T00:05:58.338Z"
},
{
"ddsource": "nimbus",
"host": "some-host",
"message": " }",
"path": "/",
"service": "healthcheck",
"source_type": "http_server",
"status": "info",
"timestamp": "2023-11-23T00:05:58.350Z"
},
{
"ddsource": "nimbus",
"host": "some-host",
"message": "}",
"path": "/",
"service": "healthcheck",
"source_type": "http_server",
"status": "info",
"timestamp": "2023-11-23T00:05:58.361Z"
}
]
  • 90% volume reduction
{
"ddsource": "nimbus",
"host": "some-host",
"message": "{ \"id\": \"2460\", \"method\": \"GET\", \"url\": \"/health\", \"query\": {}, \"params\": {}, \"headers\": { \"host\": \"100.119.27.217:8080\", \"user-agent\": \"kube-probe/1.18\", \"accept-encoding\": \"gzip\", \"connection\": \"close\" }, \"remoteAddress\": \"::ffff:172.20.65.189\", \"remotePort\": 60444}{ \"statusCode\": 200, \"headers\": { \"x-powered-by\": \"Express\" }}",
"nimdata": [
{
"ddsource": "nimbus",
"host": "some-host",
"message": "{",
"path": "/",
"service": "healthcheck",
"source_type": "http_server",
"status": "info",
"timestamp": "2023-11-23T00:05:58.108Z"
},
{
"ddsource": "nimbus",
"host": "some-host",
"message": " \"id\": \"2460\",",
"path": "/",
"service": "healthcheck",
"source_type": "http_server",
"status": "info",
"timestamp": "2023-11-23T00:05:58.134Z"
},
{
"ddsource": "nimbus",
"host": "some-host",
"message": " \"method\": \"GET\",",
"path": "/",
"service": "healthcheck",
"source_type": "http_server",
"status": "info",
"timestamp": "2023-11-23T00:05:58.147Z"
},
{
"ddsource": "nimbus",
"host": "some-host",
"message": " \"url\": \"/health\",",
"path": "/",
"service": "healthcheck",
"source_type": "http_server",
"status": "info",
"timestamp": "2023-11-23T00:05:58.160Z"
},
{
"ddsource": "nimbus",
"host": "some-host",
"message": " \"query\": {},",
"path": "/",
"service": "healthcheck",
"source_type": "http_server",
"status": "info",
"timestamp": "2023-11-23T00:05:58.174Z"
},
{
"ddsource": "nimbus",
"host": "some-host",
"message": " \"params\": {},",
"path": "/",
"service": "healthcheck",
"source_type": "http_server",
"status": "info",
"timestamp": "2023-11-23T00:05:58.187Z"
},
{
"ddsource": "nimbus",
"host": "some-host",
"message": " \"headers\": {",
"path": "/",
"service": "healthcheck",
"source_type": "http_server",
"status": "info",
"timestamp": "2023-11-23T00:05:58.199Z"
},
{
"ddsource": "nimbus",
"host": "some-host",
"message": " \"host\": \"100.119.27.217:8080\",",
"path": "/",
"service": "healthcheck",
"source_type": "http_server",
"status": "info",
"timestamp": "2023-11-23T00:05:58.210Z"
},
{
"ddsource": "nimbus",
"host": "some-host",
"message": " \"user-agent\": \"kube-probe/1.18\",",
"path": "/",
"service": "healthcheck",
"source_type": "http_server",
"status": "info",
"timestamp": "2023-11-23T00:05:58.221Z"
},
{
"ddsource": "nimbus",
"host": "some-host",
"message": " \"accept-encoding\": \"gzip\",",
"path": "/",
"service": "healthcheck",
"source_type": "http_server",
"status": "info",
"timestamp": "2023-11-23T00:05:58.233Z"
},
{
"ddsource": "nimbus",
"host": "some-host",
"message": " \"connection\": \"close\"",
"path": "/",
"service": "healthcheck",
"source_type": "http_server",
"status": "info",
"timestamp": "2023-11-23T00:05:58.245Z"
},
{
"ddsource": "nimbus",
"host": "some-host",
"message": " },",
"path": "/",
"service": "healthcheck",
"source_type": "http_server",
"status": "info",
"timestamp": "2023-11-23T00:05:58.256Z"
},
{
"ddsource": "nimbus",
"host": "some-host",
"message": " \"remoteAddress\": \"::ffff:172.20.65.189\",",
"path": "/",
"service": "healthcheck",
"source_type": "http_server",
"status": "info",
"timestamp": "2023-11-23T00:05:58.269Z"
},
{
"ddsource": "nimbus",
"host": "some-host",
"message": " \"remotePort\": 60444",
"path": "/",
"service": "healthcheck",
"source_type": "http_server",
"status": "info",
"timestamp": "2023-11-23T00:05:58.280Z"
},
{
"ddsource": "nimbus",
"host": "some-host",
"message": "}",
"path": "/",
"service": "healthcheck",
"source_type": "http_server",
"status": "info",
"timestamp": "2023-11-23T00:05:58.292Z"
},
{
"ddsource": "nimbus",
"host": "some-host",
"message": "{",
"path": "/",
"service": "healthcheck",
"source_type": "http_server",
"status": "info",
"timestamp": "2023-11-23T00:05:58.304Z"
},
{
"ddsource": "nimbus",
"host": "some-host",
"message": " \"statusCode\": 200,",
"path": "/",
"service": "healthcheck",
"source_type": "http_server",
"status": "info",
"timestamp": "2023-11-23T00:05:58.316Z"
},
{
"ddsource": "nimbus",
"host": "some-host",
"message": " \"headers\": {",
"path": "/",
"service": "healthcheck",
"source_type": "http_server",
"status": "info",
"timestamp": "2023-11-23T00:05:58.327Z"
},
{
"ddsource": "nimbus",
"host": "some-host",
"message": " \"x-powered-by\": \"Express\"",
"path": "/",
"service": "healthcheck",
"source_type": "http_server",
"status": "info",
"timestamp": "2023-11-23T00:05:58.338Z"
},
{
"ddsource": "nimbus",
"host": "some-host",
"message": " }",
"path": "/",
"service": "healthcheck",
"source_type": "http_server",
"status": "info",
"timestamp": "2023-11-23T00:05:58.350Z"
},
{
"ddsource": "nimbus",
"host": "some-host",
"message": "}",
"path": "/",
"service": "healthcheck",
"source_type": "http_server",
"status": "info",
"timestamp": "2023-11-23T00:05:58.361Z"
}
],
"nimkind": "opt",
"nimmatch": "healthcheck",
"nimsize": 21,
"path": "/",
"service": "healthcheck",
"source_type": "http_server",
"status": "info",
"timestamp": "2023-11-23T00:05:58.108Z"
}
- name: healthcheck
type: reduce
rules:
process_when:
- key: service
op: EQUAL
val: healthcheck
group_by:
- host
msg_field: message
starts_when:
- key: message
op: MATCH
val: \n\{
merge_strategies:
msg_source: concat_newline
Last modified 16d ago