Examples
Examples of log patterns identified and optimized by Nimbus.
Logs with common message patterns
These are high volume log events that repeat most of their content. For most applications most of the time, this will be the primary driver of log volume. Examples include health checks and heart beat notifications.
[
{
"ddsource": "nodejs",
"host": "itemrefresh-0",
"message": "refresh item catalogue for itemId: ITEM470",
"path": "/",
"service": "itemrefresh",
"status": "info",
"timestamp": "2023-11-23T00:16:09.970Z"
},
{
"ddsource": "nodejs",
"host": "itemrefresh-0",
"message": "refresh item catalogue for itemId: ITEM8185",
"path": "/",
"service": "itemrefresh",
"status": "info",
"timestamp": "2023-11-23T00:16:09.997Z"
},
{
"ddsource": "nodejs",
"host": "itemrefresh-0",
"message": "refresh item catalogue for itemId: ITEM7594",
"path": "/",
"service": "itemrefresh",
"status": "info",
"timestamp": "2023-11-23T00:16:10.010Z"
},
// 37 more messages
...
]97.5% event volume reduction
79% ingest volume reduction
{
"ddsource": "nodejs",
"host": "itemrefresh-0",
"path": "/",
"service": "itemrefresh",
"status": "info",
"message": [
"refresh item catalogue for itemId: ITEM470",
"refresh item catalogue for itemId: ITEM8185",
"refresh item catalogue for itemId: ITEM7594",
// 37 more messages
...
],
"nimsize": 40,
"timestamp": "2023-11-23T00:16:09.970Z",
"timestamp_end": "2023-11-23T00:16:11.322Z"
} - name: itemrefresh
type: reduce
rules:
process_when:
- key: service
op: EQUAL
val: itemrefresh
group_by:
- host
- path
pull_up:
- ddsource
- path
- status
- service
msg_field: messageLogs with common identifiers
These are logs that describe a sequence of related events. These sequences usually have some sort of common identifier like a transactionId or a jobId. Examples include a background job and business specific user flows.
[
{
"ddsource": "nodejs",
"host": "checkout-0",
"message": {
"customerId": "CU26940939",
"itemId": "ITEM1417",
"itemName": "Product 7",
"itemPrice": 2.612748019396105,
"msg": "adding ITEM9798 to cart",
"quantity": 2,
"transactionId": "TX79924095"
},
"service": "checkout",
"status": "info",
"timestamp": "2024-04-26T15:45:14.000000138Z"
},
{
"ddsource": "nodejs",
"host": "checkout-0",
"message": {
"customerId": "CU26940939",
"discountAmount": 13.837782236831986,
"msg": "applying discount DISC16",
"transactionId": "TX79924095"
},
"service": "checkout",
"status": "info",
"timestamp": "2024-04-26T15:45:14.000000831Z"
},
{
"ddsource": "nodejs",
"host": "checkout-0",
"message": {
"customerId": "CU26940939",
"estimatedDelivery": "2023-11-24",
"msg": "calculating shipping info for ITEM9798",
"shippingAddress": "902 Main St, Anytown, AN 68387",
"shippingMethod": "Standard",
"transactionId": "TX79924095"
},
"service": "checkout",
"status": "info",
"timestamp": "2024-04-26T15:45:15.000000523Z"
},
{
"ddsource": "nodejs",
"host": "checkout-0",
"message": {
"customerId": "CU26940939",
"msg": "payment for ITEM9798 succeeded",
"paymentMethod": "PayPal",
"totalAmount": 56.645267111988474,
"transactionId": "TX79924095"
},
"service": "checkout",
"status": "info",
"timestamp": "2024-04-26T15:45:16.000000214Z"
}
]75% reduction in event volume
4% reduction in ingest volume
{
"customerId": "CU26940939",
"ddsource": "nodejs",
"host": "checkout-0",
"message": [
"adding ITEM9798 to cart",
"applying discount DISC16",
"calculating shipping info for ITEM9798",
"payment for ITEM9798 succeeded",
],
"nimdata": [
{
"message": {
"itemId": "ITEM1417",
"itemName": "Product 7",
"itemPrice": 2.612748019396105,
"msg": "adding ITEM9798 to cart",
"quantity": 2
},
"timestamp": "2024-04-26T15:45:14.000000138Z"
},
{
"message": {
"discountAmount": 13.837782236831986,
"msg": "applying discount DISC16"
},
"timestamp": "2024-04-26T15:45:14.000000831Z"
},
{
"message": {
"estimatedDelivery": "2023-11-24",
"msg": "calculating shipping info for ITEM9798",
"shippingAddress": "902 Main St, Anytown, AN 68387",
"shippingMethod": "Standard"
},
"timestamp": "2024-04-26T15:45:15.000000523Z"
},
{
"message": {
"msg": "payment for ITEM9798 succeeded",
"paymentMethod": "PayPal",
"totalAmount": 56.645267111988474
},
"timestamp": "2024-04-26T15:45:16.000000214Z"
}
],
"service": "checkout",
"status": "info",
"timestamp": "2024-04-26T15:45:14.000000138Z",
"timestamp_end": "2024-04-26T15:45:16.000000905Z",
"transactionId": "TX79924095"
}- name: checkout
type: reduce
rules:
# process when service is exactly equal to "checkout"
process_when:
- key: service
op: EQUAL
val: checkout
# make sure these fields are still available at the "top level" instead of being nested
pull_up:
- message.transactionId
- message.customerId
# group all logs by the following top level keys
group_by:
- customerId
- transactionId
# specify the message field, the highlighted body of the log
msg_field: message.msg
# remove unecessary timestamp fields
remove:
- timestamp
- message.timestamp
# details how custom top level keys will be merged
merge_strategies:
transactionId: discard
customerId: discardMulti-Line Logs
Many times, an application will emit a single log across multiple lines such as the case with a JSON log. Unless you specifically account for this, most logging agents will consume each newline as a separate log event. Nimbus can identify when this happens and stitch these logs back together.
[
{
"ddsource": "nimbus",
"host": "some-host",
"message": "{",
"path": "/",
"service": "healthcheck",
"source_type": "http_server",
"status": "info",
"timestamp": "2023-11-23T00:05:58.108Z"
},
{
"ddsource": "nimbus",
"host": "some-host",
"message": " \"id\": \"2460\",",
"path": "/",
"service": "healthcheck",
"source_type": "http_server",
"status": "info",
"timestamp": "2023-11-23T00:05:58.134Z"
},
{
"ddsource": "nimbus",
"host": "some-host",
"message": " \"method\": \"GET\",",
"path": "/",
"service": "healthcheck",
"source_type": "http_server",
"status": "info",
"timestamp": "2023-11-23T00:05:58.147Z"
},
{
"ddsource": "nimbus",
"host": "some-host",
"message": " \"url\": \"/health\",",
"path": "/",
"service": "healthcheck",
"source_type": "http_server",
"status": "info",
"timestamp": "2023-11-23T00:05:58.160Z"
},
{
"ddsource": "nimbus",
"host": "some-host",
"message": " \"query\": {},",
"path": "/",
"service": "healthcheck",
"source_type": "http_server",
"status": "info",
"timestamp": "2023-11-23T00:05:58.174Z"
},
{
"ddsource": "nimbus",
"host": "some-host",
"message": " \"params\": {},",
"path": "/",
"service": "healthcheck",
"source_type": "http_server",
"status": "info",
"timestamp": "2023-11-23T00:05:58.187Z"
},
{
"ddsource": "nimbus",
"host": "some-host",
"message": " \"headers\": {",
"path": "/",
"service": "healthcheck",
"source_type": "http_server",
"status": "info",
"timestamp": "2023-11-23T00:05:58.199Z"
},
{
"ddsource": "nimbus",
"host": "some-host",
"message": " \"host\": \"100.119.27.217:8080\",",
"path": "/",
"service": "healthcheck",
"source_type": "http_server",
"status": "info",
"timestamp": "2023-11-23T00:05:58.210Z"
},
{
"ddsource": "nimbus",
"host": "some-host",
"message": " \"user-agent\": \"kube-probe/1.18\",",
"path": "/",
"service": "healthcheck",
"source_type": "http_server",
"status": "info",
"timestamp": "2023-11-23T00:05:58.221Z"
},
{
"ddsource": "nimbus",
"host": "some-host",
"message": " \"accept-encoding\": \"gzip\",",
"path": "/",
"service": "healthcheck",
"source_type": "http_server",
"status": "info",
"timestamp": "2023-11-23T00:05:58.233Z"
},
{
"ddsource": "nimbus",
"host": "some-host",
"message": " \"connection\": \"close\"",
"path": "/",
"service": "healthcheck",
"source_type": "http_server",
"status": "info",
"timestamp": "2023-11-23T00:05:58.245Z"
},
{
"ddsource": "nimbus",
"host": "some-host",
"message": " },",
"path": "/",
"service": "healthcheck",
"source_type": "http_server",
"status": "info",
"timestamp": "2023-11-23T00:05:58.256Z"
},
{
"ddsource": "nimbus",
"host": "some-host",
"message": " \"remoteAddress\": \"::ffff:172.20.65.189\",",
"path": "/",
"service": "healthcheck",
"source_type": "http_server",
"status": "info",
"timestamp": "2023-11-23T00:05:58.269Z"
},
{
"ddsource": "nimbus",
"host": "some-host",
"message": " \"remotePort\": 60444",
"path": "/",
"service": "healthcheck",
"source_type": "http_server",
"status": "info",
"timestamp": "2023-11-23T00:05:58.280Z"
},
{
"ddsource": "nimbus",
"host": "some-host",
"message": "}",
"path": "/",
"service": "healthcheck",
"source_type": "http_server",
"status": "info",
"timestamp": "2023-11-23T00:05:58.292Z"
},
{
"ddsource": "nimbus",
"host": "some-host",
"message": "{",
"path": "/",
"service": "healthcheck",
"source_type": "http_server",
"status": "info",
"timestamp": "2023-11-23T00:05:58.304Z"
},
{
"ddsource": "nimbus",
"host": "some-host",
"message": " \"statusCode\": 200,",
"path": "/",
"service": "healthcheck",
"source_type": "http_server",
"status": "info",
"timestamp": "2023-11-23T00:05:58.316Z"
},
{
"ddsource": "nimbus",
"host": "some-host",
"message": " \"headers\": {",
"path": "/",
"service": "healthcheck",
"source_type": "http_server",
"status": "info",
"timestamp": "2023-11-23T00:05:58.327Z"
},
{
"ddsource": "nimbus",
"host": "some-host",
"message": " \"x-powered-by\": \"Express\"",
"path": "/",
"service": "healthcheck",
"source_type": "http_server",
"status": "info",
"timestamp": "2023-11-23T00:05:58.338Z"
},
{
"ddsource": "nimbus",
"host": "some-host",
"message": " }",
"path": "/",
"service": "healthcheck",
"source_type": "http_server",
"status": "info",
"timestamp": "2023-11-23T00:05:58.350Z"
},
{
"ddsource": "nimbus",
"host": "some-host",
"message": "}",
"path": "/",
"service": "healthcheck",
"source_type": "http_server",
"status": "info",
"timestamp": "2023-11-23T00:05:58.361Z"
}
]90% in event volume reduction
87% in ingest volume reduction
{
"ddsource": "nimbus",
"host": "some-host",
"message": "{ \"id\": \"2460\", \"method\": \"GET\", \"url\": \"/health\", \"query\": {}, \"params\": {}, \"headers\": { \"host\": \"100.119.27.217:8080\", \"user-agent\": \"kube-probe/1.18\", \"accept-encoding\": \"gzip\", \"connection\": \"close\" }, \"remoteAddress\": \"::ffff:172.20.65.189\", \"remotePort\": 60444}{ \"statusCode\": 200, \"headers\": { \"x-powered-by\": \"Express\" }}",
"nimkind": "opt",
"nimmatch": "healthcheck",
"nimsize": 21,
"path": "/",
"service": "healthcheck",
"source_type": "http_server",
"status": "info",
"timestamp": "2023-11-23T00:05:58.108Z"
}- name: healthcheck
type: reduce
rules:
process_when:
- key: service
op: EQUAL
val: healthcheck
group_by:
- host
msg_field: message
starts_when:
- key: message
op: MATCH
val: \n\{
merge_strategies:
msg_source: concat_newlineLast updated
Was this helpful?