Skip to content

Commit 0630394

Browse files
philjleephiljlee
committed
[AI-6669] Add NiFi log pipeline, assets, manifest, and dashboard
Rebased onto master now that the AI-6668 implementation is merged. This PR completes the integration tile by adding assets the implementation PR deliberately left out of scope. Adds: - `manifest.json` with metrics, events, logs, dashboard, saved views - `assets/dashboards/nifi_overview.json` overview dashboard - `assets/dataflows.yaml` declaring nifi-metrics, nifi-events, nifi-logs - `assets/logs/nifi.yaml` pipeline with nested sub-pipelines for app logs and Combined Log Format request logs - `assets/logs/nifi_tests.yaml` with 5 pipeline test cases - `assets/saved_views/` for error logs and HTTP access logs - `images/IMAGES_README.md` media gallery guide - Log config entries in `spec.yaml` and `conf.yaml.example` for all four NiFi log files (app, user, bootstrap, request) - README log collection section with multiline stack trace guidance Drops the earlier `service_checks.json` and its manifest reference — the check only emits the `nifi.can_connect` gauge, not a service check, so the README already documents connectivity there.
1 parent d7f8f86 commit 0630394

11 files changed

Lines changed: 629 additions & 0 deletions

File tree

‎nifi/README.md‎

Lines changed: 50 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,56 @@ No additional installation is needed on your server.
3131
3232
3. [Restart the Agent][5].
3333
34+
#### Log collection
35+
36+
1. Collecting logs is disabled by default in the Datadog Agent. Enable it in your `datadog.yaml` file:
37+
38+
```yaml
39+
logs_enabled: true
40+
```
41+
42+
2. Uncomment and edit the logs configuration block in your `nifi.d/conf.yaml` file. NiFi produces several log files; configure the ones relevant to your environment:
43+
44+
```yaml
45+
logs:
46+
- type: file
47+
path: /opt/nifi/logs/nifi-app.log
48+
source: nifi
49+
service: nifi
50+
- type: file
51+
path: /opt/nifi/logs/nifi-user.log
52+
source: nifi
53+
service: nifi
54+
- type: file
55+
path: /opt/nifi/logs/nifi-bootstrap.log
56+
source: nifi
57+
service: nifi
58+
- type: file
59+
path: /opt/nifi/logs/nifi-request.log
60+
source: nifi
61+
service: nifi
62+
tags:
63+
- "log_type:request"
64+
```
65+
66+
The `log_type:request` tag on the request log entry routes HTTP access logs through a dedicated parsing pipeline that extracts standard HTTP attributes (method, status code, URL path, client IP).
67+
68+
NiFi is a Java application that produces multiline stack traces. To aggregate them into single log events, add a `log_processing_rules` entry to the application log:
69+
70+
```yaml
71+
logs:
72+
- type: file
73+
path: /opt/nifi/logs/nifi-app.log
74+
source: nifi
75+
service: nifi
76+
log_processing_rules:
77+
- type: multi_line
78+
name: java_stack_trace
79+
pattern: \d{4}-\d{2}-\d{2}
80+
```
81+
82+
3. [Restart the Agent][5].
83+
3484
### Validation
3585

3686
[Run the Agent's status subcommand][6] and look for `nifi` under the Checks section.

‎nifi/assets/configuration/spec.yaml‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -87,3 +87,17 @@ files:
8787
path: /opt/nifi/logs/nifi-app.log
8888
source: nifi
8989
service: nifi
90+
- type: file
91+
path: /opt/nifi/logs/nifi-user.log
92+
source: nifi
93+
service: nifi
94+
- type: file
95+
path: /opt/nifi/logs/nifi-bootstrap.log
96+
source: nifi
97+
service: nifi
98+
- type: file
99+
path: /opt/nifi/logs/nifi-request.log
100+
source: nifi
101+
service: nifi
102+
tags:
103+
- "log_type:request"

‎nifi/assets/dashboards/nifi_overview.json‎

Lines changed: 113 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,113 @@
1+
{
2+
"title": "NiFi Overview",
3+
"description": "Overview of Apache NiFi health, flow throughput, and queue status.",
4+
"widgets": [
5+
{
6+
"id": 0,
7+
"definition": {
8+
"title": "Can Connect",
9+
"type": "query_value",
10+
"requests": [
11+
{
12+
"q": "avg:nifi.can_connect{*}",
13+
"aggregator": "last"
14+
}
15+
],
16+
"precision": 0
17+
},
18+
"layout": {"x": 0, "y": 0, "width": 2, "height": 2}
19+
},
20+
{
21+
"id": 1,
22+
"definition": {
23+
"title": "JVM Heap Utilization",
24+
"type": "timeseries",
25+
"requests": [
26+
{
27+
"q": "avg:nifi.system.jvm.heap_utilization{*}",
28+
"display_type": "line"
29+
}
30+
]
31+
},
32+
"layout": {"x": 2, "y": 0, "width": 5, "height": 2}
33+
},
34+
{
35+
"id": 2,
36+
"definition": {
37+
"title": "Flow - Running Processors",
38+
"type": "query_value",
39+
"requests": [
40+
{
41+
"q": "avg:nifi.flow.running_count{*}",
42+
"aggregator": "last"
43+
}
44+
],
45+
"precision": 0
46+
},
47+
"layout": {"x": 7, "y": 0, "width": 2, "height": 2}
48+
},
49+
{
50+
"id": 3,
51+
"definition": {
52+
"title": "FlowFiles Queued",
53+
"type": "timeseries",
54+
"requests": [
55+
{
56+
"q": "avg:nifi.flow.flowfiles_queued{*}",
57+
"display_type": "bars"
58+
}
59+
]
60+
},
61+
"layout": {"x": 0, "y": 2, "width": 4, "height": 2}
62+
},
63+
{
64+
"id": 4,
65+
"definition": {
66+
"title": "GC Collection Rate",
67+
"type": "timeseries",
68+
"requests": [
69+
{
70+
"q": "avg:nifi.system.gc.collection_count{*} by {gc_name}",
71+
"display_type": "line"
72+
}
73+
]
74+
},
75+
"layout": {"x": 4, "y": 2, "width": 5, "height": 2}
76+
},
77+
{
78+
"id": 5,
79+
"definition": {
80+
"title": "Active Threads",
81+
"type": "timeseries",
82+
"requests": [
83+
{
84+
"q": "avg:nifi.flow.active_threads{*}",
85+
"display_type": "line"
86+
}
87+
]
88+
},
89+
"layout": {"x": 0, "y": 4, "width": 4, "height": 2}
90+
},
91+
{
92+
"id": 6,
93+
"definition": {
94+
"title": "Repository Utilization",
95+
"type": "timeseries",
96+
"requests": [
97+
{
98+
"q": "avg:nifi.system.flowfile_repo.utilization{*}",
99+
"display_type": "line"
100+
},
101+
{
102+
"q": "avg:nifi.system.content_repo.utilization{*}",
103+
"display_type": "line"
104+
}
105+
]
106+
},
107+
"layout": {"x": 4, "y": 4, "width": 5, "height": 2}
108+
}
109+
],
110+
"template_variables": [],
111+
"layout_type": "ordered",
112+
"notify_list": []
113+
}

‎nifi/assets/dataflows.yaml‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
provides:
2+
- id: nifi-metrics
3+
always_on: true
4+
data_type: metrics
5+
direction: inbound
6+
- id: nifi-events
7+
always_on: true
8+
data_type: events
9+
direction: inbound
10+
- id: nifi-logs
11+
always_on: false
12+
data_type: logs
13+
direction: inbound

‎nifi/assets/logs/nifi.yaml‎

Lines changed: 148 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,148 @@
1+
id: nifi
2+
metric_id: nifi
3+
backend_only: false
4+
facets:
5+
- groups:
6+
- Source Code
7+
name: Logger Name
8+
path: logger.name
9+
source: log
10+
- groups:
11+
- Source Code
12+
name: Thread Name
13+
path: logger.thread_name
14+
source: log
15+
- groups:
16+
- Web Access
17+
name: Method
18+
path: http.method
19+
source: log
20+
- groups:
21+
- Web Access
22+
name: Status Code
23+
path: http.status_code
24+
source: log
25+
- groups:
26+
- Web Access
27+
name: URL Path
28+
path: http.url_details.path
29+
source: log
30+
- groups:
31+
- Web Access
32+
name: Client IP
33+
path: network.client.ip
34+
source: log
35+
pipeline:
36+
type: pipeline
37+
name: NiFi
38+
enabled: true
39+
filter:
40+
query: source:nifi
41+
processors:
42+
- type: pipeline
43+
name: NiFi application logs
44+
enabled: true
45+
filter:
46+
query: -log_type:request
47+
processors:
48+
- type: grok-parser
49+
name: Parsing NiFi application logs
50+
enabled: true
51+
source: message
52+
samples:
53+
- "2026-03-20 18:08:33,065 INFO [main] o.a.nifi.NiFi NiFi PID is 1"
54+
- "2026-03-20 18:08:33,065 ERROR [Timer-Driven Process Thread-1] o.a.n.p.standard.PutFile PutFile[id=abc] Failed to write"
55+
- "2026-03-20 18:08:34,100 WARN [Flow Service Tasks Thread-1] o.a.n.c.s.impl.WriteAheadFlowFileRepository Encountered non-critical error"
56+
grok:
57+
supportRules: |
58+
_date %{date("yyyy-MM-dd HH:mm:ss,SSS"):timestamp}
59+
_level %{word:level}
60+
_thread %{regex("[^]]*"):logger.thread_name}
61+
_logger %{notSpace:logger.name}
62+
matchRules: |
63+
nifi_app %{_date} %{_level} \[%{_thread}\] %{_logger} %{data:message}
64+
- type: date-remapper
65+
name: Define `timestamp` as the official date of the log
66+
enabled: true
67+
sources:
68+
- timestamp
69+
- type: status-remapper
70+
name: Define `level` as the official status of the log
71+
enabled: true
72+
sources:
73+
- level
74+
- type: message-remapper
75+
name: Define `message` as the official message of the log
76+
enabled: true
77+
sources:
78+
- message
79+
- type: pipeline
80+
name: NiFi HTTP request logs
81+
enabled: true
82+
filter:
83+
query: log_type:request
84+
processors:
85+
- type: grok-parser
86+
name: Parsing NiFi HTTP request logs
87+
enabled: true
88+
source: message
89+
samples:
90+
- '192.168.1.100 - admin [20/Mar/2026:18:08:33 +0000] "GET /nifi-api/flow/status HTTP/1.1" 200 1234 "-" "python-requests/2.31.0"'
91+
- '10.0.0.50 - - [20/Mar/2026:18:10:45 +0000] "POST /nifi-api/process-groups/root/process-groups HTTP/1.1" 404 892 "https://localhost:8443/nifi/" "Mozilla/5.0"'
92+
grok:
93+
supportRules: |
94+
_auth %{notSpace:http.auth:nullIf("-")}
95+
_bytes_written %{integer:network.bytes_written}
96+
_client_ip %{ipOrHost:network.client.ip}
97+
_version HTTP\/%{regex("\\d+\\.\\d+"):http.version}
98+
_url %{notSpace:http.url}
99+
_ident %{notSpace:http.ident:nullIf("-")}
100+
_user_agent %{regex("[^\\\"]*"):http.useragent}
101+
_referer %{notSpace:http.referer}
102+
_status_code %{integer:http.status_code}
103+
_method %{word:http.method}
104+
_date_access %{date("dd/MMM/yyyy:HH:mm:ss Z"):date_access}
105+
matchRules: |
106+
access.common %{_client_ip} %{_ident} %{_auth} \[%{_date_access}\] "(?>%{_method} |)%{_url}(?> %{_version}|)" %{_status_code} (?>%{_bytes_written}|-)
107+
108+
access.combined %{access.common} "%{_referer}" "%{_user_agent}"
109+
- type: url-parser
110+
name: Extract URL details
111+
enabled: true
112+
sources:
113+
- http.url
114+
target: http.url_details
115+
- type: user-agent-parser
116+
name: Extract user agent details
117+
enabled: true
118+
sources:
119+
- http.useragent
120+
target: http.useragent_details
121+
encoded: false
122+
- type: date-remapper
123+
name: Define `date_access` as the official date of the log
124+
enabled: true
125+
sources:
126+
- date_access
127+
- type: category-processor
128+
name: Categorize status code
129+
enabled: true
130+
categories:
131+
- filter:
132+
query: "@http.status_code:[200 TO 299]"
133+
name: OK
134+
- filter:
135+
query: "@http.status_code:[300 TO 399]"
136+
name: notice
137+
- filter:
138+
query: "@http.status_code:[400 TO 499]"
139+
name: warning
140+
- filter:
141+
query: "@http.status_code:[500 TO 599]"
142+
name: error
143+
target: http.status_category
144+
- type: status-remapper
145+
name: Define `http.status_category` as the official status of the log
146+
enabled: true
147+
sources:
148+
- http.status_category

0 commit comments

Comments
 (0)