Splunk Cheatsheet

Last triggered by rule

| rest /servicesNS/-/-/saved/searches
| regex title = "$REGEX_to_match_your_rules_standard"
| search disabled=0 request.ui_dispatch_app="$YOUR_RULES_APP"
| rename title as ss_name
| join type=left
    [ search index=_audit action=alert_fired ss_app="$YOUR_RULES_APP"
    | stats latest(trigger_time) as lt by ss_name
    | fields ss_name,lt]
| eval Triggered_time=strftime(lt, "%d/%m/%Y %I:%M:%S %p")
| eval Triggered_time=if(isnull(Triggered_time), "Not Triggered", Triggered_time)
| eval Status=if(Triggered_time=="Not Triggered", "Not Triggered", "Triggered")
| table ss_name, Triggered_time, Status, search

Process consumption?

index=_introspection sourcetype=splunk_resource_usage | stats median(data.pct_memory) by data.process_type

Searches by user

index=_audit action="search" info="completed" NOT user="splunk-system-user"
| table user, is_realtime, total_run_time, exec_time ,result_count
| eval exec_time=strftime(exec_time,"%m/%d/%Y %H:%M:%S:%3Q")
| sort 0 - total_run_time

Use by components

feel free to add statistics in the search about resource_usage_cpu and resource_usage_mem

index=_introspection host=* source=*/resource_usage.log component=PerProcess data.process_type="search"
| stats latest(data.pct_cpu) AS resource_usage_cpu latest(data.mem_used) AS resource_usage_mem by data.pid, _time, data.search_props.type,data.search_props.mode, data.search_props.role,data.search_props.user, data.search_props.app, data.search_props.sid

Long running searches

index="_audit" action="search" (id=* OR search_id=*) | eval user=if(user=="n/a,null(), user) | stats max(total_run_time) as total_run_time first(user) as user by search_id | stats count perc95(total_run_time) median(total_run_time) by user

Searches statistics

index=_audit host=LAB-Splunk-Misc action=search sourcetype=audittrail search_id!="rsa_*"
| eval user = if(user="n/a", null(), user)
| eval search_type = case( match(search_id, "^SummaryDirector_"), "summarization", match(search_id, "^((rt_)?scheduler__|alertsmanager_)"), "scheduled", match(search_id, "\d{10}\.\d+(_[0-9A-F]{8}-[0-9A-F]{4}-[0-9A-F]{4}-[0-9A-F]{4}-[0-9A-F]{12})?$"), "ad hoc", true(), "other")
| eval search=if(isnull(savedsearch_name) OR savedsearch_name=="", search, savedsearch_name)
| stats min(_time) as _time, values(user) as user, max(total_run_time) as total_run_time, first(search) as search, first(search_type) as search_type, first(apiStartTime) as apiStartTime, first(apiEndTime) as apiEndTime by search_id
| where isnotnull(search) AND search_type="ad hoc"
| search user="*"
| fields search, total_run_time, _time, apiStartTime, apiEndTime, search_type, user
| eval earliest = case( like(apiStartTime, "%ZERO_TIME%") AND like(apiEndTime, "%ZERO_TIME%"), "all time", like(apiStartTime, "%ZERO_TIME%"), "-", 1 == 1, apiStartTime )
| eval latest = case( like(apiStartTime, "%ZERO_TIME%") AND like(apiEndTime, "%ZERO_TIME%"), "all time", like(apiEndTime, "%ZERO_TIME%"), "-", 1 == 1, apiEndTime )
| eval search = if(isnotnull(search), search, "N/A")
| eval _time = strftime(_time, "%m/%d/%Y %H:%M:%S %z")
| sort - total_run_time
| fields search, total_run_time, _time, earliest, latest, search_type, user
| rename search as Search, total_run_time as "Search Runtime", _time as "Search Start", earliest as "Earliest Time", latest as "Latest Time", search_type as Type, user as "User"