Monitoring File Director with Splunk

Version 6

    Verified Product Versions

    AppSense DataNow 3.6AppSense DataNow 3.5AppSense DataNow 3.0AppSense DataNow 4.0AppSense DataNow 4.1AppSense DataNow 4.2AppSense File Director 4.3

    Introduction

    This knowledge article provides steps to allow Splunk to receive and correctly interpret File Director syslog events. It also contains sample reports and dashboards

    Detail

    Ensure inputs.conf, props.conf and transforms.conf have been populated as below (The following example is on a single server Splunk deployment dedicated to File Director):

     

    /opt/splunk/etc/system/local/inputs.conf

     

     

     

     

    [default]

     

     

    host = splunk

     

     

     

     

    [tcp:/514]

     

     

    sourcetype = datanow-syslog

     

     

    source=datanow

     

     

     

    /opt/splunk/etc/system/local/props.conf

     

     

     

     

    [datanow-syslog]

     

     

    maxDist = 3

     

     

    TIME_FORMAT = %b %d %H:%M:%S

     

     

    MAX_TIMESTAMP_LOOKAHEAD = 32

     

     

    TRANSFORMS-datanow-syslog = datanow-syslog-host

     

     

    REPORT-datanow-syslog = datanow-syslog-extraction-process,datanow-syslog-extraction-operation,datanow-syslog-extraction-startrequest,datanow-syslog-extraction-status,datanow-syslog-extraction-path,datanow-syslog-extraction-deviceid,datanow-syslog-extraction-sessiontoken,datanow-syslog-extraction-username,datanow-syslog-extraction-error,datanow-syslog-extraction-filesxfer,datanow-syslog-extraction-bytesxfer,datanow-syslog-extraction-localcachesize,datanow-syslog-extraction-localcachefilecount,datanow-syslog-extraction-sizesyncpendingupload,datanow-syslog-extraction-filessyncpendingupload,datanow-syslog-extraction-clientplatform,datanow-syslog-extraction-deviceName,datanow-syslog-extraction-clientDataNowVersion

     

     

    SHOULD_LINEMERGE = False

     

     

     

     

     

    /opt/splunk/etc/system/local/transforms.conf

     

     

     

     

    [datanow-syslog-host]

     

     

    DEST_KEY = MetaData:Host

     

     

    REGEX = :\d\d\s+(?:\d+\s+|(?:user|daemon|local.?)\.\w+\s+)*\[?(\w[\w\.\-]{2,})\]?\s

     

     

    FORMAT = host::$1

     

     

     

     

    [datanow-syslog-extraction-process]

     

     

    REGEX = ^.+\s(.+):\s

     

     

    FORMAT = process::$1

     

     

    CLEAN_KEYS = 1

     

     

    MV_ADD = 0

     

     

     

     

    [datanow-syslog-extraction-requestid]

     

     

    REGEX = "requestId":([^,]+)

     

     

    FORMAT = requestid::$1

     

     

     

     

    [datanow-syslog-extraction-operation]

     

     

    REGEX = "operation":"([^,]+)"

     

     

    FORMAT = operation::$1

     

     

     

     

    [datanow-syslog-extraction-startrequest]

     

     

    REGEX = "startRequest":([^,]+)

     

     

    FORMAT = startrequest::$1

     

     

     

     

    [datanow-syslog-extraction-status]

     

     

    REGEX = "status":([^,]+)

     

     

    FORMAT = status::$1

     

     

     

     

    [datanow-syslog-extraction-path]

     

     

    REGEX = "path":(?:null|"([^,]+)")

     

     

    FORMAT = path::$1

     

     

     

     

    [datanow-syslog-extraction-deviceid]

     

     

    REGEX = "deviceId":(?:null|"([^,]+)")

     

     

    FORMAT = deviceid::$1

     

     

     

     

    [datanow-syslog-extraction-sessiontoken]

     

     

    REGEX = "sessionToken":(?:null|"([^,]+)")

     

     

    FORMAT = sessiontoken::$1

     

     

     

     

    [datanow-syslog-extraction-username]

     

     

    REGEX = "userName":(?:null|"([^,]+)")

     

     

    FORMAT = username::$1

     

     

     

     

    [datanow-syslog-extraction-error]

     

     

    REGEX = "error":(?:null|"([^,]+)")

     

     

    FORMAT = error::$1

     

     

     

     

    [datanow-syslog-extraction-filesxfer]

     

     

    REGEX = "filesXfer":([^,]+)

     

     

    FORMAT = filesxfer::$1

     

     

     

     

    [datanow-syslog-extraction-bytesxfer]

     

     

    REGEX = "bytesXfer":([^,]+)

     

     

    FORMAT = bytesxfer::$1

     

     

     

     

    [datanow-syslog-extraction-localcachesize]

     

     

    REGEX = "localCacheSize":([^,]+)

     

     

    FORMAT = localCacheSize::$1

     

     

     

     

    [datanow-syslog-extraction-localcachefilecount]

     

     

    REGEX = "localCacheFileCount":([^,]+)

     

     

    FORMAT = localCacheFileCount::$1

     

     

     

     

    [datanow-syslog-extraction-sizesyncpendingupload]

     

     

    REGEX = "sizeSyncPendingUpload":([^,]+)

     

     

    FORMAT = sizeSyncPendingUpload::$1

     

     

     

     

    [datanow-syslog-extraction-filessyncpendingupload]

     

     

    REGEX = "filesSyncPendingUpload":([^,]+)

     

     

    FORMAT = filesSyncPendingUpload::$1

     

     

     

     

    [datanow-syslog-extraction-clientplatform]

     

     

    REGEX = "clientPlatform":"([^,]+)"

     

     

    FORMAT = "clientPlatform"::$1

     

     

     

     

    [datanow-syslog-extraction-deviceName]

     

    REGEX = "deviceName":([^,]+)

     

    FORMAT = deviceName::$1

     

     

     

     

    [datanow-syslog-extraction-clientDataNowVersion]

     

    REGEX = "clientDataNowVersion":([^,]+)

     

    FORMAT = clientDataNowVersion::$1

     

     

     

    Configure Data Models:

     

     

     

     

     

     

    Upload the following attached .json files to create the Data models:

     

     

     

    See Attachment : MB_Remaining.json (below)

     

    See Attachment : Sync_state_of_users.json (below)

     

    See Attachment : Total_Megabytes.json (below)

     

     

     

    CAUTION: The names must match those in the graphic for the sample reports to work correctly

     

     

     

    Configure Custom Searches, Reports and Alerts:

     

     

     

     

     

     

     

     

    Click 'New' and add the following items:

     

    Name: MB Remaining for Users

     

    Expression:

     

     

    | pivot MB_Remaining SearchObject max(megabytes_remaining) AS "Max of megabytes_remaining" SPLITROW username AS username SORT 100 username ROWSUMMARY 0 COLSUMMARY 0 NUMCOLS 0 SHOWOTHER 1

     

     

     

     

     

    Name: Sync State of Users

     

    Expression:

     

     

    | pivot Sync_state_of_users SearchObject max(synced_percentage) AS "Max of synced_percentage" SPLITROW username AS username FILTER synced_percentage = * SORT 100 username ROWSUMMARY 0 COLSUMMARY 0 NUMCOLS 0 SHOWOTHER 1

     

     

     

     

     

    Name: Total Megabytes

     

    Expression:

     

     

    | pivot Total_Megabytes SearchObject max(total_megabytes) AS "Total Megabytes" SPLITROW username AS username SORT 100 username ROWSUMMARY 0 COLSUMMARY 0 NUMCOLS 0 SHOWOTHER 1

     

     

     

     

     

    Optional steps to create an experimental dashboard for demo purposes:

     

    Create a new dashboard:

     

     

     

     

    Click 'edit / edit source' once you have created the new Dashboard and paste the following XML:

     

    See Attachment : dashboard.xml (below)

     

    Click 'save' to complete

     

     

    THIS SOLUTION AND INFORMATION IS PROVIDED "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE IMPLIED WARRANTIES OF MERCHANTABILITY AND/OR FITNESS FOR A PARTICULAR PURPOSE.

     

    IMPORTANT: Please take care when executing any changes to your systems. We strongly recommend a full system backup is performed prior to use of the supplied solution.

     

     

     

     

     

    References

    www.splunk.com