关于logstash:将-nxlog-采集的-windows-日志转换为标准化-ELK-ECS-日志

一、用 nxlog 采集 windows 日志

#######################################################################
####                         根底配置                              #####
#######################################################################

# 64 零碎
define ROOT C:\Program Files (x86)\nxlog

# 32 零碎
#define ROOT C:\Program Files\nxlog

Moduledir %ROOT%\modules
CacheDir %ROOT%\data
Pidfile %ROOT%\data\nxlog.pid
SpoolDir %ROOT%\data
LogFile %ROOT%\data\nxlog.log

# 开启 GELF 格局扩大，并定义最大日志长度

<Extension gelf>
    Module xm_gelf
    ShortMessageLength 65536
</Extension>

# 开启 JSON 扩大

 <Extension json>
    Module      xm_json
</Extension>

# 开启主动转码

<Extension _charconv>
    Module      xm_charconv
    AutodetectCharsets iso8859-2, utf-8, utf-16, utf-32
</Extension>

#######################################################################
####                     输出配置 -windows 日志                      #####
#######################################################################


#   因为 NXLOG 社区版本最大只能发送 256 个 Channel，而目前 windows2016，曾经超过 300 个, 因而局部日志采集不到, 须要手动查问通道，来避免单次查问超过 256

<Input APP_Logs>

#   WIN7 以上
    Module      im_msvistalog

#   WINDOWS 2003
#    Module      im_mseventlog

    #   查问 Application 通道
    Query <QueryList><Query Id="0"><Select Path="Application">*</Select></Query></QueryList>

    #   过滤所有类型为具体的事件类型
    Exec if $EventType == 'VERBOSE' drop();
    Exec $Hostname = hostname();
</Input>


<Input SYS_Logs>

#   WIN7 以上
    Module      im_msvistalog

#   WINDOWS 2003
#    Module      im_mseventlog

    #   查问 System 通道
    Query <QueryList><Query Id="0"><Select Path="System">*</Select></Query></QueryList>

    #   过滤所有类型为具体的事件类型
    Exec if $EventType == 'VERBOSE' drop();
    Exec $Hostname = hostname();
</Input>

<Input SEC_Logs>

#   WIN7 以上
    Module      im_msvistalog

#   WINDOWS 2003
#    Module      im_mseventlog
    #   查问 Security 通道
    Query <QueryList><Query Id="0"><Select Path="Security">*</Select></Query></QueryList>

    #   过滤所有类型为具体的事件类型
    Exec if $EventType == 'VERBOSE' drop();
    Exec $Hostname = hostname();

</Input>


#######################################################################
####                     输入配置                           #####
#######################################################################

<Output Logstash>
    Module      om_udp
    Host        logstash-ip
    Port        5414
    OutputType  GELF

</Output>


<Route APP>
    Path APP_Logs => Logstash
</Route>

<Route SYS>
    Path SYS_Logs => Logstash
</Route>

<Route SYS>
    Path SEC_Logs => Logstash
</Route>

二、用 logstash，将 windows 日志进一步整顿

备注：二和三能够合并在一起，而无需 kafka

input {
  gelf {
    use_udp => "true"
    codec => json_lines {charset => CP1252}
    port => "5414"
    id => "winlog"
  }
}
# 这里过滤规定跟 winlogbeat 保持一致

filter {if [Channel] == "Security" or [Channel] == "Application" or  [Channel] == "System" {} else if [Channel] == "Windows PowerShell" {if [EventID] != 400 and [EventID] != 403 and [EventID] != 600 and [EventID] != 800 {drop {}
        }

  } else if [Channel] == "Microsoft-Windows-PowerShell/Operational" {if [EventID] != 4103 and [EventID] != 4104 and [EventID] != 4105 and [EventID] != 4106 {drop {}
        }

  } else {drop {}
  }


}
output {
  kafka {
    bootstrap_servers => "kafka-ip"
    topic_id => "winlog"
    codec => "json"
  }
}

三、将 windows gelf 日志格局换成为 ecs 规范格局

input {
    kafka {
        bootstrap_servers => "kafka-ip"
        client_id => "winlog"
        group_id => "logstash-es-winlog"
        auto_offset_reset => "latest"
        consumer_threads => 1
        decorate_events => "true"
        topics => ["winlog"]
        codec => "json"
        }
}

filter {if [Channel] == "Windows PowerShell"  {if [EventID] == "800" {
        mutate {add_field => { "[winlog][event_data][param2]" => "%{message}" }
        }
        kv {source => "[winlog][event_data][param2]"
           target => "[winlog][event_data]"
           field_split => "\n\t"
           trim_key => "\n\t"
           trim_value => "\n\t"
           value_split => "="
        }
    }
    else {
        mutate {add_field => { "[winlog][event_data][param3]" => "%{message}" }
        }
        kv {source => "[winlog][event_data][param3]"
           target => "[winlog][event_data]"
           field_split => "\n\t"
           trim_key => "\n\t"
           trim_value => "\n\t"
           value_split => "="
        }
    }
  }



  # ECS fields
  mutate {
    rename => {
      "host" => "source_name"
      "SeverityValue" => "[event][severity]"
      "EventReceivedTime" => "[event][time][received]"
      "SourceModuleType" => "[nxlog][module][type]"
      "SourceModuleName" => "[nxlog][module][name]"
      "Severity" => "[winlog][level]"
      "ThreadID" => "[winlog][process][thread][id]"
      "ProcessID" => "[winlog][process][id]"
    }
    add_field => {"[agent][name]" => "%{source_name}"
      "[agent][version]" => "2.10.2150"
      "[host][ip]" => "%{source_host}"
      "[host][name]" => "%{source_name}"
      "[host][os][platform]" => "windows"
      "[host][os][type]" => "windows"
      "[event]" => "%{EventID}"
    }
  }

  # 非凡字段, 占用了默认开展字段
  mutate {
    rename => {"url" => "[winlog][event_data][url]"
      "bytesTransferred" => "[winlog][event_data][bytesTransferred]"
      "fileLength" => "[winlog][event_data][fileLength]"
      "bytesTotal" => "[winlog][event_data][bytesTotal]"
      "error" => "[winlog][event_data][error]"
      "destination" => "[winlog][event_data][destination]"
      "bytesTransferredFromPeer" => "[winlog][event_data][bytesTransferredFromPeer]"
      "source" => "[winlog][event_data]"
    }
  }
  # winlog
  mutate {
    add_field => {"[winlog][api]" => "wineventlog"
    }
    rename => {"ActivityID" => "[winlog][activity_id]"
      "EventID" => "[winlog][event_id]"
      "EventType" => "[winlog][keywords]"
      "Channel" => "[winlog][channel]"
      "RecordNumber" => "[winlog][record_id]"
      "Opcode" => "[winlog][opcode]"
      "ProviderGuid" => "[winlog][provider_guid]"
      "SourceName" => "[winlog][provider_name]"
      "Category" => "[winlog][task]"
      "Version" => "[winlog][version]"
    }
  }
  # event_data
  mutate {
    rename => {"AuthenticationPackageName" => "[winlog][event_data][AuthenticationPackageName]"
      "Binary" => "[winlog][event_data][Binary]"
      "BitlockerUserInputTime" => "[winlog][event_data][BitlockerUserInputTime]"
      "BootMode" => "[winlog][event_data][BootMode]"
      "BootType" => "[winlog][event_data][BootType]"
      "BuildVersion" => "[winlog][event_data][BuildVersion]"
      "Company" => "[winlog][event_data][Company]"
      "CorruptionActionState" => "[winlog][event_data][CorruptionActionState]"
      "CreationUtcTime" => "[winlog][event_data][CreationUtcTime]"
      "Description" => "[winlog][event_data][Description]"
      "Detail" => "[winlog][event_data][Detail]"
      "DeviceName" => "[winlog][event_data][DeviceName]"
      "DeviceNameLength" => "[winlog][event_data][DeviceNameLength]"
      "DeviceTime" => "[winlog][event_data][DeviceTime]"
      "DeviceVersionMajor" => "[winlog][event_data][DeviceVersionMajor]"
      "DeviceVersionMinor" => "[winlog][event_data][DeviceVersionMinor]"
      "DriveName" => "[winlog][event_data][DriveName]"
      "DriverName" => "[winlog][event_data][DriverName]"
      "DriverNameLength" => "[winlog][event_data][DriverNameLength]"
      "DwordVal" => "[winlog][event_data][DwordVal]"
      "EntryCount" => "[winlog][event_data][EntryCount]"
      "ExtraInfo" => "[winlog][event_data][ExtraInfo]"
      "FailureName" => "[winlog][event_data][FailureName]"
      "FailureNameLength" => "[winlog][event_data][FailureNameLength]"
      "FileVersion" => "[winlog][event_data][FileVersion]"
      "FinalStatus" => "[winlog][event_data][FinalStatus]"
      "Group" => "[winlog][event_data][Group]"
      "IdleImplementation" => "[winlog][event_data][IdleImplementation]"
      "IdleStateCount" => "[winlog][event_data][IdleStateCount]"
      "ImpersonationLevel" => "[winlog][event_data][ImpersonationLevel]"
      "IntegrityLevel" => "[winlog][event_data][IntegrityLevel]"
      "IpAddress" => "[winlog][event_data][IpAddress]"
      "IpPort" => "[winlog][event_data][IpPort]"
      "KeyLength" => "[winlog][event_data][KeyLength]"
      "LastBootGood" => "[winlog][event_data][LastBootGood]"
      "LastShutdownGood" => "[winlog][event_data][LastShutdownGood]"
      "LmPackageName" => "[winlog][event_data][LmPackageName]"
      "LogonGuid" => "[winlog][event_data][LogonGuid]"
      "LogonId" => "[winlog][event_data][LogonId]"
      "LogonProcessName" => "[winlog][event_data][LogonProcessName]"
      "LogonType" => "[winlog][event_data][LogonType]"
      "MajorVersion" => "[winlog][event_data][MajorVersion]"
      "MaximumPerformancePercent" => "[winlog][event_data][MaximumPerformancePercent]"
      "MemberName" => "[winlog][event_data][MemberName]"
      "MemberSid" => "[winlog][event_data][MemberSid]"
      "MinimumPerformancePercent" => "[winlog][event_data][MinimumPerformancePercent]"
      "MinimumThrottlePercent" => "[winlog][event_data][MinimumThrottlePercent]"
      "MinorVersion" => "[winlog][event_data][MinorVersion]"
      "NewProcessId" => "[winlog][event_data][NewProcessId]"
      "NewProcessName" => "[winlog][event_data][NewProcessName]"
      "NewSchemeGuid" => "[winlog][event_data][NewSchemeGuid]"
      "NewTime" => "[winlog][event_data][NewTime]"
      "NominalFrequency" => "[winlog][event_data][NominalFrequency]"
      "Number" => "[winlog][event_data][Number]"
      "OldSchemeGuid" => "[winlog][event_data][OldSchemeGuid]"
      "OldTime" => "[winlog][event_data][OldTime]"
      "OriginalFileName" => "[winlog][event_data][OriginalFileName]"
      "Path" => "[winlog][event_data][Path]"
      "PerformanceImplementation" => "[winlog][event_data][PerformanceImplementation]"
      "PreviousCreationUtcTime" => "[winlog][event_data][PreviousCreationUtcTime]"
      "PreviousTime" => "[winlog][event_data][PreviousTime]"
      "PrivilegeList" => "[winlog][event_data][PrivilegeList]"
      "ProcessId" => "[winlog][event_data][ProcessId]"
      "ProcessName" => "[winlog][event_data][ProcessName]"
      "ProcessPath" => "[winlog][event_data][ProcessPath]"
      "ProcessPid" => "[winlog][event_data][ProcessPid]"
      "Product" => "[winlog][event_data][Product]"
      "PuaCount" => "[winlog][event_data][PuaCount]"
      "PuaPolicyId" => "[winlog][event_data][PuaPolicyId]"
      "QfeVersion" => "[winlog][event_data][QfeVersion]"
      "Reason" => "[winlog][event_data][Reason]"
      "SchemaVersion" => "[winlog][event_data][SchemaVersion]"
      "ServiceName" => "[winlog][event_data][ServiceName]"
      "ServiceVersion" => "[winlog][event_data][ServiceVersion]"
      "ShutdownActionType" => "[winlog][event_data][ShutdownActionType]"
      "ShutdownEventCode" => "[winlog][event_data][ShutdownEventCode]"
      "ShutdownReason" => "[winlog][event_data][ShutdownReason]"
      "Signature" => "[winlog][event_data][Signature]"
      "SignatureStatus" => "[winlog][event_data][SignatureStatus]"
      "Signed" => "[winlog][event_data][Signed]"
      "StartTime" => "[winlog][event_data][StartTime]"
      "State" => "[winlog][event_data][State]"
      "Status" => "[winlog][event_data][Status]"
      "StopTime" => "[winlog][event_data][StopTime]"
      "SubjectDomainName" => "[winlog][event_data][SubjectDomainName]"
      "SubjectLogonId" => "[winlog][event_data][SubjectLogonId]"
      "SubjectUserName" => "[winlog][event_data][SubjectUserName]"
      "SubjectUserSid" => "[winlog][event_data][SubjectUserSid]"
      "TSId" => "[winlog][event_data][TSId]"
      "TargetDomainName" => "[winlog][event_data][TargetDomainName]"
      "TargetInfo" => "[winlog][event_data][TargetInfo]"
      "TargetLogonGuid" => "[winlog][event_data][TargetLogonGuid]"
      "TargetLogonId" => "[winlog][event_data][TargetLogonId]"
      "TargetServerName" => "[winlog][event_data][TargetServerName]"
      "TargetUserName" => "[winlog][event_data][TargetUserName]"
      "TargetUserSid" => "[winlog][event_data][TargetUserSid]"
      "TerminalSessionId" => "[winlog][event_data][TerminalSessionId]"
      "TokenElevationType" => "[winlog][event_data][TokenElevationType]"
      "TransmittedServices" => "[winlog][event_data][TransmittedServices]"
      "UserSid" => "[winlog][event_data][UserSid]"
      "Version" => "[winlog][event_data][Version]"
      "param1" => "[winlog][event_data][param1]"
      "param2" => "[winlog][event_data][param2]"
      "param3" => "[winlog][event_data][param3]"
      "param4" => "[winlog][event_data][param4]"
      "param5" => "[winlog][event_data][param5]"
      "param6" => "[winlog][event_data][param6]"
      "param7" => "[winlog][event_data][param7]"
      "param8" => "[winlog][event_data][param8]"
    }
  }
  # event_data 查漏补缺
  mutate {
    rename => {"AccessList" => "[winlog][event_data][AccessList]"
      "AccessListMain" => "[winlog][event_data][AccessListMain]"
      "AccessMask" => "[winlog][event_data][AccessMask]"
      "AccessReason" => "[winlog][event_data][AccessReason]"
      "AccountName" => "[winlog][event_data][AccountName]"
      "AccountType" => "[winlog][event_data][AccountType]"
      "ActionName" => "[winlog][event_data][ActionName]"
      "AccountDomain" => "[winlog][event_data][AccountDomain]"
      "AppCorrelationID" => "[winlog][event_data][AppCorrelationID]"
      "AttributeLDAPDisplayName" => "[winlog][event_data][AttributeLDAPDisplayName]"
      "AttributeSyntaxOID" => "[winlog][event_data][AttributeSyntaxOID]"
      "AttributeValue" => "[winlog][event_data][AttributeValue]"
      "AlertDesc" => "[winlog][event_data][AlertDesc]"
      "AlgorithmName" => "[winlog][event_data][AlgorithmName]"
      "Application" => "[winlog][event_data][Application]"
      "CounterId" => "[winlog][event_data][CounterId]"
      "CounterSetGuid" => "[winlog][event_data][CounterSetGuid]"
      "ClientAddress" => "[winlog][event_data][ClientAddress]"
      "ClientName" => "[winlog][event_data][ClientName]"
      "ContextInfo" => "[winlog][event_data][ContextInfo]"
      "DestAddress" => "[winlog][event_data][DestAddress]"
      "DestPort" => "[winlog][event_data][DestPort]"
      "Direction" => "[winlog][event_data][Direction]"
      "Domain" => "[winlog][event_data][Domain]"
      "DSName" => "[winlog][event_data][DSName]"
      "DSType" => "[winlog][event_data][DSType]"
      "Error" => "[winlog][event_data][Error]"
      "ErrorCode" => "[winlog][event_data][ErrorCode]"
      "EnginePID" => "[winlog][event_data][EnginePID]"
      "EventCountTotal" => "[winlog][event_data][EventCountTotal]"
      "ElevatedToken" => "[winlog][event_data][ElevatedToken]"
      "FilterRTID" => "[winlog][event_data][FilterRTID]"
      "FailureReason" => "[winlog][event_data][FailureReason]"
      "GroupMembership" => "[winlog][event_data][GroupMembership]"
      "HandleId" => "[winlog][event_data][HandleId]"
      "InstanceId" => "[winlog][event_data][InstanceId]"
      "InstanceName" => "[winlog][event_data][InstanceName]"
      "KeyName" => "[winlog][event_data][KeyName]"
      "KeyType" => "[winlog][event_data][KeyType]"
      "LayerName" => "[winlog][event_data][LayerName]"
      "LogString" => "[winlog][event_data][LogString]"
      "LayerRTID" => "[winlog][event_data][LayerRTID]"
      "MandatoryLabel" => "[winlog][event_data][MandatoryLabel]"
      "NewUacValue" => "[winlog][event_data][NewUacValue]"
      "ObjectName" => "[winlog][event_data][ObjectName]"
      "ObjectServer" => "[winlog][event_data][ObjectServer]"
      "ObjectType" => "[winlog][event_data][ObjectType]"
      "ObjectClass" => "[winlog][event_data][ObjectClass]"
      "ObjectDN" => "[winlog][event_data][ObjectDN]"
      "ObjectGUID" => "[winlog][event_data][ObjectGUID]"
      "OpCorrelationID" => "[winlog][event_data][OpCorrelationID]"
      "OperationType" => "[winlog][event_data][OperationType]"
      "Operation" => "[winlog][event_data][Operation]"
      "OldTargetUserName" => "[winlog][event_data][OldTargetUserName]"
      "Protocol" => "[winlog][event_data][Protocol]"
      "PreAuthType" => "[winlog][event_data][PreAuthType]"
      "Payload" => "[winlog][event_data][Payload]"
      "PackageName" => "[winlog][event_data][PackageName]"
      "ParentProcessName" => "[winlog][event_data][ParentProcessName]"
      "RestrictedAdminMode" => "[winlog][event_data][RestrictedAdminMode]"
      "RelativeTargetName" => "[winlog][event_data][RelativeTargetName]"
      "ReturnCode" => "[winlog][event_data][ReturnCode]"
      "RemoteMachineID" => "[winlog][event_data][RemoteMachineID]"
      "RemoteUserID" => "[winlog][event_data][RemoteUserID]"
      "ShareLocalPath" => "[winlog][event_data][ShareLocalPath]"
      "ShareName" => "[winlog][event_data][ShareName]"
      "SubcategoryGuid" => "[winlog][event_data][SubcategoryGuid]"
      "SourceAddress" => "[winlog][event_data][SourceAddress]"
      "SourcePort" => "[winlog][event_data][SourcePort]"
      "ServiceSid" => "[winlog][event_data][ServiceSid]"
      "SubStatus" => "[winlog][event_data][SubStatus]"
      "Service" => "[winlog][event_data][Service]"
      "SessionName" => "[winlog][event_data][SessionName]"
      "TaskInstanceId" => "[winlog][event_data][TaskInstanceId]"
      "TicketEncryptionType" => "[winlog][event_data][TicketEncryptionType]"
      "TicketOptions" => "[winlog][event_data][TicketOptions]"
      "TargetLinkedLogonId" => "[winlog][event_data][TargetLinkedLogonId]"
      "TargetOutboundDomainName" => "[winlog][event_data][TargetOutboundDomainName]"
      "TargetOutboundUserName" => "[winlog][event_data][TargetOutboundUserName]"
      "TdoType" => "[winlog][event_data][TdoType]"
      "TdoDirection" => "[winlog][event_data][TdoDirection]"
      "TdoAttributes" => "[winlog][event_data][TdoAttributes]"
      "TargetSid" => "[winlog][event_data][TargetSid]"
      "TaskName" => "[winlog][event_data][TaskName]"
      "UserID" => "[winlog][event_data][UserID]"
      "UserContext" => "[winlog][event_data][UserContext]"
      "VolumeNameLength" => "[winlog][event_data][VolumeNameLength]"
      "VolumeGuid" => "[winlog][event_data][VolumeGuid]"
      "VirtualAccount" => "[winlog][event_data][VirtualAccount]"
      "VolumeName" => "[winlog][event_data][VolumeName]"
      "Workstation" => "[winlog][event_data][Workstation]"
      "WorkstationName" => "[winlog][event_data][WorkstationName]"
    }
  }
  # powershell
  mutate {
    rename => {"ConnectedUser" => "[winlog][event_data][ConnectedUser]"
      "CommandLine" => "[winlog][event_data][CommandLine]"
      "CommandPath" => "[winlog][event_data][CommandPath]"
      "CommandName" => "[winlog][event_data][CommandName]"
      "CommandType" => "[winlog][event_data][CommandType]"
      "DetailTotal" => "[winlog][event_data][DetailTotal]"
      "DetailSequence" => "[winlog][event_data][DetailSequence]"
      "EngineVersion" => "[winlog][event_data][EngineVersion]"
      "HostId" => "[winlog][event_data][HostId]"
      "HostApplication" => "[winlog][event_data][HostApplication]"
      "HostName" => "[winlog][event_data][HostName]"
      "HostVersion" => "[winlog][event_data][HostVersion]"
      "NewEngineState" => "[winlog][event_data][NewEngineState]"
      "NewProviderState" => "[winlog][event_data][NewProviderState]"
      "PreviousEngineState" => "[winlog][event_data][PreviousEngineState]"
      "ProviderName" => "[winlog][event_data][ProviderName]"
      "PipelineId" => "[winlog][event_data][PipelineId]"
      "RunspaceId" => "[winlog][event_data][RunspaceId]"
      "SequenceNumber" => "[winlog][event_data][SequenceNumber]"
      "ScriptName" => "[winlog][event_data][ScriptName]"
      "ShellID" => "[winlog][event_data][ShellID]"
      "ScriptBlockId" => "[winlog][event_data][ScriptBlockId]"
      "ScriptBlockText" => "[winlog][event_data][ScriptBlockText]"
      "User" => "[winlog][event_data][User]"
    }
  }
  # nxlog to ECS
}
output {if [@metadata][pipeline] {
    elasticsearch {pipeline => "%{[@metadata][pipeline]}"
      hosts => ["ES-IP:9200"]
      manage_template => false
      ilm_rollover_alias => "winlogbeat"
      ilm_pattern => "{now/M{YYYY.MM}}-000001"
      ilm_policy => "all-hot-50"
      user => "****"
      password => "*****"
      timeout => 300
    }
  }
  else {
    elasticsearch {
      pipeline => "winlogbeat-8.0.1-routing"
      hosts => ["ES-IP:9200"]
      manage_template => false
      ilm_rollover_alias => "winlogbeat"
      ilm_pattern => "{now/M{YYYY.MM}}-000001"
      ilm_policy => "all-hot-50"
      user => "****"
      password => "*****"
      timeout => 300
    }
  }
}

通过下面的解决后，大部分日志曾经能够跟 winlogbeat 采集的日志统一