一、用nxlog采集windows日志
########################################################################### 根底配置 ############################################################################# 64零碎define ROOT C:\Program Files (x86)\nxlog# 32零碎#define ROOT C:\Program Files\nxlogModuledir %ROOT%\modulesCacheDir %ROOT%\dataPidfile %ROOT%\data\nxlog.pidSpoolDir %ROOT%\dataLogFile %ROOT%\data\nxlog.log# 开启GELF格局扩大，并定义最大日志长度<Extension gelf> Module xm_gelf ShortMessageLength 65536</Extension># 开启JSON扩大 <Extension json> Module xm_json</Extension># 开启主动转码<Extension _charconv> Module xm_charconv AutodetectCharsets iso8859-2, utf-8, utf-16, utf-32</Extension>########################################################################### 输出配置-windows日志 ############################################################################# 因为NXLOG社区版本最大只能发送256个Channel，而目前windows2016，曾经超过300个,因而局部日志采集不到,须要手动查问通道，来避免单次查问超过256<Input APP_Logs># WIN7 以上 Module im_msvistalog# WINDOWS 2003# Module im_mseventlog # 查问Application通道 Query <QueryList><Query Id="0"><Select Path="Application">*</Select></Query></QueryList> # 过滤所有类型为具体的事件类型 Exec if $EventType == 'VERBOSE' drop(); Exec $Hostname = hostname();</Input><Input SYS_Logs># WIN7 以上 Module im_msvistalog# WINDOWS 2003# Module im_mseventlog # 查问System通道 Query <QueryList><Query Id="0"><Select Path="System">*</Select></Query></QueryList> # 过滤所有类型为具体的事件类型 Exec if $EventType == 'VERBOSE' drop(); Exec $Hostname = hostname();</Input><Input SEC_Logs># WIN7 以上 Module im_msvistalog# WINDOWS 2003# Module im_mseventlog # 查问Security通道 Query <QueryList><Query Id="0"><Select Path="Security">*</Select></Query></QueryList> # 过滤所有类型为具体的事件类型 Exec if $EventType == 'VERBOSE' drop(); Exec $Hostname = hostname();</Input>########################################################################### 输入配置 ############################################################################<Output Logstash> Module om_udp Host logstash-ip Port 5414 OutputType GELF</Output><Route APP> Path APP_Logs => Logstash</Route><Route SYS> Path SYS_Logs => Logstash</Route><Route SYS> Path SEC_Logs => Logstash</Route>二、用logstash，将windows日志进一步整顿
备注：二和三能够合并在一起，而无需kafka
input { gelf { use_udp => "true" codec => json_lines { charset => CP1252 } port => "5414" id => "winlog" }}# 这里过滤规定跟 winlogbeat 保持一致filter { if [Channel] == "Security" or [Channel] == "Application" or [Channel] == "System" { } else if [Channel] == "Windows PowerShell" { if [EventID] != 400 and [EventID] != 403 and [EventID] != 600 and [EventID] != 800 { drop {} } } else if [Channel] == "Microsoft-Windows-PowerShell/Operational" { if [EventID] != 4103 and [EventID] != 4104 and [EventID] != 4105 and [EventID] != 4106 { drop {} } } else { drop {} }}output { kafka { bootstrap_servers => "kafka-ip" topic_id => "winlog" codec => "json" }}三、将windows gelf日志格局换成为ecs规范格局
input { kafka { bootstrap_servers => "kafka-ip" client_id => "winlog" group_id => "logstash-es-winlog" auto_offset_reset => "latest" consumer_threads => 1 decorate_events => "true" topics => ["winlog"] codec => "json" }}filter { if [Channel] == "Windows PowerShell" { if [EventID] == "800" { mutate { add_field => { "[winlog][event_data][param2]" => "%{message}" } } kv { source => "[winlog][event_data][param2]" target => "[winlog][event_data]" field_split => "\n\t" trim_key => "\n\t" trim_value => "\n\t" value_split => "=" } } else { mutate { add_field => { "[winlog][event_data][param3]" => "%{message}" } } kv { source => "[winlog][event_data][param3]" target => "[winlog][event_data]" field_split => "\n\t" trim_key => "\n\t" trim_value => "\n\t" value_split => "=" } } } # ECS fields mutate { rename => { "host" => "source_name" "SeverityValue" => "[event][severity]" "EventReceivedTime" => "[event][time][received]" "SourceModuleType" => "[nxlog][module][type]" "SourceModuleName" => "[nxlog][module][name]" "Severity" => "[winlog][level]" "ThreadID" => "[winlog][process][thread][id]" "ProcessID" => "[winlog][process][id]" } add_field => { "[agent][name]" => "%{source_name}" "[agent][version]" => "2.10.2150" "[host][ip]" => "%{source_host}" "[host][name]" => "%{source_name}" "[host][os][platform]" => "windows" "[host][os][type]" => "windows" "[event][code]" => "%{EventID}" } } # 非凡字段,占用了默认开展字段 mutate { rename => { "url" => "[winlog][event_data][url]" "bytesTransferred" => "[winlog][event_data][bytesTransferred]" "fileLength" => "[winlog][event_data][fileLength]" "bytesTotal" => "[winlog][event_data][bytesTotal]" "error" => "[winlog][event_data][error]" "destination" => "[winlog][event_data][destination]" "bytesTransferredFromPeer" => "[winlog][event_data][bytesTransferredFromPeer]" "source" => "[winlog][event_data][source]" } } # winlog mutate { add_field => { "[winlog][api]" => "wineventlog" } rename => { "ActivityID" => "[winlog][activity_id]" "EventID" => "[winlog][event_id]" "EventType" => "[winlog][keywords]" "Channel" => "[winlog][channel]" "RecordNumber" => "[winlog][record_id]" "Opcode" => "[winlog][opcode]" "ProviderGuid" => "[winlog][provider_guid]" "SourceName" => "[winlog][provider_name]" "Category" => "[winlog][task]" "Version" => "[winlog][version]" } } # event_data mutate { rename => { "AuthenticationPackageName" => "[winlog][event_data][AuthenticationPackageName]" "Binary" => "[winlog][event_data][Binary]" "BitlockerUserInputTime" => "[winlog][event_data][BitlockerUserInputTime]" "BootMode" => "[winlog][event_data][BootMode]" "BootType" => "[winlog][event_data][BootType]" "BuildVersion" => "[winlog][event_data][BuildVersion]" "Company" => "[winlog][event_data][Company]" "CorruptionActionState" => "[winlog][event_data][CorruptionActionState]" "CreationUtcTime" => "[winlog][event_data][CreationUtcTime]" "Description" => "[winlog][event_data][Description]" "Detail" => "[winlog][event_data][Detail]" "DeviceName" => "[winlog][event_data][DeviceName]" "DeviceNameLength" => "[winlog][event_data][DeviceNameLength]" "DeviceTime" => "[winlog][event_data][DeviceTime]" "DeviceVersionMajor" => "[winlog][event_data][DeviceVersionMajor]" "DeviceVersionMinor" => "[winlog][event_data][DeviceVersionMinor]" "DriveName" => "[winlog][event_data][DriveName]" "DriverName" => "[winlog][event_data][DriverName]" "DriverNameLength" => "[winlog][event_data][DriverNameLength]" "DwordVal" => "[winlog][event_data][DwordVal]" "EntryCount" => "[winlog][event_data][EntryCount]" "ExtraInfo" => "[winlog][event_data][ExtraInfo]" "FailureName" => "[winlog][event_data][FailureName]" "FailureNameLength" => "[winlog][event_data][FailureNameLength]" "FileVersion" => "[winlog][event_data][FileVersion]" "FinalStatus" => "[winlog][event_data][FinalStatus]" "Group" => "[winlog][event_data][Group]" "IdleImplementation" => "[winlog][event_data][IdleImplementation]" "IdleStateCount" => "[winlog][event_data][IdleStateCount]" "ImpersonationLevel" => "[winlog][event_data][ImpersonationLevel]" "IntegrityLevel" => "[winlog][event_data][IntegrityLevel]" "IpAddress" => "[winlog][event_data][IpAddress]" "IpPort" => "[winlog][event_data][IpPort]" "KeyLength" => "[winlog][event_data][KeyLength]" "LastBootGood" => "[winlog][event_data][LastBootGood]" "LastShutdownGood" => "[winlog][event_data][LastShutdownGood]" "LmPackageName" => "[winlog][event_data][LmPackageName]" "LogonGuid" => "[winlog][event_data][LogonGuid]" "LogonId" => "[winlog][event_data][LogonId]" "LogonProcessName" => "[winlog][event_data][LogonProcessName]" "LogonType" => "[winlog][event_data][LogonType]" "MajorVersion" => "[winlog][event_data][MajorVersion]" "MaximumPerformancePercent" => "[winlog][event_data][MaximumPerformancePercent]" "MemberName" => "[winlog][event_data][MemberName]" "MemberSid" => "[winlog][event_data][MemberSid]" "MinimumPerformancePercent" => "[winlog][event_data][MinimumPerformancePercent]" "MinimumThrottlePercent" => "[winlog][event_data][MinimumThrottlePercent]" "MinorVersion" => "[winlog][event_data][MinorVersion]" "NewProcessId" => "[winlog][event_data][NewProcessId]" "NewProcessName" => "[winlog][event_data][NewProcessName]" "NewSchemeGuid" => "[winlog][event_data][NewSchemeGuid]" "NewTime" => "[winlog][event_data][NewTime]" "NominalFrequency" => "[winlog][event_data][NominalFrequency]" "Number" => "[winlog][event_data][Number]" "OldSchemeGuid" => "[winlog][event_data][OldSchemeGuid]" "OldTime" => "[winlog][event_data][OldTime]" "OriginalFileName" => "[winlog][event_data][OriginalFileName]" "Path" => "[winlog][event_data][Path]" "PerformanceImplementation" => "[winlog][event_data][PerformanceImplementation]" "PreviousCreationUtcTime" => "[winlog][event_data][PreviousCreationUtcTime]" "PreviousTime" => "[winlog][event_data][PreviousTime]" "PrivilegeList" => "[winlog][event_data][PrivilegeList]" "ProcessId" => "[winlog][event_data][ProcessId]" "ProcessName" => "[winlog][event_data][ProcessName]" "ProcessPath" => "[winlog][event_data][ProcessPath]" "ProcessPid" => "[winlog][event_data][ProcessPid]" "Product" => "[winlog][event_data][Product]" "PuaCount" => "[winlog][event_data][PuaCount]" "PuaPolicyId" => "[winlog][event_data][PuaPolicyId]" "QfeVersion" => "[winlog][event_data][QfeVersion]" "Reason" => "[winlog][event_data][Reason]" "SchemaVersion" => "[winlog][event_data][SchemaVersion]" "ServiceName" => "[winlog][event_data][ServiceName]" "ServiceVersion" => "[winlog][event_data][ServiceVersion]" "ShutdownActionType" => "[winlog][event_data][ShutdownActionType]" "ShutdownEventCode" => "[winlog][event_data][ShutdownEventCode]" "ShutdownReason" => "[winlog][event_data][ShutdownReason]" "Signature" => "[winlog][event_data][Signature]" "SignatureStatus" => "[winlog][event_data][SignatureStatus]" "Signed" => "[winlog][event_data][Signed]" "StartTime" => "[winlog][event_data][StartTime]" "State" => "[winlog][event_data][State]" "Status" => "[winlog][event_data][Status]" "StopTime" => "[winlog][event_data][StopTime]" "SubjectDomainName" => "[winlog][event_data][SubjectDomainName]" "SubjectLogonId" => "[winlog][event_data][SubjectLogonId]" "SubjectUserName" => "[winlog][event_data][SubjectUserName]" "SubjectUserSid" => "[winlog][event_data][SubjectUserSid]" "TSId" => "[winlog][event_data][TSId]" "TargetDomainName" => "[winlog][event_data][TargetDomainName]" "TargetInfo" => "[winlog][event_data][TargetInfo]" "TargetLogonGuid" => "[winlog][event_data][TargetLogonGuid]" "TargetLogonId" => "[winlog][event_data][TargetLogonId]" "TargetServerName" => "[winlog][event_data][TargetServerName]" "TargetUserName" => "[winlog][event_data][TargetUserName]" "TargetUserSid" => "[winlog][event_data][TargetUserSid]" "TerminalSessionId" => "[winlog][event_data][TerminalSessionId]" "TokenElevationType" => "[winlog][event_data][TokenElevationType]" "TransmittedServices" => "[winlog][event_data][TransmittedServices]" "UserSid" => "[winlog][event_data][UserSid]" "Version" => "[winlog][event_data][Version]" "param1" => "[winlog][event_data][param1]" "param2" => "[winlog][event_data][param2]" "param3" => "[winlog][event_data][param3]" "param4" => "[winlog][event_data][param4]" "param5" => "[winlog][event_data][param5]" "param6" => "[winlog][event_data][param6]" "param7" => "[winlog][event_data][param7]" "param8" => "[winlog][event_data][param8]" } } # event_data 查漏补缺 mutate { rename => { "AccessList" => "[winlog][event_data][AccessList]" "AccessListMain" => "[winlog][event_data][AccessListMain]" "AccessMask" => "[winlog][event_data][AccessMask]" "AccessReason" => "[winlog][event_data][AccessReason]" "AccountName" => "[winlog][event_data][AccountName]" "AccountType" => "[winlog][event_data][AccountType]" "ActionName" => "[winlog][event_data][ActionName]" "AccountDomain" => "[winlog][event_data][AccountDomain]" "AppCorrelationID" => "[winlog][event_data][AppCorrelationID]" "AttributeLDAPDisplayName" => "[winlog][event_data][AttributeLDAPDisplayName]" "AttributeSyntaxOID" => "[winlog][event_data][AttributeSyntaxOID]" "AttributeValue" => "[winlog][event_data][AttributeValue]" "AlertDesc" => "[winlog][event_data][AlertDesc]" "AlgorithmName" => "[winlog][event_data][AlgorithmName]" "Application" => "[winlog][event_data][Application]" "CounterId" => "[winlog][event_data][CounterId]" "CounterSetGuid" => "[winlog][event_data][CounterSetGuid]" "ClientAddress" => "[winlog][event_data][ClientAddress]" "ClientName" => "[winlog][event_data][ClientName]" "ContextInfo" => "[winlog][event_data][ContextInfo]" "DestAddress" => "[winlog][event_data][DestAddress]" "DestPort" => "[winlog][event_data][DestPort]" "Direction" => "[winlog][event_data][Direction]" "Domain" => "[winlog][event_data][Domain]" "DSName" => "[winlog][event_data][DSName]" "DSType" => "[winlog][event_data][DSType]" "Error" => "[winlog][event_data][Error]" "ErrorCode" => "[winlog][event_data][ErrorCode]" "EnginePID" => "[winlog][event_data][EnginePID]" "EventCountTotal" => "[winlog][event_data][EventCountTotal]" "ElevatedToken" => "[winlog][event_data][ElevatedToken]" "FilterRTID" => "[winlog][event_data][FilterRTID]" "FailureReason" => "[winlog][event_data][FailureReason]" "GroupMembership" => "[winlog][event_data][GroupMembership]" "HandleId" => "[winlog][event_data][HandleId]" "InstanceId" => "[winlog][event_data][InstanceId]" "InstanceName" => "[winlog][event_data][InstanceName]" "KeyName" => "[winlog][event_data][KeyName]" "KeyType" => "[winlog][event_data][KeyType]" "LayerName" => "[winlog][event_data][LayerName]" "LogString" => "[winlog][event_data][LogString]" "LayerRTID" => "[winlog][event_data][LayerRTID]" "MandatoryLabel" => "[winlog][event_data][MandatoryLabel]" "NewUacValue" => "[winlog][event_data][NewUacValue]" "ObjectName" => "[winlog][event_data][ObjectName]" "ObjectServer" => "[winlog][event_data][ObjectServer]" "ObjectType" => "[winlog][event_data][ObjectType]" "ObjectClass" => "[winlog][event_data][ObjectClass]" "ObjectDN" => "[winlog][event_data][ObjectDN]" "ObjectGUID" => "[winlog][event_data][ObjectGUID]" "OpCorrelationID" => "[winlog][event_data][OpCorrelationID]" "OperationType" => "[winlog][event_data][OperationType]" "Operation" => "[winlog][event_data][Operation]" "OldTargetUserName" => "[winlog][event_data][OldTargetUserName]" "Protocol" => "[winlog][event_data][Protocol]" "PreAuthType" => "[winlog][event_data][PreAuthType]" "Payload" => "[winlog][event_data][Payload]" "PackageName" => "[winlog][event_data][PackageName]" "ParentProcessName" => "[winlog][event_data][ParentProcessName]" "RestrictedAdminMode" => "[winlog][event_data][RestrictedAdminMode]" "RelativeTargetName" => "[winlog][event_data][RelativeTargetName]" "ReturnCode" => "[winlog][event_data][ReturnCode]" "RemoteMachineID" => "[winlog][event_data][RemoteMachineID]" "RemoteUserID" => "[winlog][event_data][RemoteUserID]" "ShareLocalPath" => "[winlog][event_data][ShareLocalPath]" "ShareName" => "[winlog][event_data][ShareName]" "SubcategoryGuid" => "[winlog][event_data][SubcategoryGuid]" "SourceAddress" => "[winlog][event_data][SourceAddress]" "SourcePort" => "[winlog][event_data][SourcePort]" "ServiceSid" => "[winlog][event_data][ServiceSid]" "SubStatus" => "[winlog][event_data][SubStatus]" "Service" => "[winlog][event_data][Service]" "SessionName" => "[winlog][event_data][SessionName]" "TaskInstanceId" => "[winlog][event_data][TaskInstanceId]" "TicketEncryptionType" => "[winlog][event_data][TicketEncryptionType]" "TicketOptions" => "[winlog][event_data][TicketOptions]" "TargetLinkedLogonId" => "[winlog][event_data][TargetLinkedLogonId]" "TargetOutboundDomainName" => "[winlog][event_data][TargetOutboundDomainName]" "TargetOutboundUserName" => "[winlog][event_data][TargetOutboundUserName]" "TdoType" => "[winlog][event_data][TdoType]" "TdoDirection" => "[winlog][event_data][TdoDirection]" "TdoAttributes" => "[winlog][event_data][TdoAttributes]" "TargetSid" => "[winlog][event_data][TargetSid]" "TaskName" => "[winlog][event_data][TaskName]" "UserID" => "[winlog][event_data][UserID]" "UserContext" => "[winlog][event_data][UserContext]" "VolumeNameLength" => "[winlog][event_data][VolumeNameLength]" "VolumeGuid" => "[winlog][event_data][VolumeGuid]" "VirtualAccount" => "[winlog][event_data][VirtualAccount]" "VolumeName" => "[winlog][event_data][VolumeName]" "Workstation" => "[winlog][event_data][Workstation]" "WorkstationName" => "[winlog][event_data][WorkstationName]" } } # powershell mutate { rename => { "ConnectedUser" => "[winlog][event_data][ConnectedUser]" "CommandLine" => "[winlog][event_data][CommandLine]" "CommandPath" => "[winlog][event_data][CommandPath]" "CommandName" => "[winlog][event_data][CommandName]" "CommandType" => "[winlog][event_data][CommandType]" "DetailTotal" => "[winlog][event_data][DetailTotal]" "DetailSequence" => "[winlog][event_data][DetailSequence]" "EngineVersion" => "[winlog][event_data][EngineVersion]" "HostId" => "[winlog][event_data][HostId]" "HostApplication" => "[winlog][event_data][HostApplication]" "HostName" => "[winlog][event_data][HostName]" "HostVersion" => "[winlog][event_data][HostVersion]" "NewEngineState" => "[winlog][event_data][NewEngineState]" "NewProviderState" => "[winlog][event_data][NewProviderState]" "PreviousEngineState" => "[winlog][event_data][PreviousEngineState]" "ProviderName" => "[winlog][event_data][ProviderName]" "PipelineId" => "[winlog][event_data][PipelineId]" "RunspaceId" => "[winlog][event_data][RunspaceId]" "SequenceNumber" => "[winlog][event_data][SequenceNumber]" "ScriptName" => "[winlog][event_data][ScriptName]" "ShellID" => "[winlog][event_data][ShellID]" "ScriptBlockId" => "[winlog][event_data][ScriptBlockId]" "ScriptBlockText" => "[winlog][event_data][ScriptBlockText]" "User" => "[winlog][event_data][User]" } } # nxlog to ECS}output { if [@metadata][pipeline] { elasticsearch { pipeline => "%{[@metadata][pipeline]}" hosts => ["ES-IP:9200"] manage_template => false ilm_rollover_alias => "winlogbeat" ilm_pattern => "{now/M{YYYY.MM}}-000001" ilm_policy => "all-hot-50" user => "****" password => "*****" timeout => 300 } } else { elasticsearch { pipeline => "winlogbeat-8.0.1-routing" hosts => ["ES-IP:9200"] manage_template => false ilm_rollover_alias => "winlogbeat" ilm_pattern => "{now/M{YYYY.MM}}-000001" ilm_policy => "all-hot-50" user => "****" password => "*****" timeout => 300 } }}通过下面的解决后，大部分日志曾经能够跟 winlogbeat采集的日志统一