grok支持下述的过滤配置选项

下面是grok基本表达式映射

下面我来看看grok使用的例子，我们来匹配下nginx的日志

113.57.176.61 - - [07/Nov/2019:10:06:08 +0800] "POST /zabbix.php?sid=f7e1f4f8d197181a&action=widget.problems.view HTTP/1.1" 200 5385 "http://49.233.176.23:8888/zabbix.php?action=dashboard.view" "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:70.0) Gecko/20100101 Firefox/70.0" "-"

可以通过下面的grok正则匹配

%{IP:client} (%{WORD:ident}|-) (%{USERNAME:auth}|-) \[%{HTTPDATE:timestamp}\] \"(%{WORD:method} %{NOTSPACE:request} HTTP/%{NUMBER:httpversion})\" %{NUMBER:response_code} (?:%{NUMBER:response_time}|-) "(?:%{URI:referrer}|-)" (?:%{QS:user_agent}|-) \"(%{WORD:x_forword_for}|-)\"

匹配的结果是

{
  "client": [
    [
      "113.57.176.61"
    ]
  ],
  "IPV6": [
    [
      null,
      null
    ]
  ],
  "IPV4": [
    [
      "113.57.176.61",
      null
    ]
  ],
  "ident": [
    [
      null
    ]
  ],
  "auth": [
    [
      "-"
    ]
  ],
  "timestamp": [
    [
      "07/Nov/2019:10:06:08 +0800"
    ]
  ],
  "MONTHDAY": [
    [
      "07"
    ]
  ],
  "MONTH": [
    [
      "Nov"
    ]
  ],
  "YEAR": [
    [
      "2019"
    ]
  ],
  "TIME": [
    [
      "10:06:08"
    ]
  ],
  "HOUR": [
    [
      "10"
    ]
  ],
  "MINUTE": [
    [
      "06"
    ]
  ],
  "SECOND": [
    [
      "08"
    ]
  ],
  "INT": [
    [
      "+0800"
    ]
  ],
  "method": [
    [
      "POST"
    ]
  ],
  "request": [
    [
      "/zabbix.php?sid=f7e1f4f8d197181a&action=widget.problems.view"
    ]
  ],
  "httpversion": [
    [
      "1.1"
    ]
  ],
  "BASE10NUM": [
    [
      "1.1",
      "200",
      "5385"
    ]
  ],
  "response_code": [
    [
      "200"
    ]
  ],
  "response_time": [
    [
      "5385"
    ]
  ],
  "referrer": [
    [
      "http://49.233.176.23:8888/zabbix.php?action=dashboard.view"
    ]
  ],
  "URIPROTO": [
    [
      "http"
    ]
  ],
  "USER": [
    [
      null
    ]
  ],
  "USERNAME": [
    [
      null
    ]
  ],
  "URIHOST": [
    [
      "49.233.176.23:8888"
    ]
  ],
  "IPORHOST": [
    [
      "49.233.176.23"
    ]
  ],
  "HOSTNAME": [
    [
      "49.233.176.23"
    ]
  ],
  "IP": [
    [
      null
    ]
  ],
  "port": [
    [
      "8888"
    ]
  ],
  "URIPATHPARAM": [
    [
      "/zabbix.php?action=dashboard.view"
    ]
  ],
  "URIPATH": [
    [
      "/zabbix.php"
    ]
  ],
  "URIPARAM": [
    [
      "?action=dashboard.view"
    ]
  ],
  "user_agent": [
    [
      ""Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:70.0) Gecko/20100101 Firefox/70.0""
    ]
  ],
  "QUOTEDSTRING": [
    [
      ""Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:70.0) Gecko/20100101 Firefox/70.0""
    ]
  ],
  "x_forword_for": [
    [
      null
    ]
  ]
}

我们可以通过一个grok在线的调试进行调试，地址是http://grokdebug.herokuapp.com/，其次官方已经有一些正则匹配模板，我们可以在这里获取https://github.com/logstash-plugins/logstash-patterns-core/tree/master/patterns

上面一个例子已经匹配出来很多字段，其实使用匹配是可以让我们有更多的字段信息可以使用，我们来看看grok的过滤方法

input {
  beats {
    port => 5044
    codec => plain {
          charset => "UTF-8"
    }
  }
}

filter {
  grok {
    match => ["message", '%{IP:client} (%{WORD:ident}|-) (%{USERNAME:auth}|-) \[%{HTTPDATE:timestamp}\] \"(%{WORD:method} %{NOTSPACE:request} HTTP/%{NUMBER:httpversion})\" %{NUMBER:response_code} (?:%{NUMBER:response_time}|-) "(?:%{URI:referrer}|-)" (?:%{QS:user_agent}|-) \"(%{WORD:x_forword_for}|-)\"']
  }
  if "beats_input_codec_plain_applied" in [tags] {
    mutate {
      remove_tag => ["beats_input_codec_plain_applied"]
    }
  }
}


output {
  elasticsearch {
    hosts => "172.26.61.61:9200"
    manage_template => false
    index => "%{[@metadata][beat]}-%{+YYYY.MM.dd}"
    #document_type => "%{[@metadata][type]}"
  }
}

通过上述修改，再登陆kibana查看发现已经多了刚才匹配出来的字段

{
  "_index": "filebeat-2019.10.28",
  "_type": "_doc",
  "_id": "WtuCEW4BW_ylP464mifs",
  "_version": 1,
  "_score": null,
  "_source": {
    "method": "POST",
    "Client": "172.26.61.49",
    "agent": {
      "type": "filebeat",
      "id": "9b6a51a8-a6ed-413a-8f7e-2fe7a172b450",
      "version": "7.4.0",
      "ephemeral_id": "35d9e2f2-8e8c-4408-a034-433e310a3018",
      "hostname": "localhost.localdomain"
    },
    "request": "/zabbix.php?sid=37ae6650e649a1f2&action=widget.problems.view",
    "@version": "1",
    "ecs": {
      "version": "1.1.0"
    },
    "input": {
      "type": "log"
    },
    "@timestamp": "2019-10-28T08:36:41.241Z",
    "log": {
      "offset": 1146484,
      "file": {
        "path": "/usr/local/nginx/logs/access.log"
      }
    },
    "response": "200",
    "httpversion": "1.1",
    "tags": [
      "nginx-access"
    ],
    "referrer": "\"http://172.26.61.58/zabbix.php?action=dashboard.view\"",
    "timestamp": "28/Oct/2019:16:36:39 +0800",
    "bytes": "629",
    "host": {
      "id": "7017e1f0128944ceaedb6189b101351c",
      "name": "localhost.localdomain",
      "os": {
        "family": "redhat",
        "platform": "centos",
        "name": "CentOS Linux",
        "version": "7 (Core)",
        "kernel": "3.10.0-1062.1.2.el7.x86_64",
        "codename": "Core"
      },
      "containerized": false,
      "architecture": "x86_64",
      "hostname": "localhost.localdomain"
    },
    "message": "172.26.61.49 - - [28/Oct/2019:16:36:39 +0800] \"POST /zabbix.php?sid=37ae6650e649a1f2&action=widget.problems.view HTTP/1.1\" 200 629 \"http://172.26.61.58/zabbix.php?action=dashboard.view\" \"Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:70.0) Gecko/20100101 Firefox/70.0\""
  },
  "fields": {
    "@timestamp": [
      "2019-10-28T08:36:41.241Z"
    ]
  },
  "sort": [
    1572251801241
  ]
}