1. apache日志

27.60.18.21 - - [21/Jul/2020:10:10:10 +0530] "GET /api/v1.2/places/search/json?username=liusg&location=28.5359586,77.3677936&query=elasticsearch&explain=true&bridge=true HTTP/1.1" 200 3284
27.60.18.22 - - [22/Jul/2020:10:11:11 +0530] "POST /api/v1.0/places/search/json?username=liushangguo&location=28.5359586,77.3677936&query=docker&explain=true&bridge=true HTTP/1.1" 200 1452
27.60.18.23 - - [23/Jul/2020:12:12:12 +0530] "HEAD /api/v1.2/places/nearby/json?&refLocation=28.5359586,77.3677936&keyword=FINATM HTTP/1.1" 200 3283
27.60.18.24 - - [24/Jul/2020:13:13:13 +0530] "POST /api/v2.0/places/search/json?username=liu.sg&location=28.5359586,77.3677936&query=iphone&explain=true&bridge=true HTTP/1.1" 200 3415
27.60.18.25 - - [25/Jul/2020:16:16:16 +0530] "GET /api/v1.2/places/search/json?username=pradeep.pgu&location=28.5359586,77.3677936&query=15000227329&explain=true&bridge HTTP/1.1" 200 2476

2. logstash配置

input {file {path => "/var/log/aaa.log"stat_interval => 1start_position => "beginning"}
}filter {grok {match => { "message" => "%{IPORHOST:client_ip} %{HTTPDUSER:ident} %{HTTPDUSER:auth} \[%{HTTPDATE:req_time}\] \"(?:%{WORD:method} /api/v%{NUMBER:api_version}/.*/json\?%{NOTSPACE:req_data}(?: HTTP/%{NUMBER:http_version})?|%{DATA:rawrequest})\" %{NUMBER:resp_code} (?:%{NUMBER:req_byte}|-)"}}kv { source => "req_data"field_split => "&"}if [query] {mutate {add_field => { "search" => "%{query}" }}} else if [keyword] {mutate {add_field => { "search" => "%{keyword}" }}}if [refLocation] {mutate {rename => { "refLocation" => "location" }}}#  新增timestamp字段，将@timestamp时间增加8小时ruby { code => "event.set('timestamp', event.get('@timestamp').time.localtime + 8*60*60)" }ruby { code => "event.set('@timestamp',event.get('timestamp'))" }mutate{split=>["location",","]add_field => { "jing" => "%{[location][0]}" }add_field => { "wei" => "%{[location][1]}"	}}mutate {# 删除指定字段,messageremove_field => ["@version","host","path","tags"]# 如果将@timestamp字段删除，自动生成索引的日期配置就为空		# remove_field => ["@timestamp"]}
}output {stdout {}elasticsearch {hosts => ["http://192.168.1.58:9200"]index => "httpd-%{+YYYY.MM.dd}"user => "elastic"password => "123456"}
}

3. 追加日志

echo '20.10.20.22 - - [25/Aug/2010:20:20:20 +0530] "GET /api/v2.2/places/search/json?username=sdd.pgu&location=28.2222233,77.3333333&query=elasticssss&explain=true&bridge HTTP/1.1" 200 2223' >> /var/log/aaa.log

4. 索引查询结果

{"took" : 4,"timed_out" : false,"_shards" : {"total" : 1,"successful" : 1,"skipped" : 0,"failed" : 0},"hits" : {"total" : {"value" : 1,"relation" : "eq"},"max_score" : 1.0,"hits" : [{"_index" : "httpd-2020.07.23","_type" : "_doc","_id" : "S415enMB5upHvde6CV8J","_score" : 1.0,"_source" : {"auth" : "-","api_version" : "2.2","ident" : "-","method" : "GET","location" : ["28.2222233","77.3333333"],"@timestamp" : "2020-07-23T15:00:19.003Z","req_time" : "25/Aug/2010:20:20:20 +0530","username" : "sdd.pgu","query" : "elasticssss","message" : """20.10.20.22 - - [25/Aug/2010:20:20:20 +0530] "GET /api/v2.2/places/search/json?username=sdd.pgu&location=28.2222233,77.3333333&query=elasticssss&explain=true&bridge HTTP/1.1" 200 2223""","wei" : "77.3333333","explain" : "true","req_byte" : "2223","search" : "elasticssss","http_version" : "1.1","client_ip" : "20.10.20.22","timestamp" : "2020-07-23T15:00:19.003Z","resp_code" : "200","req_data" : "username=sdd.pgu&location=28.2222233,77.3333333&query=elasticssss&explain=true&bridge","jing" : "28.2222233"}}]}
}