ICode9
精准搜索请尝试： 精确搜索
首页
编程语言>
数据库
系统相关
互联网
其他分享
Python
Java
JavaScript
android
PHP
首页 > 其他分享> 文章详细

Apache Httpd的访问日志，主要是统计IP访问次数

2021-03-23 15:02:55  阅读：173  来源： 互联网

标签：Httpd return String get ip 访问 IP new con


新上线了一个自己的小服务

每天都有很多来扫描服务器的

很烦人

就想着吧这些IP地址都记下来

然后去读apache的access_log

太多了实在麻烦

写了一个小工具类。写小记一下吧

需要说一下

1 IPAddress.class中读取IP地址的归属地我找了4个API（有些是抓包来的：）

其中IP138返回的相对最好，不过需要付费

IP126方式暂时不受限制，以后不知道

2 hosts.deny是我自己弄的黑名单，主要是放在http配置中，用来将这些IP直接403干掉

3 Apache/2.4.6 (CentOS)，如果你的日志格式不一样就自己调整下函数吧

 

附代码：

HttpdLogTest.class

package com.sys.util;

import java.io.BufferedReader;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStreamReader;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Comparator;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Set;  
import net.sf.json.JSONObject;
 
/**
 * 用于分析Apache Httpd的访问日志，主要是统计IP访问次数。
 * 其中有白名单和黑名单的过滤功能。白名单在构造函数初始化时自己定义，黑名单是读了我的Apache hosts.deny文件
 * 如果读取到非名单中的新IP地址，调用了外部的IPAddress.get来获取IP的归属地
 * 提供了accessLog用于分析成功访问的日志，errorLog是错误日志，printlogByKeyword是按关键字查询日志
 * 
 * @author qing 2021-03-20
 *
 */
public class HttpdLogTest {
	//黑名单文件本地存储位置
	public static String hostsDenyPath = "D:\\hosts.deny";
	//访问日志本地路径
	public static String access_log_path = "C:\\Users\\Administrator\\AppData\\Roaming\\Private Shell\\Temp\\access_log";
	//错误日志本地路径
	public static String error_log_path = "C:\\Users\\Administrator\\AppData\\Roaming\\Private Shell\\Temp\\error_log";
	
	//是否调用查询IP地址
	public static boolean isfondIpAddress = true;
	
	//是否过滤意见被403的请求
	public static boolean filter403 = false;
	
	//白名单
	Set<String> whiteList = null;
	
	//黑名单
	Set<String> blacklist = null;
	
	
	public HttpdLogTest() {
		// 初始化时加载白名单和黑名单
		initWhiteList();
		initBlackList();
	}
	
	
	public static void main(String[] args) {
		HttpdLogTest httpLogTest = new HttpdLogTest();
		//System.out.println("===============accessLog===============");
		httpLogTest.accessLog();
		//System.out.println("===============accessLog end===============");
		//System.out.println("===============errorLog===============");
		//httpLogTest.errorLog();
		//System.out.println("===============errorLog end===============");
		//httpLogTest.printlogByKeyword(access_log_path,"1.1.1.1");
	}
	//加载白名单
	void initWhiteList(){
		whiteList = new HashSet<>();
		whiteList.add("::1");//内部IP
	}
	
	//读取本地的黑名单
    void initBlackList(){
		blacklist = new HashSet<>();
		
		try(FileInputStream fin = new FileInputStream(hostsDenyPath);
			InputStreamReader reader = new InputStreamReader(fin);
			BufferedReader buffReader = new BufferedReader(reader);) { 
			
			String strTmp = "";
			while ((strTmp = buffReader.readLine()) != null) {
				//本地黑名单hosts.Deny格式 8.8.8.8 -
				if(strTmp.endsWith("-")){
					blacklist.add(strTmp.replace("-", "").trim());
				}
			}
		} catch (Exception e) {
			e.printStackTrace();
		} 
    	
    }
    
	/**
	 * 获取IP地址之前判断是否需要联网获取
	 * @param ip
	 * @return
	 */
	String getIpAddress(String ip){
		if(isfondIpAddress){
			return IPAddress.get(ip);
		}
		return "false";
	} 
	

	//解析错误日志
	public void errorLog() {
		String splitChar = "\\[client ";// 分割关键字
		
		try (FileInputStream fin = new FileInputStream(error_log_path);
			InputStreamReader reader = new InputStreamReader(fin);
			BufferedReader buffReader = new BufferedReader(reader);){
			String strTmp = "";
			HashMap<String, Integer> res = new HashMap<String, Integer>();
			Integer idx = 1;
			//采用每次读一行的方式，因为日志文件可能比较大。一次读完会卡顿
			while ((strTmp = buffReader.readLine()) != null) {
				String arr[] = strTmp.split(splitChar);
				if (arr.length > 1) {
					String ip = arr[1].trim().substring(0,arr[1].trim().indexOf(":"));
					if (!res.containsKey(ip)) {
						res.put(ip, 1);
					} else {
						res.put(ip, res.get(ip) + 1);
					}
				}
				idx++;
			}
			System.out.println("读取完毕：" + idx + "行");
			sortMapAndPrint(res);

		} catch (IOException e) {
			e.printStackTrace();
		} 
	}
	
	
	//解析访问记录
	public void accessLog() {
		String splitChar = "- -";// 分割关键字
		try (FileInputStream fin = new FileInputStream(access_log_path);
				InputStreamReader reader = new InputStreamReader(fin);
				BufferedReader buffReader = new BufferedReader(reader);){

			String strTmp = null;
			HashMap<String, Integer> res = new HashMap<String, Integer>();
			Integer idx = 0;
			while ((strTmp = buffReader.readLine()) != null) {
				idx++;
				//过滤掉已经被拦截的403请求信息
				if(filter403 && strTmp.indexOf(" 403 ")>0){continue;}
				//只查看访问成功的
				//if(strTmp.indexOf(" 200 ")<0){continue;}
				
				String arr[] = strTmp.split(splitChar);
				if (arr.length > 0) {
					String ip = arr[0].trim();
					if (!res.containsKey(ip)) {
						res.put(ip, 1);
					} else {
						res.put(ip, res.get(ip) + 1);
					}
				}
			}

			System.out.println("读取完毕：" + idx);
			sortMapAndPrint(res);
			

		} catch (IOException e) {
			e.printStackTrace();
		}

	} 
	 
	//Map 按value倒序 并且打印
	void sortMapAndPrint(HashMap<String, Integer> map){
		List<Entry<String, Integer>> list = new ArrayList<Entry<String, Integer>>(map.entrySet());
		// Map按value倒序
		Collections.sort(list,new Comparator<Map.Entry<String, Integer>>() {
			public int compare(Map.Entry<String, Integer> o1,
					Map.Entry<String, Integer> o2) {
				return (o2.getValue() - o1.getValue());
			}
		});
		
		//安全访问的IP
		List<JSONObject> safeList = new ArrayList<JSONObject>();
		//新访问的IP
		List<JSONObject> newList = new ArrayList<JSONObject>();
		//已经被拦截的IP
		List<JSONObject> filterList = new ArrayList<JSONObject>();
		for (int i = 0; i < list.size(); i++) {
			String ip = list.get(i).getKey();
			Integer num = list.get(i).getValue();
			
			JSONObject obj = new JSONObject();
			obj.put("ip", ip);
			obj.put("num", num); 
			if(whiteList.contains(ip)){
				safeList.add(obj);
			}else if(blacklist.contains(ip)){
				filterList.add(obj);
			}else{
				newList.add(obj);
			}
			
		}
		//IP地址只有新发现的才去获取，这里没采用异步线程访问，可能会出现卡顿的现象，不过用来测试足够了
		for (int i = 0; i < newList.size(); i++) {
			JSONObject obj = newList.get(i);
			System.out.println("[NEW]\t" + obj.get("ip") + "\t" +obj.get("num")+"\t"+getIpAddress(obj.get("ip").toString()) );
		}
		for (int i = 0; i < safeList.size(); i++) {
			JSONObject obj = safeList.get(i);
			System.out.println("[安 全]\t" + obj.get("ip") + "\t" +obj.get("num") );
		}
		for (int i = 0; i < filterList.size(); i++) {
			JSONObject obj = filterList.get(i);
			System.out.println("[拦 截]\t" + obj.get("ip") + "\t" +obj.get("num") );
		}
		
	}
	
	/**
	 * 按关键字查询日志后打印日志
	 * @param ipaddress
	 */
	public void printlogByKeyword(String logPath,String keyword){
		//Java7的try-with-resources可以优雅关闭文件，异常时自动关闭文件；详细解读https://stackoverflow.com/a/12665271
		try(FileInputStream fin = new FileInputStream(logPath);
			InputStreamReader reader = new InputStreamReader(fin);
			BufferedReader buffReader = new BufferedReader(reader);) {
			String strTmp = "";
			Integer idx = 1;
			while ((strTmp = buffReader.readLine()) != null) {
				if(strTmp.indexOf(keyword)>=0){
					System.out.println((idx++) + " : "+strTmp);
				}
			}
			
			} catch (IOException e) {
				e.printStackTrace();
			}
	}
	


}

IPAddress.class

package com.sys.util;

import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.net.HttpURLConnection;
import java.net.URL;
import net.sf.json.JSONArray;
import net.sf.json.JSONObject;

public class IPAddress {
	
	//适用的IPapi类型
	/**
	 * IP138（付费）：https://api.ip138.com/ip/?ip=1.1.1.1
	 * IPIP(请求过多会错误)：http://freeapi.ipip.net/1.1.1.1
	 * IP126（敞开用）：http://ip.ws.126.net/ipquery?ip=1.1.1.1
	 * IP5ME(感觉不太可靠) https://q.ip5.me/?s=123.58.4.233
	 *
	 */
	public enum ApiType
	{
		IP138,IP126,IPIP,IP5ME;
	}
	
	public static void main(String[] args) {
		System.out.println(IPAddress.get("123.58.4.233",ApiType.IP126));
		System.out.println(IPAddress.get("123.58.4.233",ApiType.IPIP));
		System.out.println(IPAddress.get("123.58.4.233",ApiType.IP138));
		System.out.println(IPAddress.get("123.58.4.233",ApiType.IP5ME));
	}
	/**
	 * 读取IP地址的API，默认适用IP126接口
	 * @param ip 
	 * @return
	 */
	public static String get(String ip){
		return get(ip,ApiType.IP126);
	}
	
	/**
	 * 
	 * @param apiType
	 * @param ip
	 * @return
	 */
	public static String get(String ip,ApiType apiType){
		switch (apiType) {
			case IP126:return getIP126(ip);
			case IPIP: return getIPIP(ip);
			case IP138:return getIP138(ip);
			case IP5ME:return getIP5ME(ip);
		}
		return "NOT API";
	}
	
	/**
	 * https://q.ip5.me/?s=123.58.4.233
	 * 这个估计改版就不能用了
	 */
	 public static String getIP5ME(String ip) {		
		HttpURLConnection con = null;
		BufferedReader buffer = null;
		try {
			URL url = new URL("https://q.ip5.me/?s=" + ip);
			// 得到连接对象
			con = (HttpURLConnection) url.openConnection();
			// 设置请求类型
			con.setRequestMethod("GET");
			// 设置请求需要返回的数据类型和字符集类型
			con.setRequestProperty("content-type", "application/x-www-form-urlencoded; charset=gb2312");
			con.setRequestProperty("user-agent","Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/89.0.4389.90 Safari/537.36");			
			// 允许写出
			con.setDoOutput(true);
			// 允许读入
			con.setDoInput(true);
			// 不使用缓存
			con.setUseCaches(false);
			// 得到响应码
			int responseCode = con.getResponseCode();
			//System.out.println(responseCode);
			if (responseCode == HttpURLConnection.HTTP_OK) {
				// 得到响应流
				InputStream inputStream = con.getInputStream();
				String line;
				buffer = new BufferedReader(new InputStreamReader(inputStream,"gb2312"));
				String startKey = "<td><div id=\"ip_pos\" style=\"color:#FF0000\">";
				String endKey = "</div></td>";				
				while ((line = buffer.readLine()) != null) {
					if(line.startsWith(startKey)&&line.endsWith(endKey)){
						return line.substring(startKey.length(),line.indexOf(endKey));
					}
					//resultBuffer.append(line);
				}
			}

		} catch (Exception e) {
			e.printStackTrace();
		}
		return "ERROR";
	}
	
	/**
	 * http://ip.ws.126.net/ipquery?ip=1.1.1.1
	 * 
	 */
	 public static String getIP126(String ip) {
		
		HttpURLConnection con = null;
		BufferedReader buffer = null;
		StringBuffer resultBuffer = null;

		try {
			URL url = new URL("http://ip.ws.126.net/ipquery?ip=" + ip);
			// 得到连接对象
			con = (HttpURLConnection) url.openConnection();
			// 设置请求类型
			con.setRequestMethod("GET");
			// 设置请求需要返回的数据类型和字符集类型
			con.setRequestProperty("Host", "ip.ws.126.net");
			con.setRequestProperty("accept", "*/*");
			con.setRequestProperty("connection", "Keep-Alive");
			con.setRequestProperty(
					"user-agent",
					"Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/89.0.4389.90 Safari/537.36");
			con.setRequestProperty("Content-Type",
					"application/json;charset=utf-8");
			// 允许写出
			con.setDoOutput(true);
			// 允许读入
			con.setDoInput(true);
			// 不使用缓存
			con.setUseCaches(false);
			// 得到响应码
			int responseCode = con.getResponseCode();

			if (responseCode == HttpURLConnection.HTTP_OK) {
				// 得到响应流
				InputStream inputStream = con.getInputStream();
				// 将响应流转换成字符串
				resultBuffer = new StringBuffer();
				String line;
				buffer = new BufferedReader(new InputStreamReader(inputStream,"GBK"));
				while ((line = buffer.readLine()) != null) {
					resultBuffer.append(line);
				}
				if(resultBuffer.toString().indexOf("{")>=0){
					JSONObject addArr = JSONObject.fromObject(resultBuffer.toString().substring(resultBuffer.toString().indexOf("{")));
					return addArr.getString("province")+","+addArr.getString("city");
					
				}else{
					return "NOT";
				}
				

				
			}

		} catch (Exception e) {
			e.printStackTrace();
		}
		return "ERROR";
	}

	
	
	/**
	 * http://freeapi.ipip.net/1.1.1.1
	 * 
	 */
	public static String getIPIP(String ip) {
		
		HttpURLConnection con = null;
		BufferedReader buffer = null;
		StringBuffer resultBuffer = null;

		try {
			URL url = new URL("http://freeapi.ipip.net/" + ip);
			// 得到连接对象
			con = (HttpURLConnection) url.openConnection();
			// 设置请求类型
			con.setRequestMethod("GET");
			// 设置请求需要返回的数据类型和字符集类型
			con.setRequestProperty("Host", "freeapi.ipip.net");
			con.setRequestProperty("accept", "*/*");
			con.setRequestProperty("connection", "Keep-Alive");
			con.setRequestProperty(
					"user-agent",
					"Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/89.0.4389.90 Safari/537.36");
			con.setRequestProperty("Content-Type",
					"application/json;charset=utf-8");
			// 允许写出
			con.setDoOutput(true);
			// 允许读入
			con.setDoInput(true);
			// 不使用缓存
			con.setUseCaches(false);
			// 得到响应码
			int responseCode = con.getResponseCode();

			if (responseCode == HttpURLConnection.HTTP_OK) {
				// 得到响应流
				InputStream inputStream = con.getInputStream();
				// 将响应流转换成字符串
				resultBuffer = new StringBuffer();
				String line;
				buffer = new BufferedReader(new InputStreamReader(inputStream,
						"UTF-8"));
				while ((line = buffer.readLine()) != null) {
					resultBuffer.append(line);
				}

				JSONArray addArr = JSONArray
						.fromObject(resultBuffer.toString());
				String address = "";
				for (int i = 0; i < addArr.size(); i++) {
					address += addArr.get(i).toString();
				}

				return address;
			}

		} catch (Exception e) {
			e.printStackTrace();
		}
		return "ERROR";
	}
	
	/**
	 * https://api.ip138.com/ip/?ip=1.1.1.1
	 * 
	 */
	public static String getIP138(String ip) {
		 String urlString="https://api.ip138.com/ip/?ip="+ip+"&datatype=jsonp";
	    String token="去IP138申请token";
		try {
            URL url = new URL(urlString);
            HttpURLConnection conn = (HttpURLConnection) url.openConnection();
            conn.setConnectTimeout(5 * 1000);
            conn.setReadTimeout(5 * 1000);
            conn.setDoInput(true);
            conn.setDoOutput(true);
            conn.setUseCaches(false);
            conn.setInstanceFollowRedirects(false);
            conn.setRequestMethod("GET"); 
            conn.setRequestProperty("token",token);
            int responseCode = conn.getResponseCode();
            if (responseCode == 200) {
                StringBuilder builder = new StringBuilder();
                BufferedReader br = new BufferedReader(new InputStreamReader(conn.getInputStream(),"utf-8"));
                for (String s = br.readLine(); s != null; s = br.readLine()) {
                    builder.append(s);
                }
                br.close();
                
                return ((JSONArray)(JSONObject.fromObject(builder.toString()).get("data"))).join(",").replace("\"", "");
            }else{
            	return "error:"+responseCode;
            }
        } catch (IOException e) {
            e.printStackTrace();
        }
        return null;
	}

}

hosts.deny

# HTTP Server Blacklist
# qing

## 2021-03-22
193.112.94.225 -
123.207.151.66 -
195.3.147.56 -
202.164.138.92 -
124.74.40.6 -
202.164.138.92 -
123.160.221.52 -
223.104.96.188 -
107.161.50.66 -
172.104.242.173 -
197.232.1.182 -
194.183.10.237 -
192.241.226.191 -
185.36.81.52 -

输出效果：

 

标签：Httpd,return,String,get,ip,访问,IP,new,con
来源： https://blog.csdn.net/t18liqing/article/details/115122831

本站声明： 1. iCode9 技术分享网（下文简称本站）提供的所有内容，仅供技术学习、探讨和分享；
2. 关于本站的所有留言、评论、转载及引用，纯属内容发起人的个人观点，与本站观点和立场无关；
3. 关于本站的所有言论和文字，纯属内容发起人的个人观点，与本站观点和立场无关；
4. 本站文章均是网友提供，不完全保证技术分享内容的完整性、准确性、时效性、风险性和版权归属；如您发现该文章侵犯了您的权益，可联系我们第一时间进行删除；
5. 本站为非盈利性的个人网站，所有内容不会用来进行牟利，也不会利用任何形式的广告来间接获益，纯粹是为了广大技术爱好者提供技术内容和技术思想的分享性交流网站。

关于我们 | 联系我们 | 留言反馈

专注分享技术，共同学习，共同进步。侵权联系[81616952@qq.com]

Copyright (C)ICode9.com, All Rights Reserved.

ICode9版权所有