大数据。基于hadoop的网站日志分析系统（附带web展示页面）

package com.zhiyou100.log.job; import java.sql.Connection; import java.sql.DriverManager; import java.sql.PreparedStatement; import java.sql.ResultSet; import java.sql.SQLException; import java.util.HashMap; import java.util.Map; import java.util.Set; public class HiveOperate { // 创建一个连接Hive的连接 public static Connection getConnection() { Connection conn = null; try { Class.forName("org.apache.hive.jdbc.HiveDriver"); conn = DriverManager.getConnection("jdbc:hive2://master:10000/zut_bigdata", "root", ""); } catch (Exception e) { // TODO Auto-generated catch block e.printStackTrace(); } return conn; } // 关闭连接 public static void close(Connection conn, ResultSet rs, PreparedStatement ps) { try { if (rs != null) { rs.close(); } if (ps != null) { ps.close(); } if (conn != null) { conn.close(); } } catch (Exception e) { // TODO Auto-generated catch block e.printStackTrace(); } } /** * 创建表 * * @throws Exception */ public static void createTable() throws Exception { // String sql = "create table table_0314_01(id int,name string) row format // delimited fields terminated by ','"; String sql = "create table log(ip string,date_1 string,url string) partitioned by (logDate string) row format delimited fields terminated by '\t'"; Connection con = getConnection(); // SQL语句预编译 PreparedStatement ps = con.prepareStatement(sql); ps.execute(); close(con, null, ps); } // 加载数据 public static void loadDataToHiveTable(String date) { System.out.println("开始加载"+date+"日志数据到Hive"); Connection con = null; PreparedStatement ps = null; try { String hql = "load data inpath '/zhangyanfeng/logs/output/" + date + "/part-r-00000' into table log partition(logDate=" + date + ")"; con = getConnection(); ps = con.prepareStatement(hql); ps.execute(); System.out.println(date+"日志加载成功"); } catch (SQLException e) { // TODO Auto-generated catch block e.printStackTrace(); } finally { close(con, null, ps); } } // 查询 public static Map<String, String> queryTable(String date) { Connection con = getConnection(); Map<String, String> map = new HashMap<>(); Map<String, String> result = new HashMap<String, String>(); String sql_pv = "select COUNT(1) from log where logDate=" + date; String sql_regNum = "select COUNT(1) from log where logDate=" + date + " and instr(url,'member.php?mod=reg')>0"; String sql_ip = "select COUNT(distinct ip) from log where logDate=" + date; String sql_jumper = "SELECT COUNT(1) FROM (SELECT COUNT(ip) AS ts FROM log where logDate=" + date + " GROUP BY ip HAVING ts=1) AS tmp"; map.put("PV", sql_pv); map.put("RegNum", sql_regNum); map.put("IP", sql_ip); map.put("Jumper", sql_jumper); // SQL语句预编译 PreparedStatement ps = null; ResultSet rs = null; Set<String> set = map.keySet(); for (String s : set) { try { ps = con.prepareStatement(map.get(s)); rs = ps.executeQuery(); while (rs.next()) { result.put(s, rs.getObject(1).toString()); } } catch (SQLException e) { // TODO Auto-generated catch block e.printStackTrace(); } } close(con, null, ps); result.put("logDate", new StringBuffer(date).insert(4, "-").insert(7, "-").toString()); return result; } public static void main(String[] args) throws Exception { createTable(); // loadDataToHiveTable(); // queryTable(); } }