From e4c0742e2d39d3b32dce3568aed235d4910ea52b Mon Sep 17 00:00:00 2001 From: Benjamin Erb Date: Tue, 21 Dec 2010 18:40:45 +0100 Subject: [PATCH 1/2] added access logging using the "Combined Log Format" --- .../ioexception/www/server/AccessLogger.java | 12 +++ src/de/ioexception/www/server/HttpServer.java | 7 ++ src/de/ioexception/www/server/HttpWorker.java | 2 + .../www/server/impl/BasicAccessLogger.java | 101 ++++++++++++++++++ .../www/server/impl/BasicHttpServer.java | 13 +++ 5 files changed, 135 insertions(+) create mode 100644 src/de/ioexception/www/server/AccessLogger.java create mode 100644 src/de/ioexception/www/server/impl/BasicAccessLogger.java diff --git a/src/de/ioexception/www/server/AccessLogger.java b/src/de/ioexception/www/server/AccessLogger.java new file mode 100644 index 0000000..3ae0af8 --- /dev/null +++ b/src/de/ioexception/www/server/AccessLogger.java @@ -0,0 +1,12 @@ +package de.ioexception.www.server; + +import java.util.concurrent.Callable; + +import de.ioexception.www.http.HttpRequest; +import de.ioexception.www.http.HttpResponse; + +public interface AccessLogger extends Callable +{ + public void log(String clientHost, HttpRequest request, HttpResponse response); + public void log(String s); +} diff --git a/src/de/ioexception/www/server/HttpServer.java b/src/de/ioexception/www/server/HttpServer.java index 55e1fda..c801c63 100644 --- a/src/de/ioexception/www/server/HttpServer.java +++ b/src/de/ioexception/www/server/HttpServer.java @@ -33,5 +33,12 @@ public interface HttpServer * @return */ public String getServerSignature(); + + /** + * Returns the signature of the webserver. + * + * @return + */ + public AccessLogger getAccessLogger(); } diff --git a/src/de/ioexception/www/server/HttpWorker.java b/src/de/ioexception/www/server/HttpWorker.java index bbc7f2c..e779c0e 100644 --- a/src/de/ioexception/www/server/HttpWorker.java +++ b/src/de/ioexception/www/server/HttpWorker.java @@ -58,6 +58,8 @@ public Void call() throws Exception sendResponse(response, socket.getOutputStream()); socket.close(); } + //Log + server.getAccessLogger().log(socket.getInetAddress().getHostAddress(), request, response); // We do not return anything here. return null; diff --git a/src/de/ioexception/www/server/impl/BasicAccessLogger.java b/src/de/ioexception/www/server/impl/BasicAccessLogger.java new file mode 100644 index 0000000..fa75b38 --- /dev/null +++ b/src/de/ioexception/www/server/impl/BasicAccessLogger.java @@ -0,0 +1,101 @@ +package de.ioexception.www.server.impl; + +import java.io.BufferedWriter; +import java.io.File; +import java.io.FileWriter; +import java.io.IOException; +import java.text.SimpleDateFormat; +import java.util.Date; +import java.util.Locale; +import java.util.concurrent.LinkedBlockingQueue; + +import de.ioexception.www.http.HttpRequest; +import de.ioexception.www.http.HttpResponse; +import de.ioexception.www.server.AccessLogger; + +public class BasicAccessLogger implements AccessLogger +{ + private final BufferedWriter log; + private volatile boolean running = true; + private final LinkedBlockingQueue logQueue = new LinkedBlockingQueue(); + + private final SimpleDateFormat dateFormat = new SimpleDateFormat("dd/MMM/yyyy:HH:mm:ss Z", Locale.ENGLISH); + + /* + * [10/Oct/2000:13:55:36 -0700] (%t) + The time that the request was received. The format is: + + [day/month/year:hour:minute:second zone] + day = 2*digit + month = 3*letter + year = 4*digit + hour = 2*digit + minute = 2*digit + second = 2*digit + zone = (`+' | `-') 4*digit + */ + + public BasicAccessLogger(File logFile) throws IOException + { + log = new BufferedWriter(new FileWriter(logFile, true)); + } + + + @Override + public void log(String logline) + { + try + { + logQueue.put(logline); + } + catch (InterruptedException e) + { + e.printStackTrace(); + } + } + + + @Override + public Void call() throws Exception + { + try + { + while(running || logQueue.size() > 0) + { + log.write(logQueue.take()); + log.flush(); + } + log.flush(); + return null; + } + finally + { + System.err.println("closing.."); + log.close(); + } + } + + public void shutdown() + { + running = false; + } + + + @Override + public void log(String clientHost, HttpRequest request, HttpResponse response) + { + StringBuilder s = new StringBuilder(); + s.append(clientHost+" "); + s.append("- "); + s.append("- "); + s.append("["+dateFormat.format(new Date())+"] "); + s.append("\""+request.getHttpMethod().toString()+" "+request.getRequestUri()+" "+request.getHttpVersion().toString()+"\" "); + s.append(response.getStatusCode().getCode()+" "); + s.append((response.getEntity() != null ? response.getEntity().length : 0)+" "); + s.append("\""+(request.getHeaders().containsKey("Referer") ? request.getHeaders().get("Referer") : "")+"\" " ); + s.append("\""+(request.getHeaders().containsKey("User-Agent") ? request.getHeaders().get("User-Agent") : "")+"\""); + s.append("\r\n"); + log(s.toString()); + } + +} diff --git a/src/de/ioexception/www/server/impl/BasicHttpServer.java b/src/de/ioexception/www/server/impl/BasicHttpServer.java index ddca550..6ea16c4 100644 --- a/src/de/ioexception/www/server/impl/BasicHttpServer.java +++ b/src/de/ioexception/www/server/impl/BasicHttpServer.java @@ -1,5 +1,6 @@ package de.ioexception.www.server.impl; +import java.io.File; import java.io.IOException; import java.net.ServerSocket; import java.net.Socket; @@ -7,6 +8,7 @@ import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; +import de.ioexception.www.server.AccessLogger; import de.ioexception.www.server.HttpServer; /** @@ -26,7 +28,9 @@ public class BasicHttpServer implements HttpServer private final ExecutorService workerPool; private final ExecutorService dispatcherService; + private final ExecutorService loggingService; private final ServerSocket serverSocket; + private final AccessLogger accessLogger; /** @@ -51,6 +55,8 @@ public BasicHttpServer(int port) serverSocket = new ServerSocket(port); workerPool = Executors.newFixedThreadPool(16); dispatcherService = Executors.newSingleThreadExecutor(); + loggingService = Executors.newSingleThreadExecutor(); + accessLogger = new BasicAccessLogger(new File("log/access.log")); } catch (IOException e) { @@ -69,6 +75,7 @@ public void dispatchRequest(Socket socket) public void start() { running = true; + loggingService.submit(accessLogger); // Initiate the main server loop accepting incoming connections. dispatcherService.submit(new Runnable() { @@ -123,4 +130,10 @@ public String getServerSignature() return BasicHttpServer.SERVER_SIGNATURE; } + @Override + public AccessLogger getAccessLogger() + { + return accessLogger; + } + } From a0da5869578628ecca4bfa14d924dccfb3f98c40 Mon Sep 17 00:00:00 2001 From: Benjamin Erb Date: Wed, 22 Dec 2010 23:45:27 +0100 Subject: [PATCH 2/2] added more loggers and buildfile --- build.xml | 38 +++++++ src/de/ioexception/www/server/HttpServer.java | 2 + .../www/server/impl/BasicAccessLogger.java | 101 ----------------- .../www/server/impl/BasicHttpServer.java | 7 +- .../www/server/{ => log}/AccessLogger.java | 2 +- .../log/impl/BufferedFileAccessLogger.java | 25 ++++ .../server/log/impl/ConsoleAccessLogger.java | 32 ++++++ .../www/server/log/impl/FileAccessLogger.java | 23 ++++ .../server/log/impl/GenericAccessLogger.java | 107 ++++++++++++++++++ 9 files changed, 233 insertions(+), 104 deletions(-) create mode 100644 build.xml delete mode 100644 src/de/ioexception/www/server/impl/BasicAccessLogger.java rename src/de/ioexception/www/server/{ => log}/AccessLogger.java (88%) create mode 100644 src/de/ioexception/www/server/log/impl/BufferedFileAccessLogger.java create mode 100644 src/de/ioexception/www/server/log/impl/ConsoleAccessLogger.java create mode 100644 src/de/ioexception/www/server/log/impl/FileAccessLogger.java create mode 100644 src/de/ioexception/www/server/log/impl/GenericAccessLogger.java diff --git a/build.xml b/build.xml new file mode 100644 index 0000000..f201982 --- /dev/null +++ b/build.xml @@ -0,0 +1,38 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/src/de/ioexception/www/server/HttpServer.java b/src/de/ioexception/www/server/HttpServer.java index c801c63..ab1c239 100644 --- a/src/de/ioexception/www/server/HttpServer.java +++ b/src/de/ioexception/www/server/HttpServer.java @@ -2,6 +2,8 @@ import java.net.Socket; +import de.ioexception.www.server.log.AccessLogger; + /** * A basic HTTP server interface. * diff --git a/src/de/ioexception/www/server/impl/BasicAccessLogger.java b/src/de/ioexception/www/server/impl/BasicAccessLogger.java deleted file mode 100644 index fa75b38..0000000 --- a/src/de/ioexception/www/server/impl/BasicAccessLogger.java +++ /dev/null @@ -1,101 +0,0 @@ -package de.ioexception.www.server.impl; - -import java.io.BufferedWriter; -import java.io.File; -import java.io.FileWriter; -import java.io.IOException; -import java.text.SimpleDateFormat; -import java.util.Date; -import java.util.Locale; -import java.util.concurrent.LinkedBlockingQueue; - -import de.ioexception.www.http.HttpRequest; -import de.ioexception.www.http.HttpResponse; -import de.ioexception.www.server.AccessLogger; - -public class BasicAccessLogger implements AccessLogger -{ - private final BufferedWriter log; - private volatile boolean running = true; - private final LinkedBlockingQueue logQueue = new LinkedBlockingQueue(); - - private final SimpleDateFormat dateFormat = new SimpleDateFormat("dd/MMM/yyyy:HH:mm:ss Z", Locale.ENGLISH); - - /* - * [10/Oct/2000:13:55:36 -0700] (%t) - The time that the request was received. The format is: - - [day/month/year:hour:minute:second zone] - day = 2*digit - month = 3*letter - year = 4*digit - hour = 2*digit - minute = 2*digit - second = 2*digit - zone = (`+' | `-') 4*digit - */ - - public BasicAccessLogger(File logFile) throws IOException - { - log = new BufferedWriter(new FileWriter(logFile, true)); - } - - - @Override - public void log(String logline) - { - try - { - logQueue.put(logline); - } - catch (InterruptedException e) - { - e.printStackTrace(); - } - } - - - @Override - public Void call() throws Exception - { - try - { - while(running || logQueue.size() > 0) - { - log.write(logQueue.take()); - log.flush(); - } - log.flush(); - return null; - } - finally - { - System.err.println("closing.."); - log.close(); - } - } - - public void shutdown() - { - running = false; - } - - - @Override - public void log(String clientHost, HttpRequest request, HttpResponse response) - { - StringBuilder s = new StringBuilder(); - s.append(clientHost+" "); - s.append("- "); - s.append("- "); - s.append("["+dateFormat.format(new Date())+"] "); - s.append("\""+request.getHttpMethod().toString()+" "+request.getRequestUri()+" "+request.getHttpVersion().toString()+"\" "); - s.append(response.getStatusCode().getCode()+" "); - s.append((response.getEntity() != null ? response.getEntity().length : 0)+" "); - s.append("\""+(request.getHeaders().containsKey("Referer") ? request.getHeaders().get("Referer") : "")+"\" " ); - s.append("\""+(request.getHeaders().containsKey("User-Agent") ? request.getHeaders().get("User-Agent") : "")+"\""); - s.append("\r\n"); - log(s.toString()); - } - -} diff --git a/src/de/ioexception/www/server/impl/BasicHttpServer.java b/src/de/ioexception/www/server/impl/BasicHttpServer.java index 6ea16c4..f111acf 100644 --- a/src/de/ioexception/www/server/impl/BasicHttpServer.java +++ b/src/de/ioexception/www/server/impl/BasicHttpServer.java @@ -8,8 +8,10 @@ import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; -import de.ioexception.www.server.AccessLogger; import de.ioexception.www.server.HttpServer; +import de.ioexception.www.server.log.AccessLogger; +import de.ioexception.www.server.log.impl.BufferedFileAccessLogger; +import de.ioexception.www.server.log.impl.ConsoleAccessLogger; /** * A simple HTTP server implementation. @@ -56,7 +58,8 @@ public BasicHttpServer(int port) workerPool = Executors.newFixedThreadPool(16); dispatcherService = Executors.newSingleThreadExecutor(); loggingService = Executors.newSingleThreadExecutor(); - accessLogger = new BasicAccessLogger(new File("log/access.log")); +// accessLogger = new BufferedFileAccessLogger(new File("log/access.log")); + accessLogger = new ConsoleAccessLogger(); } catch (IOException e) { diff --git a/src/de/ioexception/www/server/AccessLogger.java b/src/de/ioexception/www/server/log/AccessLogger.java similarity index 88% rename from src/de/ioexception/www/server/AccessLogger.java rename to src/de/ioexception/www/server/log/AccessLogger.java index 3ae0af8..662389b 100644 --- a/src/de/ioexception/www/server/AccessLogger.java +++ b/src/de/ioexception/www/server/log/AccessLogger.java @@ -1,4 +1,4 @@ -package de.ioexception.www.server; +package de.ioexception.www.server.log; import java.util.concurrent.Callable; diff --git a/src/de/ioexception/www/server/log/impl/BufferedFileAccessLogger.java b/src/de/ioexception/www/server/log/impl/BufferedFileAccessLogger.java new file mode 100644 index 0000000..c5f4cd0 --- /dev/null +++ b/src/de/ioexception/www/server/log/impl/BufferedFileAccessLogger.java @@ -0,0 +1,25 @@ +package de.ioexception.www.server.log.impl; + +import java.io.BufferedWriter; +import java.io.File; +import java.io.FileWriter; +import java.io.IOException; + +/** + * A File-based Logger using the "Combined Log Format". It buffers log entries + * and writes them into the logfile when a certain amount of chunks is + * available. This decreases the overhead of i/o operations, but will cause loss + * of entries in case of ungrateful server shutdowns. + * + * @author Benjamin Erb + * + */ +public class BufferedFileAccessLogger extends GenericAccessLogger +{ + + public BufferedFileAccessLogger(File logFile) throws IOException + { + super(new BufferedWriter(new FileWriter(logFile, true))); + } + +} diff --git a/src/de/ioexception/www/server/log/impl/ConsoleAccessLogger.java b/src/de/ioexception/www/server/log/impl/ConsoleAccessLogger.java new file mode 100644 index 0000000..9091b42 --- /dev/null +++ b/src/de/ioexception/www/server/log/impl/ConsoleAccessLogger.java @@ -0,0 +1,32 @@ +package de.ioexception.www.server.log.impl; + +import java.io.IOException; +import java.io.PrintWriter; + +/** + * A logger for console output. Flushes every entry immediately. + * + * @author Benjamin Erb + * + */ +public class ConsoleAccessLogger extends GenericAccessLogger +{ + public ConsoleAccessLogger() + { + super(new PrintWriter(System.out)); + } + + @Override + public void log(String logline) + { + super.log(logline); + try + { + super.flush(); + } + catch (IOException e) + { + e.printStackTrace(); + } + } +} diff --git a/src/de/ioexception/www/server/log/impl/FileAccessLogger.java b/src/de/ioexception/www/server/log/impl/FileAccessLogger.java new file mode 100644 index 0000000..7ff433f --- /dev/null +++ b/src/de/ioexception/www/server/log/impl/FileAccessLogger.java @@ -0,0 +1,23 @@ +package de.ioexception.www.server.log.impl; + +import java.io.File; +import java.io.FileWriter; +import java.io.IOException; + +/** + * A File-based Logger using the "Combined Log Format". It directly writes each + * log entry into file and thus leads to poor i/o efficiency under heavy server + * load. + * + * @author Benjamin Erb + * + */ +public class FileAccessLogger extends GenericAccessLogger +{ + + public FileAccessLogger(File logFile) throws IOException + { + super(new FileWriter(logFile, true)); + } + +} diff --git a/src/de/ioexception/www/server/log/impl/GenericAccessLogger.java b/src/de/ioexception/www/server/log/impl/GenericAccessLogger.java new file mode 100644 index 0000000..e29dad3 --- /dev/null +++ b/src/de/ioexception/www/server/log/impl/GenericAccessLogger.java @@ -0,0 +1,107 @@ +package de.ioexception.www.server.log.impl; + +import java.io.Flushable; +import java.io.IOException; +import java.io.Writer; +import java.text.SimpleDateFormat; +import java.util.Date; +import java.util.Locale; +import java.util.concurrent.LinkedBlockingQueue; + +import de.ioexception.www.http.HttpRequest; +import de.ioexception.www.http.HttpResponse; +import de.ioexception.www.server.log.AccessLogger; + +/** + * An abstract logger class with provides basic functionalities for logging + * access, using the "Combined Log Format". + * + * @author Benjamin Erb + */ +public abstract class GenericAccessLogger implements AccessLogger, Flushable +{ + private final SimpleDateFormat dateFormat = new SimpleDateFormat("dd/MMM/yyyy:HH:mm:ss Z", Locale.ENGLISH); + + private volatile boolean running = true; + private final LinkedBlockingQueue logQueue = new LinkedBlockingQueue(); + private final Writer writer; + + /** + * The writer instance to be used. + * + * @param writer + */ + public GenericAccessLogger(Writer writer) + { + this.writer = writer; + } + + @Override + public void log(String logline) + { + try + { + logQueue.put(logline); + } + catch (InterruptedException e) + { + e.printStackTrace(); + } + } + + @Override + public Void call() throws Exception + { + try + { + while (running || logQueue.size() > 0) + { + writer.write(logQueue.take()); + } + writer.flush(); + return null; + } + finally + { + writer.close(); + } + } + + public void shutdown() + { + try + { + writer.flush(); + } + catch (IOException e) + { + e.printStackTrace(); + } + running = false; + } + + + @Override + public void log(String clientHost, HttpRequest request, HttpResponse response) + { + StringBuilder s = new StringBuilder(); + s.append(clientHost + " "); + s.append("- "); + s.append("- "); + s.append("[" + dateFormat.format(new Date()) + "] "); + s.append("\"" + request.getHttpMethod().toString() + " " + request.getRequestUri() + " " + request.getHttpVersion().toString() + "\" "); + s.append(response.getStatusCode().getCode() + " "); + s.append((response.getEntity() != null ? response.getEntity().length : 0) + " "); + s.append("\"" + (request.getHeaders().containsKey("Referer") ? request.getHeaders().get("Referer") : "") + "\" "); + s.append("\"" + (request.getHeaders().containsKey("User-Agent") ? request.getHeaders().get("User-Agent") : "") + "\""); + s.append("\n"); + log(s.toString()); + } + + @Override + public void flush() throws IOException + { + writer.flush(); + } + +}