mirror of https://github.com/OpenIdentityPlatform/OpenDJ.git

Jean-Noel Rouvignac
13.05.2013 29199b507f80b88f98dfee74bb508b37926c7cda
OPENDJ-902 (CR-1654) Add connectionID to the HTTP access log + move to extended log format

HTTPConnectionHandler.java, HTTPClientConnection.java, CollectClientConnectionsFilter.java, SdkConnectionAdapter.java:
Added support for statistics.

02-config.ldif, config.ldif, HTTPConnectionHandlerConfiguration.xml, HTTPConnectionHandlerCfgDefn.properties:
Added "keep-stats" property.

02-config.ldif:
Added attributes "ds-mon-http-*" for monitoring HTTP statistics.

LDAPStatistics.java:
Changed getMonitorData() return type to List.
Made createAttribute protected.

HTTPStatsProbe.java, HTTPStatistics.java: ADDED
7 files modified
210 ■■■■■ changed files
opends/resource/schema/02-config.ldif 1 ●●●● patch | view | raw | blame | history
opends/src/admin/defn/org/opends/server/admin/std/FileBasedHTTPAccessLogPublisherConfiguration.xml 36 ●●●●● patch | view | raw | blame | history
opends/src/admin/messages/FileBasedHTTPAccessLogPublisherCfgDefn.properties 2 ●●●●● patch | view | raw | blame | history
opends/src/server/org/opends/server/api/HTTPAccessLogPublisher.java 35 ●●●● patch | view | raw | blame | history
opends/src/server/org/opends/server/loggers/HTTPRequestInfo.java 23 ●●●●● patch | view | raw | blame | history
opends/src/server/org/opends/server/loggers/TextHTTPAccessLogPublisher.java 109 ●●●●● patch | view | raw | blame | history
opends/src/server/org/opends/server/protocols/http/CollectClientConnectionsFilter.java 4 ●●● patch | view | raw | blame | history
opends/resource/schema/02-config.ldif
@@ -3994,6 +3994,7 @@
        ds-cfg-auto-flush $
        ds-cfg-append $
        ds-cfg-queue-size $
        ds-cfg-log-format $
        ds-cfg-log-record-time-format )
  X-ORIGIN 'OpenDJ Directory Server' )
objectClasses: ( 1.3.6.1.4.1.26027.1.2.26
opends/src/admin/defn/org/opends/server/admin/std/FileBasedHTTPAccessLogPublisherConfiguration.xml
@@ -285,6 +285,42 @@
      </ldap:attribute>
    </adm:profile>
  </adm:property>
  <adm:property name="log-format">
    <adm:synopsis>
      Specifies how log records should be formatted and written to the HTTP
      access log.
    </adm:synopsis>
    <adm:default-behavior>
      <adm:defined>
        <adm:value>cs-host c-ip cs-username datetime cs-method cs-uri-query
        cs-version sc-status sc-bytes cs(User-Agent) x-connection-id</adm:value>
      </adm:defined>
    </adm:default-behavior>
    <adm:syntax>
     <adm:string>
       <adm:pattern>
        <adm:regex>[a-zA-Z0-9-()]+( [a-zA-Z0-9-()]+)*</adm:regex>
        <adm:usage>FORMAT</adm:usage>
          <adm:synopsis>
            A string describing the extended log format to be used for logging
            HTTP accesses. Available values are listed on the W3C working draft
            http://www.w3.org/TR/WD-logfile.html and Microsoft website
            http://www.microsoft.com/technet/prodtechnol/WindowsServer2003/Library/IIS/676400bc-8969-4aa7-851a-9319490a9bbb.mspx?mfr=true
            . Please note that "date" and "time" have been merged into
            "datetime" and that its ouput is controlled by
            "ds-cfg-log-record-time-format" property. OpenDJ adds the
            application specific field "x-connection-id" which displays the
            internal connection ID assigned to the HTTP client connection.
          </adm:synopsis>
        </adm:pattern>
      </adm:string>
    </adm:syntax>
    <adm:profile name="ldap">
      <ldap:attribute>
        <ldap:name>ds-cfg-log-format</ldap:name>
      </ldap:attribute>
    </adm:profile>
  </adm:property>
  <adm:property name="log-record-time-format">
    <adm:synopsis>
      Specifies the format string that is used to generate log record
opends/src/admin/messages/FileBasedHTTPAccessLogPublisherCfgDefn.properties
@@ -12,6 +12,8 @@
property.log-file.syntax.string.pattern.synopsis=A path to an existing file that is readable by the server.
property.log-file-permissions.synopsis=The UNIX permissions of the log files created by this File Based HTTP Access Log Publisher.
property.log-file-permissions.syntax.string.pattern.synopsis=A valid UNIX mode string. The mode string must contain three digits between zero and seven.
property.log-format.synopsis=Specifies how log records should be formatted and written to the HTTP access log.
property.log-format.syntax.string.pattern.synopsis=A string describing the extended log format to be used for logging HTTP accesses. Available values are listed on the W3C working draft http://www.w3.org/TR/WD-logfile.html and Microsoft website http://www.microsoft.com/technet/prodtechnol/WindowsServer2003/Library/IIS/676400bc-8969-4aa7-851a-9319490a9bbb.mspx?mfr=true . Please note that "date" and "time" have been merged into "datetime" and that its ouput is controlled by "ds-cfg-log-record-time-format" property. OpenDJ adds the application specific field "x-connection-id" which displays the internal connection ID assigned to the HTTP client connection.
property.log-record-time-format.synopsis=Specifies the format string that is used to generate log record timestamps.
property.log-record-time-format.syntax.string.pattern.synopsis=Any valid format string that can be used with the java.text.SimpleDateFormat class.
property.queue-size.synopsis=The maximum number of log records that can be stored in the asynchronous queue.
opends/src/server/org/opends/server/api/HTTPAccessLogPublisher.java
@@ -61,37 +61,16 @@
  }
  /**
   * Logs the request info according to the common logfile format. The common
   * logfile format is as follows:
   *
   * <pre>
   * remotehost rfc931 authuser [date] "request" status bytes "useragent"
   * </pre>
   * <dl>
   * <dt>remotehost</dt>
   * <dd>Remote hostname (or IP number if DNS hostname is not available, or if
   * DNSLookup is Off.</dd>
   * <dt>rfc931</dt>
   * <dd>The remote logname of the user.</dd>
   * <dt>authuser</dt>
   * <dd>The username as which the user has authenticated himself.</dd>
   * <dt>[date]</dt>
   * <dd>Date and time of the request.</dd>
   * <dt>"request"</dt>
   * <dd>The request line exactly as it came from the client.</dd>
   * <dt>status</dt>
   * <dd>The HTTP status code returned to the client.</dd>
   * <dt>bytes</dt>
   * <dd>The content-length of the document transferred.</dd>
   * <dt>"useragent"</dt>
   * <dd>The user agent that issued the request.</dd>
   * </dl>
   * <p>
   * <b>NOTE:</b> The bytes field is not currently supported.
   * </p>
   * Logs the request info according to the configured extended log format.
   *
   * @param requestInfo
   *          The request info to log
   * @see <a href="http://www.w3.org/TR/WD-logfile.html">W3C's Extended Log File
   *      Format</a>
   * @see <a href=
   *      "http://www.microsoft.com/technet/prodtechnol/WindowsServer2003/
   *      Library/IIS/676400bc-8969-4aa7-851a-9319490a9bbb.mspx?mfr=true">
   *      Microsoft's W3C Extended Log File Format (IIS 6.0)</a>
   */
  public void logRequestInfo(HTTPRequestInfo requestInfo)
  {
opends/src/server/org/opends/server/loggers/HTTPRequestInfo.java
@@ -56,14 +56,22 @@
   * was set since it is not .
   */
  private AtomicInteger statusCode = new AtomicInteger(0);
  /**
   * The unique identifier that has been assigned to the client connection for
   * this HTTP request.
   */
  private long connectionID;
  /**
   * Constructor for this class.
   *
   * @param request
   *          The {@link HttpServletRequest} for which to log the information
   * @param connectionID
   *          The unique identifier that has been assigned to the client
   *          connection for this HTTP request
   */
  public HTTPRequestInfo(HttpServletRequest request)
  public HTTPRequestInfo(HttpServletRequest request, long connectionID)
  {
    this.remoteHost = request.getRemoteHost();
    this.remoteAddress = request.getRemoteAddr();
@@ -71,6 +79,7 @@
    this.query = request.getRequestURI() + "/" + request.getQueryString();
    this.protocol = request.getProtocol();
    this.userAgent = request.getHeader("User-Agent");
    this.connectionID = connectionID;
  }
  /**
@@ -166,6 +175,18 @@
  }
  /**
   * Returns the unique identifier that has been assigned to the client
   * connection for this HTTP request.
   *
   * @return The unique identifier that has been assigned to the client
   *         connection for this HTTP request
   */
  public long getConnectionID()
  {
    return this.connectionID;
  }
  /**
   * Logs the current request info in the HTTP access log.
   *
   * @param statusCode
opends/src/server/org/opends/server/loggers/TextHTTPAccessLogPublisher.java
@@ -31,10 +31,11 @@
import java.io.File;
import java.io.IOException;
import java.text.DateFormat;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import org.opends.messages.Message;
import org.opends.server.admin.server.ConfigurationChangeListener;
@@ -82,9 +83,8 @@
  private TextWriter writer = null;
  private FileBasedHTTPAccessLogPublisherCfg cfg = null;
  private String[] logFormatFields;
  private String timeStampFormat = "dd/MMM/yyyy:HH:mm:ss Z";
  private DateFormat dateFormatter = new SimpleDateFormat(timeStampFormat);
  /** {@inheritDoc} */
@@ -195,10 +195,11 @@
        if (!config.getLogRecordTimeFormat().equals(timeStampFormat))
        {
          TimeThread.removeUserDefinedFormatter(timeStampFormat);
          setTimeStampFormat(config.getLogRecordTimeFormat());
          timeStampFormat = config.getLogRecordTimeFormat();
        }
        cfg = config;
        logFormatFields = extractFieldsOrder(cfg.getLogFormat());
      }
    }
    catch (final Exception e)
@@ -213,6 +214,14 @@
  }
  private String[] extractFieldsOrder(String logFormat)
  {
    if (logFormat != null)
    {
      return logFormat.split("\\s");
    }
    return null;
  }
  /** {@inheritDoc} */
  @Override
@@ -284,7 +293,8 @@
    }
    this.cfg = cfg;
    setTimeStampFormat(cfg.getLogRecordTimeFormat());
    logFormatFields = extractFieldsOrder(cfg.getLogFormat());
    timeStampFormat = cfg.getLogRecordTimeFormat();
    cfg.addFileBasedHTTPAccessChangeListener(this);
  }
@@ -357,12 +367,6 @@
    }
  }
  private void setTimeStampFormat(String timeStampFormat)
  {
    this.timeStampFormat = timeStampFormat;
    this.dateFormatter = new SimpleDateFormat(timeStampFormat);
  }
  /** {@inheritDoc} */
  @Override
  public final DN getDN()
@@ -374,38 +378,69 @@
  @Override
  public void logRequestInfo(HTTPRequestInfo ri)
  {
    final StringBuilder sb = new StringBuilder(100);
    final Map<String, Object> fields = new HashMap<String, Object>();
    fields.put("cs-host", ri.getRemoteHost());
    fields.put("c-ip", ri.getRemoteAddress());
    fields.put("cs-username", ri.getAuthUser());
    fields.put("datetime", TimeThread.getUserDefinedTime(timeStampFormat));
    fields.put("cs-method", ri.getMethod());
    fields.put("cs-uri-query", ri.getQuery());
    fields.put("cs-version", ri.getProtocol());
    fields.put("sc-status", ri.getStatusCode());
    fields.put("cs(User-Agent)", ri.getUserAgent());
    fields.put("x-connection-id", ri.getConnectionID());
    // remotehost
    if (ri.getRemoteHost() != null)
    writeLogRecord(fields, logFormatFields);
  }
  private void writeLogRecord(Map<String, Object> fields, String... fieldnames)
  {
    if (fieldnames == null)
    {
      sb.append(ri.getRemoteHost());
      return;
    }
    final StringBuilder sb = new StringBuilder(100);
    for (String fieldname : fieldnames)
    {
      append(sb, fields.get(fieldname));
    }
    writer.writeRecord(sb.toString());
  }
  /**
   * Appends the value to the string builder using the default separator if
   * needed.
   *
   * @param sb
   *          the StringBuilder where to append.
   * @param value
   *          the value to append.
   */
  private void append(final StringBuilder sb, Object value)
  {
    final char separator = '\t'; // as encouraged by the W3C working draft
    if (sb.length() > 0)
    {
      sb.append(separator);
    }
    if (value != null)
    {
      String val = String.valueOf(value);
      boolean useQuotes = val.contains(Character.toString(separator));
      if (useQuotes)
      {
        sb.append('"').append(val.replaceAll("\"", "\"\"")).append('"');
      }
      else
      {
        sb.append(val);
      }
    }
    else
    {
      sb.append(ri.getRemoteAddress());
      sb.append('-');
    }
    // rfc931 - not supported
    // authuser
    sb.append(" ");
    if (ri.getAuthUser() != null)
    {
      sb.append(ri.getAuthUser());
    }
    // [dateAndTime]
    sb.append(" [").append(TimeThread.getUserDefinedTime(timeStampFormat))
        .append("]");
    // "request"
    sb.append(" \"").append(ri.getMethod());
    sb.append(" ").append(ri.getQuery());
    sb.append(" ").append(ri.getProtocol()).append("\"");
    // HTTP response status code
    sb.append(" ").append(ri.getStatusCode());
    // bytes - not supported
    // "user agent"
    sb.append(" \"").append(ri.getUserAgent()).append("\"");
    writer.writeRecord(sb.toString());
  }
}
opends/src/server/org/opends/server/protocols/http/CollectClientConnectionsFilter.java
@@ -259,6 +259,7 @@
      }
      /** {@inheritDoc} */
      @SuppressWarnings("deprecation")
      @Override
      public void setStatus(int sc, String sm)
      {
@@ -269,13 +270,14 @@
    ctx.chain = chain;
    ctx.prettyPrint =
        Boolean.parseBoolean(request.getParameter("_prettyPrint"));
    ctx.requestInfo = new HTTPRequestInfo(ctx.request);
    final HTTPClientConnection clientConnection =
        new HTTPClientConnection(this.connectionHandler, request);
    this.connectionHandler.addClientConnection(clientConnection);
    ctx.clientConnection = clientConnection;
    ctx.requestInfo =
        new HTTPRequestInfo(ctx.request, clientConnection.getConnectionID());
    try
    {