Warning: This message has had one or more attachments removed
Warning: (Logs.map.xml).
Warning: Please read the "VirusWarning.txt" attachment(s) for more information.
Hi All,
I must say I've been very happy with Cayenne so far. I've been using it
for a month or sow now on an log reporting tool. This tool takes log
files and puts them into a database and from there will provide a series
of reports.
At the moment I seem to have run into a problem where I run out of
memory when I'm trying to insert the log files into the database (the
files total over 300MB of data). The database, currently has three tables:
Message - the log message.
Log_Level - the level of the message, e.g. warn, info.
Host - The host the message was logged on.
At the moment I seem to be able to insert about 44000 entries before I
run out of memory (Heap size: 64MB). Now I believe that I'm getting rid
of all of the instances of Message, LogLevel and Host (the java
objects) that I am creating, but I'm not sure, and I was hoping someone
could help me out.
The following code are the two classes that sit above the Cayenne
classes. The only modifications I have made to the Cayenne classes is
to over ride the toString methods.
Code over view:
TestDB finds the log files and then iterates over it and adds them to
the DBLog. DBLog will cache N (100 in this case) entries before
flushing them to the database (context.commitChanges()). I also call
flush between files to ensure that all of the entries have been saved to
the DB.
Any help would be greatly appreciated.
== TestDB.java ==
<imports removed>
public class TestDB {
public static void main(String[] argv){
config = Configuration.getSharedConfiguration();
domain = config.getDomain();
context = DataContext.createDataContext();
String files[] = {
"e:/tmp/NTGTest_Logs/log/skip.log.old-20041009-080513",
"e:/tmp/NTGTest_Logs/log/skip.log.old-20041011-025444",
"e:/tmp/NTGTest_Logs/log/skip.log.old-20041011-025444"
};
for(int i = 0; i < files.length; i++){
insertIntoLog(files[i]);
}
}
private static void insertIntoLog(String file){
//Log outputLog = new DBLog(context, 10000);
Log outputLog = new DBLog(context, 100);
System.out.println("Processing " + file);
Log inputLog = new SpiderLog(file, inputTimeZone);
Entry entry;
Map params;
List messages;
SelectQuery sq;
try{
while(inputLog.hasNext()){
params = new HashMap();
entry =(Entry)inputLog.next() ;
entry.setHost("search.nt.gov.au");
entry.setProcessID(-1);
outputLog.add(entry);
/*
params.put("date", entry.getDate());
params.put("message", entry.getMessage());
sq = checkQuery.queryWithParameters(params);
messages = context.performQuery(sq);
if(messages.size() == 0){
outputLog.add(entry);
System.out.print(".");
} else {
System.out.println ("Diplicate '" + entry.getDate()
+ ": " + entry.getMessage()
+ "' - skipping");
}
*/
}
outputLog.flush();
} catch(LogUncheckedException f){
System.err.println("Unchecked: " + f.getMessage());
} catch(LogCheckedException f){
System.err.println("Checked: " + f.getMessage());
}
}
private static Configuration config =
Configuration.getSharedConfiguration();
private static DataDomain domain = config.getDomain();
private static DataContext context =
DataContext.createDataContext();
private static TimeZone inputTimeZone =TimeZone.getTimeZone(
"Australia/Darwin");
private static final Expression checkTemplate =
Expression.fromString(
"loggedDate = $date and message = $message");
private static final SelectQuery checkQuery = new SelectQuery(
Message.class);
static {
checkQuery.setQualifier(checkTemplate);
}
}
== End TestDB.java ==
== DBLog.java ==
package au.com.relevance.log.db;
<imports removed>
/**
* DBLog implements the Log interface for databases
* conforming to the Relevance Logging Database. This interface uses
* the Cayenne ORM to achieve this goal.<p>
*
* The DBLog automatically flushes the log entries based on the cache
* size. The default cache size is 1.
*
* <strong>NB: Currently does not support reading from the data
* store </strong>
*
*..ee Log
*/
public class DBLog extends AbstractLog {
< removed un-used constructors >
/**
*
*..aram context the Cayenne context for the database.
*..aram cacheSize the number of entries to store before flushing
* the buffer automatically.
*/
public DBLog(DataContext context, int cacheSize) {
this.ctxt = context;
this.cacheSize = cacheSize;
createQueue();
}
/**
* ..nheritDoc}
*/
public void add(Entry entry) throws LogAttributeException,
LogCheckedException, LogHostException {
entryQueue.add(entry);
if(entryQueue.isFull()){
flush();
} else {
emptyEntry = false;
}
}
/**
* ..nheritDoc}
*/
public void flush() throws LogAttributeException,
LogCheckedException,
LogHostException, LogLogLevelException {
Message m;
Entry e;
Iterator i = entryQueue.iterator();
while(i.hasNext()){
e = (Entry) i.next();
checkAttributes(e);
m = (Message)
ctxt.createAndRegisterNewObject(Message.class);
m.setHost(host);
m.setLogLevel(level);
m.setMessage(e.getMessage());
m.setProcessID(e.getProcessID());
m.setLoggedDate(e.getDate());
m.setApplication(e.getApplication());
m.setEnteredDate(Calendar.getInstance().getTime());
}
i = null;
ctxt.commitChanges();
createQueue();
emptyEntry = true;
}
/**
* Ensure that the required attributes are set for the
* entry.
*
*..aram entry the entry to check.
*
*..eturn true if the attributes are set appropriately, false
* otherwise.
*/
private boolean checkAttributes(Entry entry)
throws LogAttributeException,
LogHostException, LogLogLevelException {
boolean passed = true;
boolean firstAttribute = true;
StringBuffer attributeExceptionMsg = new StringBuffer(
"The follwoing attribute(s) were not set: ");
/*
* Verify all of the attributes
*/
if(entry.getHost() == null){
firstAttribute = false;
attributeExceptionMsg.append("Host name");
}
if(entry.getDate() == null){
firstAttribute = false;
if(!firstAttribute){
attributeExceptionMsg.append(", ");
}
attributeExceptionMsg.append("Date");
}
< other attribute test removed for brefity >
if(!firstAttribute){
throw new
LogAttributeException(attributeExceptionMsg.toString());
}
/*
* Verify the host name and log level
*/
host = getHostObject(entry.getHost());
level = getLogLevelObject(entry.getLogLevel());
return passed;
}
/**
* Retrieve the host object from the database.
*
*..aram host the name of the host.
*
*..eturn the host object.
*/
private Host getHostObject(String host) throws LogHostException {
Map params = new HashMap();
List hosts;
params.put("name", host);
hosts = ctxt.performQuery("HostQuery", params, false);
if(hosts.size() == 0){
throw new LogHostException(
"The host for the entry wasn't found in the database");
}
return (Host) hosts.get(0);
}
/**
* ..nheritDoc}
*/
public void snapShot(){
snapShot = true;
}
/**
* ..nheritDoc}
*/
public void keepUpToDate(){
snapShot = false;
}
/**
* ..nheritDoc}
* <strong> Not Currently Supported </strong>
*/
public Object next() {
throw new LogUncheckedException(
"This part of the interface is not currently supported");
}
/**
* ..nheritDoc}
* <strong> Not Currently Supported </strong>
*/
public boolean hasNext() {
throw new LogUncheckedException(
"This part of the interface is not currently supported");
}
/**
* Retrieve the log level object from the database.
*
*..aram level the log level to retrieve.
*
*..eturn the log level object representing the <code>level</code>
*/
private LogLevel getLogLevelObject(String level) throws
LogLogLevelException {
Map params = new HashMap();
//SelectQuery sq;
List logLevels;
params.put("name", level);
//sq = logLevelQuery.queryWithParameters(params);
//logLevels = ctxt.performQuery(sq);
logLevels = ctxt.performQuery("LogLevelQuery", params, false);
if(logLevels.size() == 0){
throw new LogLogLevelException(
"The log level for the entry wasn't found in the
database");
}
return (LogLevel) logLevels.get(0);
}
/**
* Create a new queue.
*/
private void createQueue(){
entryQueue = new BoundedFifoBuffer(cacheSize);
}
/*
* Database details
*/
private DataContext ctxt;
/*
* The entry details
*/
private Entry entry = null;
private Host host = null;
private LogLevel level = null;
/*
* Internal State
*/
private boolean emptyEntry = true;
private boolean snapShot = true;
private BoundedFifoBuffer entryQueue;
/*
* Default Cache Size
*/
private int cacheSize = 1;
/*
* Queries to find things.
*/
private static final SelectQuery hostQuery =
new SelectQuery(Host.class);
private static final SelectQuery logLevelQuery =
new SelectQuery(LogLevel.class);
/*
* Message strings
*/
private static final String startAttributeMessage =
"The attribute(s) ";
private static final String endAttributeMessage = ", were not set.";
}
== End DBLog.java ==
-- Justin DeutschRelevance... because results count
Relevance Phone: +61 (0)2 6241 6400 A.B.N. 86 063 403 690 Fax: +61 (0)2 6241 6422 Unit 11, Mobile: +61 (0)40 246 55 88 Cnr Brookes & Heffernan Sts, E-mail: Justin.Deutsc..elevance.com.au Mitchell ACT 2911 Web: http://www.relevance.com.au
This archive was generated by hypermail 2.0.0 : Wed Oct 27 2004 - 23:01:13 EDT
