From 22b810c99170b69869e5c6c30fae6d3d62faf5de Mon Sep 17 00:00:00 2001 From: Knut Forkalsrud Date: Sat, 8 May 2010 15:22:05 -0700 Subject: [PATCH] Instead of using cache.properties to keep the parsed directory contents, put it in the database. --- photos/portraits/album.properties | 4 +- .../album/db/DirectoryDatabase.java | 118 ++++++++++++++++++ .../forkalsrud/album/db/DirectoryProps.java | 23 ++++ .../album/db/ThumbnailDatabase.java | 12 +- .../forkalsrud/album/exif/DirectoryEntry.java | 41 +++--- .../forkalsrud/album/web/AlbumServlet.java | 56 ++++----- .../forkalsrud/album/web/PictureScaler.java | 4 + src/main/webapp/WEB-INF/velocity.properties | 1 + 8 files changed, 196 insertions(+), 63 deletions(-) create mode 100644 src/main/java/org/forkalsrud/album/db/DirectoryDatabase.java create mode 100644 src/main/java/org/forkalsrud/album/db/DirectoryProps.java diff --git a/photos/portraits/album.properties b/photos/portraits/album.properties index 428415f..5b15fbd 100644 --- a/photos/portraits/album.properties +++ b/photos/portraits/album.properties @@ -1,4 +1,2 @@ - -#cover=valdemar-dahl.jpg cover=anton-rudi.jpg -caption=Old Portraits. 1953-05-02, Kirsten Høystad, Anne Marie (søskenbarn av Erna), Erna, ukjent, ukjent, Sigrid Dokk, ukjent. De ukjente er barndomsvenninner av Erna fra Oslo. +caption=Old Portraits. 1953-05-02, Kirsten H\u00f8ystad, Anne Marie (s\u00f8skenbarn av Erna), Erna, ukjent, ukjent, Sigrid Dokk, ukjent. De ukjente er barndomsvenninner av Erna fra Oslo. diff --git a/src/main/java/org/forkalsrud/album/db/DirectoryDatabase.java b/src/main/java/org/forkalsrud/album/db/DirectoryDatabase.java new file mode 100644 index 0000000..af631d8 --- /dev/null +++ b/src/main/java/org/forkalsrud/album/db/DirectoryDatabase.java @@ -0,0 +1,118 @@ +/** + * + */ +package org.forkalsrud.album.db; + +import java.io.IOException; +import java.io.StringReader; +import java.io.StringWriter; +import java.nio.charset.Charset; + +import com.sleepycat.bind.tuple.TupleBinding; +import com.sleepycat.bind.tuple.TupleInput; +import com.sleepycat.bind.tuple.TupleOutput; +import com.sleepycat.je.Database; +import com.sleepycat.je.DatabaseConfig; +import com.sleepycat.je.DatabaseEntry; +import com.sleepycat.je.Environment; +import com.sleepycat.je.OperationStatus; +import com.sleepycat.je.Transaction; + +/** + * @author knut + * + */ +public class DirectoryDatabase extends TupleBinding { + + private static Charset UTF8 = Charset.forName("utf-8"); + + private Environment environment; + private Database db; + + public DirectoryDatabase(Environment environment) { + + this.environment = environment; + + String dbname = "directories"; + + DatabaseConfig databaseConfig = new DatabaseConfig(); + databaseConfig.setAllowCreate(true); + databaseConfig.setTransactional(true); + // perform other database configurations + this.db = environment.openDatabase(null, dbname, databaseConfig); + } + + public void destroy() { + db.close(); + } + + public long size() { + return db.count(); + } + + public DirectoryProps load(String key) { + + DatabaseEntry data = new DatabaseEntry(); + Transaction txn = environment.beginTransaction(null, null); + OperationStatus status = db.get(txn, key(key), data, null); + txn.commitNoSync(); + if (OperationStatus.SUCCESS.equals(status)) { + return entryToObject(data); + } else { + return null; + } + } + + public void store(String key, DirectoryProps img) { + + DatabaseEntry data = new DatabaseEntry(); + objectToEntry(img, data); + DatabaseEntry binKey = key(key); + Transaction txn = environment.beginTransaction(null, null); + db.delete(txn, binKey); + db.put(txn, binKey, data); + txn.commitSync(); + } + + + private DatabaseEntry key(String key) { + DatabaseEntry returnValue = new DatabaseEntry(); + returnValue.setData(key.getBytes(UTF8)); + return returnValue; + } + + + @Override + public DirectoryProps entryToObject(TupleInput in) { + + DirectoryProps props = new DirectoryProps(); + int version = in.readInt(); + if (version != 1) { + throw new RuntimeException("I only understand version 1"); + } + props.setTimestamp(in.readLong()); + String val = in.readString(); + try { + props.load(new StringReader(val)); + } catch (IOException e) { + e.printStackTrace(); + } + return props; + } + + + @Override + public void objectToEntry(DirectoryProps props, TupleOutput out) { + + out.writeInt(1); // version 1 + out.writeLong(props.getTimestamp()); + StringWriter writer = new StringWriter(); + try { + props.store(writer, ""); + } catch (IOException e) { + e.printStackTrace(); + } + out.writeString(writer.toString()); + } + +} diff --git a/src/main/java/org/forkalsrud/album/db/DirectoryProps.java b/src/main/java/org/forkalsrud/album/db/DirectoryProps.java new file mode 100644 index 0000000..57de7b8 --- /dev/null +++ b/src/main/java/org/forkalsrud/album/db/DirectoryProps.java @@ -0,0 +1,23 @@ +/** + * + */ +package org.forkalsrud.album.db; + +import java.util.Properties; + +/** + * @author knut + * + */ +public class DirectoryProps extends Properties { + + private long timestamp; + + public long getTimestamp() { + return timestamp; + } + + public void setTimestamp(long timestamp) { + this.timestamp = timestamp; + } +} diff --git a/src/main/java/org/forkalsrud/album/db/ThumbnailDatabase.java b/src/main/java/org/forkalsrud/album/db/ThumbnailDatabase.java index 18f5e61..ae04c70 100644 --- a/src/main/java/org/forkalsrud/album/db/ThumbnailDatabase.java +++ b/src/main/java/org/forkalsrud/album/db/ThumbnailDatabase.java @@ -3,7 +3,6 @@ */ package org.forkalsrud.album.db; -import java.io.File; import java.nio.charset.Charset; import org.forkalsrud.album.web.CachedImage; @@ -15,7 +14,6 @@ import com.sleepycat.je.Database; import com.sleepycat.je.DatabaseConfig; import com.sleepycat.je.DatabaseEntry; import com.sleepycat.je.Environment; -import com.sleepycat.je.EnvironmentConfig; import com.sleepycat.je.OperationStatus; import com.sleepycat.je.Transaction; @@ -30,15 +28,12 @@ public class ThumbnailDatabase extends TupleBinding { private Environment environment; private Database db; - public void init(File dir) { + public ThumbnailDatabase(Environment environment) { + + this.environment = environment; String dbname = "thumbnails"; - EnvironmentConfig environmentConfig = new EnvironmentConfig(); - environmentConfig.setAllowCreate(true); - environmentConfig.setTransactional(true); - // perform other environment configurations - environment = new Environment(dir, environmentConfig); DatabaseConfig databaseConfig = new DatabaseConfig(); databaseConfig.setAllowCreate(true); databaseConfig.setTransactional(true); @@ -48,7 +43,6 @@ public class ThumbnailDatabase extends TupleBinding { public void destroy() { db.close(); - environment.close(); } public long size() { diff --git a/src/main/java/org/forkalsrud/album/exif/DirectoryEntry.java b/src/main/java/org/forkalsrud/album/exif/DirectoryEntry.java index f4e6be4..c653ff2 100644 --- a/src/main/java/org/forkalsrud/album/exif/DirectoryEntry.java +++ b/src/main/java/org/forkalsrud/album/exif/DirectoryEntry.java @@ -27,6 +27,9 @@ import javax.imageio.ImageReadParam; import javax.imageio.ImageReader; import javax.imageio.stream.ImageInputStream; +import org.forkalsrud.album.db.DirectoryDatabase; +import org.forkalsrud.album.db.DirectoryProps; + import com.drew.imaging.jpeg.JpegMetadataReader; import com.drew.metadata.Directory; import com.drew.metadata.Metadata; @@ -42,31 +45,31 @@ public class DirectoryEntry extends Entry { final static String CACHE_FILE = "cache.properties"; final static String OVERRIDE_FILE = "album.properties"; - File cache; + DirectoryDatabase db; + DirectoryProps cache; List children = new ArrayList(); boolean childrenLoaded = false; Comparator sort = null; Date earliest = null; - public DirectoryEntry(File file) { + public DirectoryEntry(DirectoryDatabase db, File file) { super(file); if (!file.isDirectory()) { throw new RuntimeException("Use DirectoryEntry only for directories: " + file); } - cache = new File(file, CACHE_FILE); + this.db = db; + cache = db.load(file.getAbsolutePath()); + if (cache == null) { + cache = new DirectoryProps(); + } } - public DirectoryEntry(Entry parent, File file) { - this(file); + public DirectoryEntry(DirectoryDatabase db, Entry parent, File file) { + this(db, file); this.parent = parent; } - public static Entry getEntry(File f) { - return new DirectoryEntry(f.getParentFile()).get(f); - } - - @Override public boolean isFile() { return false; @@ -110,7 +113,7 @@ public class DirectoryEntry extends Entry { return; } try { - Properties cachedProps = isCacheCurrent() ? loadFromFile(cache) : generateCache(); + Properties cachedProps = isCacheCurrent() ? cache : generateCache(); Properties combined = new Properties(); combined.putAll(cachedProps); File override = new File(file, OVERRIDE_FILE); @@ -128,7 +131,7 @@ public class DirectoryEntry extends Entry { } boolean isCacheCurrent() { - return cache.exists() && cache.isFile() && cache.canRead() && file.lastModified() <= cache.lastModified(); + return file.lastModified() <= cache.getTimestamp(); } Properties loadFromFile(File propFile) throws IOException { @@ -143,19 +146,17 @@ public class DirectoryEntry extends Entry { Properties generateCache() throws IOException { long start = System.currentTimeMillis(); - Properties props = new Properties(); + DirectoryProps props = new DirectoryProps(); generateFileEntries(props); - FileOutputStream fos = new FileOutputStream(cache); - props.store(fos, "Extra Comments"); - fos.close(); long end = System.currentTimeMillis(); + props.setTimestamp(end); + db.store(file.getAbsolutePath(), props); System.out.println("Time to generate properties for " + file.getPath() + ": " + (end - start)/1000d + " s"); - return props; + return props; } - void generateFileEntries(Properties props) - throws IOException { + void generateFileEntries(Properties props) throws IOException { File[] files = file.listFiles(); for (File f : files) { @@ -353,7 +354,7 @@ public class DirectoryEntry extends Entry { String name = key.substring("dir.".length()); boolean hidden = Boolean.parseBoolean(props.getProperty("dir." + name + ".hidden")); if (!hidden) { - DirectoryEntry dir = new DirectoryEntry(this, new File(file, name)); + DirectoryEntry dir = new DirectoryEntry(db, this, new File(file, name)); children.add(dir); if (name != null && name.equals(coverFileName)) { setThumbnail(dir.getThumbnail()); diff --git a/src/main/java/org/forkalsrud/album/web/AlbumServlet.java b/src/main/java/org/forkalsrud/album/web/AlbumServlet.java index bfaa282..a9fd4e8 100644 --- a/src/main/java/org/forkalsrud/album/web/AlbumServlet.java +++ b/src/main/java/org/forkalsrud/album/web/AlbumServlet.java @@ -18,12 +18,16 @@ import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; import org.apache.log4j.PropertyConfigurator; +import org.forkalsrud.album.db.DirectoryDatabase; import org.forkalsrud.album.db.ThumbnailDatabase; import org.forkalsrud.album.exif.DirectoryEntry; import org.forkalsrud.album.exif.Entry; import org.forkalsrud.album.exif.FileEntry; import org.forkalsrud.album.exif.Thumbnail; +import com.sleepycat.je.Environment; +import com.sleepycat.je.EnvironmentConfig; + public class AlbumServlet extends HttpServlet { @@ -56,7 +60,6 @@ public class AlbumServlet } hh = ll.getHandlers(); System.out.println((Arrays.asList(hh))); -// new FileHandler("album.log", 5*1024*1024, 10, true)); } catch (Exception e) { e.printStackTrace(); } @@ -74,11 +77,11 @@ public class AlbumServlet File base; String basePrefix; -// Cache imageCache; -// CacheManager cacheManager; PictureScaler pictureScaler; long lastCacheFlushTime; - ThumbnailDatabase db = new ThumbnailDatabase(); + private Environment environment; + ThumbnailDatabase thumbDb; + DirectoryDatabase dirDb; @Override public void init() @@ -107,7 +110,15 @@ public class AlbumServlet String dbDirName = props.getProperty("dbdir"); File dbDir = dbDirName != null ? new File(dbDirName) : new File(System.getProperty("java.io.tmpdir"), "album"); dbDir.mkdirs(); - db.init(dbDir); + + EnvironmentConfig environmentConfig = new EnvironmentConfig(); + environmentConfig.setAllowCreate(true); + environmentConfig.setTransactional(true); + environment = new Environment(dbDir, environmentConfig); + + thumbDb = new ThumbnailDatabase(environment); + dirDb = new DirectoryDatabase(environment); + pictureScaler = new PictureScaler(); lastCacheFlushTime = System.currentTimeMillis(); } @@ -125,9 +136,9 @@ public class AlbumServlet @Override public void destroy() { System.out.println("Shutting down Album"); - db.destroy(); -// imageCache.flush(); - // cacheManager.shutdown(); + dirDb.destroy(); + thumbDb.destroy(); + environment.close(); } @Override @@ -169,7 +180,7 @@ public class AlbumServlet if (size != null) { try { - FileEntry e = (FileEntry)DirectoryEntry.getEntry(file); + FileEntry e = (FileEntry)resolve(file); procesScaledImageRequest(req, res, file, e.getThumbnail(), size); return; } catch (Exception e) { @@ -241,36 +252,19 @@ public class AlbumServlet String key = file.getPath() + ":" + size; - CachedImage cimg = db.load(key); -// Element element = imageCache.get(key); -// if (element != null) { -// cimg = (CachedImage) element.getObjectValue(); + CachedImage cimg = thumbDb.load(key); if (cimg != null) { if (cimg.lastModified == file.lastModified()) { System.out.println("cache hit on " + key); } else { System.out.println(" " + key + " has changed so cache entry wil be refreshed"); - // imageCache.remove(key); cimg = null; } } if (cimg == null) { -// try { - cimg = pictureScaler.scalePicture(file, thumbnail, size); - db.store(key, cimg); -// imageCache.put(new Element(key, cimg)); - /* - long millisSinceLastFlush = System.currentTimeMillis() - lastCacheFlushTime; - if (millisSinceLastFlush > 10 * 60 * 1000L) { - imageCache.flush(); - lastCacheFlushTime = System.currentTimeMillis(); - } - */ - System.out.println(" " + key + " added to the cache with size " + cimg.bits.length + " -- now " + db.size() + " entries"); -// } catch (TimeoutException e) { -// res.setStatus(HttpServletResponse.SC_SERVICE_UNAVAILABLE); -// return; -// } + cimg = pictureScaler.scalePicture(file, thumbnail, size); + thumbDb.store(key, cimg); + System.out.println(" " + key + " added to the cache with size " + cimg.bits.length + " -- now " + thumbDb.size() + " entries"); } res.setStatus(HttpServletResponse.SC_OK); res.setDateHeader("Last-Modified", file.lastModified()); @@ -293,7 +287,7 @@ public class AlbumServlet Entry resolve(File file) { if (base.equals(file.getAbsoluteFile())) { - return new DirectoryEntry(null, file); + return new DirectoryEntry(dirDb, null, file); } else { return ((DirectoryEntry)resolve(file.getParentFile())).get(file); } diff --git a/src/main/java/org/forkalsrud/album/web/PictureScaler.java b/src/main/java/org/forkalsrud/album/web/PictureScaler.java index 7fe7ae9..88c8424 100644 --- a/src/main/java/org/forkalsrud/album/web/PictureScaler.java +++ b/src/main/java/org/forkalsrud/album/web/PictureScaler.java @@ -124,6 +124,10 @@ public class PictureScaler { } catch (InterruptedException e) { e.printStackTrace(); return null; + } catch (OutOfMemoryError e) { + System.out.println(file); + e.printStackTrace(); + return null; } catch (ExecutionException e) { throw new RuntimeException(e); } finally { diff --git a/src/main/webapp/WEB-INF/velocity.properties b/src/main/webapp/WEB-INF/velocity.properties index 8392748..bee571b 100644 --- a/src/main/webapp/WEB-INF/velocity.properties +++ b/src/main/webapp/WEB-INF/velocity.properties @@ -3,3 +3,4 @@ velocimacro.permissions.allow.inline.to.replace.global=false velocimacro.permissions.allow.inline.local.scope=true velocimacro.context.localscope=false velocimacro.library.autoreload=true +output.encoding = UTF-8