Instead of using cache.properties to keep the parsed directory contents, put it in the database.

photos/portraits/album.properties

@@ -1,4 +1,2 @@
-
-#cover=valdemar-dahl.jpg
 cover=anton-rudi.jpg
-caption=Old Portraits. 1953-05-02, Kirsten Høystad, Anne Marie (søskenbarn av Erna), Erna, ukjent, ukjent, Sigrid Dokk, ukjent. De ukjente er barndomsvenninner av Erna fra Oslo.
+caption=Old Portraits. 1953-05-02, Kirsten H\u00f8ystad, Anne Marie (s\u00f8skenbarn av Erna), Erna, ukjent, ukjent, Sigrid Dokk, ukjent. De ukjente er barndomsvenninner av Erna fra Oslo.

src/main/java/org/forkalsrud/album/db/DirectoryDatabase.java

@@ -0,0 +1,118 @@
+/**
+ * 
+ */
+package org.forkalsrud.album.db;
+
+import java.io.IOException;
+import java.io.StringReader;
+import java.io.StringWriter;
+import java.nio.charset.Charset;
+
+import com.sleepycat.bind.tuple.TupleBinding;
+import com.sleepycat.bind.tuple.TupleInput;
+import com.sleepycat.bind.tuple.TupleOutput;
+import com.sleepycat.je.Database;
+import com.sleepycat.je.DatabaseConfig;
+import com.sleepycat.je.DatabaseEntry;
+import com.sleepycat.je.Environment;
+import com.sleepycat.je.OperationStatus;
+import com.sleepycat.je.Transaction;
+
+/**
+ * @author knut
+ * 
+ */
+public class DirectoryDatabase extends TupleBinding<DirectoryProps> {
+
+    private static Charset UTF8 = Charset.forName("utf-8");
+    
+    private Environment environment;
+    private Database db;
+
+    public DirectoryDatabase(Environment environment) {
+
+        this.environment = environment;
+
+        String dbname = "directories";
+
+        DatabaseConfig databaseConfig = new DatabaseConfig();
+        databaseConfig.setAllowCreate(true);
+        databaseConfig.setTransactional(true);
+        // perform other database configurations
+        this.db = environment.openDatabase(null, dbname, databaseConfig);
+    }
+
+    public void destroy() {
+        db.close();
+    }
+    
+    public long size() {
+        return db.count();
+    }
+
+    public DirectoryProps load(String key) {
+
+        DatabaseEntry data = new DatabaseEntry();
+        Transaction txn = environment.beginTransaction(null, null);
+        OperationStatus status = db.get(txn, key(key), data, null);
+        txn.commitNoSync();
+        if (OperationStatus.SUCCESS.equals(status)) {
+            return entryToObject(data);
+        } else {
+            return null;
+        }
+    }
+
+    public void store(String key, DirectoryProps img) {
+
+        DatabaseEntry data = new DatabaseEntry();
+        objectToEntry(img, data);
+        DatabaseEntry binKey = key(key);
+        Transaction txn = environment.beginTransaction(null, null);
+        db.delete(txn, binKey);
+        db.put(txn, binKey, data);
+        txn.commitSync();
+    }
+
+
+    private DatabaseEntry key(String key) {
+        DatabaseEntry returnValue = new DatabaseEntry();
+        returnValue.setData(key.getBytes(UTF8));
+        return returnValue;
+    }
+
+
+    @Override
+    public DirectoryProps entryToObject(TupleInput in) {
+
+        DirectoryProps props = new DirectoryProps();
+        int version = in.readInt();
+        if (version != 1) {
+            throw new RuntimeException("I only understand version 1");
+        }
+        props.setTimestamp(in.readLong());
+        String val = in.readString();
+        try {
+            props.load(new StringReader(val));
+        } catch (IOException e) {
+            e.printStackTrace();
+        }
+        return props;
+    }
+
+
+    @Override
+    public void objectToEntry(DirectoryProps props, TupleOutput out) {
+
+        out.writeInt(1);  // version 1
+        out.writeLong(props.getTimestamp());
+        StringWriter writer = new StringWriter();
+        try {
+            props.store(writer, "");
+        } catch (IOException e) {
+            e.printStackTrace();
+        }
+        out.writeString(writer.toString());
+    }
+
+}

src/main/java/org/forkalsrud/album/db/DirectoryProps.java

@@ -0,0 +1,23 @@
+/**
+ * 
+ */
+package org.forkalsrud.album.db;
+
+import java.util.Properties;
+
+/**
+ * @author knut
+ *
+ */
+public class DirectoryProps extends Properties {
+
+    private long timestamp;
+
+    public long getTimestamp() {
+        return timestamp;
+    }
+
+    public void setTimestamp(long timestamp) {
+        this.timestamp = timestamp;
+    }
+}

src/main/java/org/forkalsrud/album/db/ThumbnailDatabase.java

@@ -3,7 +3,6 @@
  */
 package org.forkalsrud.album.db;
 
-import java.io.File;
 import java.nio.charset.Charset;
 
 import org.forkalsrud.album.web.CachedImage;
@@ -15,7 +14,6 @@ import com.sleepycat.je.Database;
 import com.sleepycat.je.DatabaseConfig;
 import com.sleepycat.je.DatabaseEntry;
 import com.sleepycat.je.Environment;
-import com.sleepycat.je.EnvironmentConfig;
 import com.sleepycat.je.OperationStatus;
 import com.sleepycat.je.Transaction;
 
@@ -30,15 +28,12 @@ public class ThumbnailDatabase extends TupleBinding<CachedImage> {
     private Environment environment;
     private Database db;
 
-    public void init(File dir) {
+    public ThumbnailDatabase(Environment environment) {
+
+        this.environment = environment;
 
         String dbname = "thumbnails";
 
-        EnvironmentConfig environmentConfig = new EnvironmentConfig();
-        environmentConfig.setAllowCreate(true);
-        environmentConfig.setTransactional(true);
-        // perform other environment configurations
-        environment = new Environment(dir, environmentConfig);
         DatabaseConfig databaseConfig = new DatabaseConfig();
         databaseConfig.setAllowCreate(true);
         databaseConfig.setTransactional(true);
@@ -48,7 +43,6 @@ public class ThumbnailDatabase extends TupleBinding<CachedImage> {
 
     public void destroy() {
         db.close();
-        environment.close();
     }
     
     public long size() {

src/main/java/org/forkalsrud/album/exif/DirectoryEntry.java

@@ -27,6 +27,9 @@ import javax.imageio.ImageReadParam;
 import javax.imageio.ImageReader;
 import javax.imageio.stream.ImageInputStream;
 
+import org.forkalsrud.album.db.DirectoryDatabase;
+import org.forkalsrud.album.db.DirectoryProps;
+
 import com.drew.imaging.jpeg.JpegMetadataReader;
 import com.drew.metadata.Directory;
 import com.drew.metadata.Metadata;
@@ -42,31 +45,31 @@ public class DirectoryEntry extends Entry {
     final static String CACHE_FILE = "cache.properties";
     final static String OVERRIDE_FILE = "album.properties";
 
-    File cache;
+    DirectoryDatabase db;
+    DirectoryProps cache;
     List<Entry> children = new ArrayList<Entry>();
     boolean childrenLoaded = false;
     Comparator<Entry> sort = null;
     Date earliest = null;
 
-    public DirectoryEntry(File file) {
+    public DirectoryEntry(DirectoryDatabase db, File file) {
         super(file);
         if (!file.isDirectory()) {
             throw new RuntimeException("Use DirectoryEntry only for directories: " + file);
         }
-        cache = new File(file, CACHE_FILE);
+        this.db = db;
+        cache = db.load(file.getAbsolutePath());
+        if (cache == null) {
+            cache = new DirectoryProps();
+        }
     }
 
-    public DirectoryEntry(Entry parent, File file) {
-        this(file);
+    public DirectoryEntry(DirectoryDatabase db, Entry parent, File file) {
+        this(db, file);
         this.parent = parent;
     }
 
 
-    public static Entry getEntry(File f) {
-        return new DirectoryEntry(f.getParentFile()).get(f);
-    }
-
-
     @Override
     public boolean isFile() {
         return false;
@@ -110,7 +113,7 @@ public class DirectoryEntry extends Entry {
             return;
         }
         try {
-            Properties cachedProps = isCacheCurrent() ? loadFromFile(cache) : generateCache();
+            Properties cachedProps = isCacheCurrent() ? cache : generateCache();
             Properties combined = new Properties();
             combined.putAll(cachedProps);
             File override = new File(file, OVERRIDE_FILE);
@@ -128,7 +131,7 @@ public class DirectoryEntry extends Entry {
     }
 
     boolean isCacheCurrent() {
-        return cache.exists() && cache.isFile() && cache.canRead() && file.lastModified() <= cache.lastModified();
+        return file.lastModified() <= cache.getTimestamp();
     }
 
     Properties loadFromFile(File propFile) throws IOException {
@@ -143,19 +146,17 @@ public class DirectoryEntry extends Entry {
     Properties generateCache() throws IOException {
 
         long start = System.currentTimeMillis();
-        Properties props = new Properties();
+        DirectoryProps props = new DirectoryProps();
         generateFileEntries(props);
-        FileOutputStream fos = new FileOutputStream(cache);
-        props.store(fos, "Extra Comments");
-        fos.close();
         long end = System.currentTimeMillis();
+        props.setTimestamp(end);
+        db.store(file.getAbsolutePath(), props);
         System.out.println("Time to generate properties for " + file.getPath() + ": " + (end - start)/1000d + " s");
-        return props;    
+        return props;
     }
 
 
-    void generateFileEntries(Properties props)
-    throws IOException {
+    void generateFileEntries(Properties props) throws IOException {
 
         File[] files = file.listFiles();
         for (File f : files) {
@@ -353,7 +354,7 @@ public class DirectoryEntry extends Entry {
                 String name = key.substring("dir.".length());
                 boolean hidden = Boolean.parseBoolean(props.getProperty("dir." + name + ".hidden"));
                 if (!hidden) {
-                    DirectoryEntry dir = new DirectoryEntry(this, new File(file, name));
+                    DirectoryEntry dir = new DirectoryEntry(db, this, new File(file, name));
                     children.add(dir);
                     if (name != null && name.equals(coverFileName)) {
                         setThumbnail(dir.getThumbnail());

src/main/java/org/forkalsrud/album/web/AlbumServlet.java

@@ -18,12 +18,16 @@ import javax.servlet.http.HttpServletRequest;
 import javax.servlet.http.HttpServletResponse;
 
 import org.apache.log4j.PropertyConfigurator;
+import org.forkalsrud.album.db.DirectoryDatabase;
 import org.forkalsrud.album.db.ThumbnailDatabase;
 import org.forkalsrud.album.exif.DirectoryEntry;
 import org.forkalsrud.album.exif.Entry;
 import org.forkalsrud.album.exif.FileEntry;
 import org.forkalsrud.album.exif.Thumbnail;
 
+import com.sleepycat.je.Environment;
+import com.sleepycat.je.EnvironmentConfig;
+
 public class AlbumServlet
     extends HttpServlet
 {
@@ -56,7 +60,6 @@ public class AlbumServlet
             }
             hh = ll.getHandlers();
             System.out.println((Arrays.asList(hh)));
-//                    new FileHandler("album.log", 5*1024*1024, 10, true));
         } catch (Exception e) {
             e.printStackTrace();
         }
@@ -74,11 +77,11 @@ public class AlbumServlet
     
     File base;
 	String basePrefix;
-//	Cache imageCache;
-//	CacheManager cacheManager;
     PictureScaler pictureScaler;
     long lastCacheFlushTime;
-    ThumbnailDatabase db = new ThumbnailDatabase();
+    private Environment environment;
+    ThumbnailDatabase thumbDb;
+    DirectoryDatabase dirDb;
 
     @Override
     public void init()
@@ -107,7 +110,15 @@ public class AlbumServlet
         String dbDirName = props.getProperty("dbdir");
         File dbDir = dbDirName != null ? new File(dbDirName) : new File(System.getProperty("java.io.tmpdir"), "album");
         dbDir.mkdirs();
-        db.init(dbDir);
+
+        EnvironmentConfig environmentConfig = new EnvironmentConfig();
+        environmentConfig.setAllowCreate(true);
+        environmentConfig.setTransactional(true);
+        environment = new Environment(dbDir, environmentConfig);
+
+        thumbDb = new ThumbnailDatabase(environment);
+        dirDb = new DirectoryDatabase(environment);
+
         pictureScaler = new PictureScaler();
         lastCacheFlushTime = System.currentTimeMillis();
     }
@@ -125,9 +136,9 @@ public class AlbumServlet
     @Override
     public void destroy() {
         System.out.println("Shutting down Album");
-        db.destroy();
-//    	imageCache.flush();
- //   	cacheManager.shutdown();
+        dirDb.destroy();
+        thumbDb.destroy();
+        environment.close();
     }
 
     @Override
@@ -169,7 +180,7 @@ public class AlbumServlet
         if (size != null) {
 
             try {
-                FileEntry e = (FileEntry)DirectoryEntry.getEntry(file);
+                FileEntry e = (FileEntry)resolve(file);
                 procesScaledImageRequest(req, res, file, e.getThumbnail(), size);
                 return;
             } catch (Exception e) {
@@ -241,36 +252,19 @@ public class AlbumServlet
 
         String key = file.getPath() + ":" + size;
 
-        CachedImage cimg = db.load(key);
-//        Element element = imageCache.get(key);
-//        if (element != null) {
-//            cimg = (CachedImage) element.getObjectValue();
+        CachedImage cimg = thumbDb.load(key);
         if (cimg != null) {
             if (cimg.lastModified == file.lastModified()) {
                 System.out.println("cache hit on " + key);
             } else {
                 System.out.println("             " + key + " has changed so cache entry wil be refreshed");
-                // imageCache.remove(key);
                 cimg = null;
             }
         }
         if (cimg == null) {
-//        	try {
-        		cimg = pictureScaler.scalePicture(file, thumbnail, size);
-        		db.store(key, cimg);
-//        		imageCache.put(new Element(key, cimg));
-        		/*
-        		long millisSinceLastFlush = System.currentTimeMillis() - lastCacheFlushTime;
-        		if (millisSinceLastFlush > 10 * 60 * 1000L) {
-        		    imageCache.flush();
-        		    lastCacheFlushTime = System.currentTimeMillis();
-        		}
-        		*/
-        		System.out.println("             " + key + " added to the cache with size " + cimg.bits.length + " -- now " + db.size() + " entries");
-//        	} catch (TimeoutException e) {
-//        		res.setStatus(HttpServletResponse.SC_SERVICE_UNAVAILABLE);
-//        		return;
-//        	}
+            cimg = pictureScaler.scalePicture(file, thumbnail, size);
+            thumbDb.store(key, cimg);
+            System.out.println("             " + key + " added to the cache with size " + cimg.bits.length + " -- now " + thumbDb.size() + " entries");
         }
         res.setStatus(HttpServletResponse.SC_OK);
         res.setDateHeader("Last-Modified", file.lastModified());
@@ -293,7 +287,7 @@ public class AlbumServlet
     Entry resolve(File file) {
 
     	if (base.equals(file.getAbsoluteFile())) {
-    		return new DirectoryEntry(null, file);
+    		return new DirectoryEntry(dirDb, null, file);
     	} else {
     		return ((DirectoryEntry)resolve(file.getParentFile())).get(file);
     	}

src/main/java/org/forkalsrud/album/web/PictureScaler.java

@@ -124,6 +124,10 @@ public class PictureScaler {
     	} catch (InterruptedException e) {
     		e.printStackTrace();
     		return null;
+        } catch (OutOfMemoryError e) {
+            System.out.println(file);
+            e.printStackTrace();
+            return null;
     	} catch (ExecutionException e) {
     		throw new RuntimeException(e);
     	} finally {

src/main/webapp/WEB-INF/velocity.properties

@@ -3,3 +3,4 @@ velocimacro.permissions.allow.inline.to.replace.global=false
 velocimacro.permissions.allow.inline.local.scope=true 
 velocimacro.context.localscope=false
 velocimacro.library.autoreload=true
+output.encoding = UTF-8