From 167d068c7b48a941ef7ddbfa9aef70d0b175743a Mon Sep 17 00:00:00 2001 From: Knut Forkalsrud Date: Fri, 5 Feb 2010 21:16:10 -0800 Subject: [PATCH 1/3] Using BerkeleyDB for resized images. --- .gitignore | 2 + pom.xml | 19 ++++-- .../forkalsrud/album/web/AlbumServlet.java | 58 +++++++++++++------ .../forkalsrud/album/web/PictureScaler.java | 1 + webapp/WEB-INF/web.xml | 4 ++ 5 files changed, 59 insertions(+), 25 deletions(-) diff --git a/.gitignore b/.gitignore index 51eab7f..18aed1a 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,5 @@ target cache.properties build +db + diff --git a/pom.xml b/pom.xml index 9909a11..27e53e6 100644 --- a/pom.xml +++ b/pom.xml @@ -108,16 +108,17 @@ 2.4 provided - - net.sf.ehcache - ehcache - 1.5.0 - log4j log4j 1.2.14 + + com.sleepycat + je + 4.0.92 + + junitjunit3.8.2 @@ -125,8 +126,14 @@ forkalsrud.org maven proxy true false - http://www.forkalsrud.org/maven-proxy/repository + http://forkalsrud.org:8080/maven-proxy/repository legacy + + oracleReleases + Oracle Released Java Packages + http://download.oracle.com/maven + default + diff --git a/src/org/forkalsrud/album/web/AlbumServlet.java b/src/org/forkalsrud/album/web/AlbumServlet.java index 83012cd..b96f9fd 100644 --- a/src/org/forkalsrud/album/web/AlbumServlet.java +++ b/src/org/forkalsrud/album/web/AlbumServlet.java @@ -4,6 +4,7 @@ import java.io.File; import java.io.IOException; import java.util.Calendar; import java.util.Date; +import java.util.Properties; import javax.servlet.RequestDispatcher; import javax.servlet.ServletException; @@ -11,12 +12,9 @@ import javax.servlet.http.HttpServlet; import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; -import net.sf.ehcache.Cache; -import net.sf.ehcache.CacheManager; -import net.sf.ehcache.Element; - import org.apache.commons.logging.LogFactory; import org.apache.log4j.PropertyConfigurator; +import org.forkalsrud.album.db.ThumbnailDatabase; import org.forkalsrud.album.exif.DirectoryEntry; import org.forkalsrud.album.exif.Entry; import org.forkalsrud.album.exif.FileEntry; @@ -27,38 +25,56 @@ public class AlbumServlet { File base; String basePrefix; - Cache imageCache; - CacheManager cacheManager; +// Cache imageCache; +// CacheManager cacheManager; PictureScaler pictureScaler; long lastCacheFlushTime; + ThumbnailDatabase db = new ThumbnailDatabase(); @Override public void init() throws ServletException { + log4jInit("/log4j.properties"); System.out.println("in init of Album"); base = new File(getServletConfig().getInitParameter("base")).getAbsoluteFile(); basePrefix = "/" + base.getName(); - PropertyConfigurator.configure("log4j.properties"); LogFactory.getFactory().setAttribute("org.apache.commons.logging.Log", "org.apache.commons.logging.impl.Log4JLogger"); - cacheManager = CacheManager.create(); - imageCache = cacheManager.getCache("imageCache"); + + String dbDirName = getServletConfig().getInitParameter("dbdir"); + File dbDir = dbDirName != null ? new File(dbDirName) : new File(System.getProperty("java.io.tmpdir"), "album"); + dbDir.mkdir(); + db.init(dbDir); +// cacheManager = CacheManager.create(); +// imageCache = cacheManager.getCache("imageCache"); pictureScaler = new PictureScaler(); lastCacheFlushTime = System.currentTimeMillis(); } + private void log4jInit(String resource) { + try { + Properties props = new Properties(); + props.load(getClass().getResourceAsStream(resource)); + PropertyConfigurator.configure(props); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + @Override public void destroy() { - imageCache.flush(); - cacheManager.shutdown(); + System.out.println("Shutting down Album"); + db.destroy(); +// imageCache.flush(); + // cacheManager.shutdown(); } @Override public void doGet(HttpServletRequest req, HttpServletResponse res) throws ServletException, IOException { - if (req.getParameter("cacheFlush") != null) imageCache.flush(); +// if (req.getParameter("cacheFlush") != null) imageCache.flush(); req.setAttribute("assets", req.getContextPath() + "/assets"); req.setAttribute("req", req); @@ -166,28 +182,32 @@ public class AlbumServlet String key = file.getPath() + ":" + size; - CachedImage cimg = null; - Element element = imageCache.get(key); - if (element != null) { - cimg = (CachedImage) element.getObjectValue(); + CachedImage cimg = db.load(key); +// Element element = imageCache.get(key); +// if (element != null) { +// cimg = (CachedImage) element.getObjectValue(); + if (cimg != null) { if (cimg.lastModified == file.lastModified()) { System.out.println("cache hit on " + key); } else { System.out.println(" " + key + " has changed so cache entry wil be refreshed"); - imageCache.remove(key); + // imageCache.remove(key); cimg = null; } } if (cimg == null) { // try { cimg = pictureScaler.scalePicture(file, thumbnail, size); - imageCache.put(new Element(key, cimg)); + db.store(key, cimg); +// imageCache.put(new Element(key, cimg)); + /* long millisSinceLastFlush = System.currentTimeMillis() - lastCacheFlushTime; if (millisSinceLastFlush > 10 * 60 * 1000L) { imageCache.flush(); lastCacheFlushTime = System.currentTimeMillis(); } - System.out.println(" " + key + " added to the cache with size " + cimg.bits.length + " -- now " + imageCache.getSize() + " entries"); + */ + System.out.println(" " + key + " added to the cache with size " + cimg.bits.length + " -- now " + db.size() + " entries"); // } catch (TimeoutException e) { // res.setStatus(HttpServletResponse.SC_SERVICE_UNAVAILABLE); // return; diff --git a/src/org/forkalsrud/album/web/PictureScaler.java b/src/org/forkalsrud/album/web/PictureScaler.java index bf22fb5..0859888 100644 --- a/src/org/forkalsrud/album/web/PictureScaler.java +++ b/src/org/forkalsrud/album/web/PictureScaler.java @@ -171,6 +171,7 @@ public class PictureScaler { reader.setInput(iis, true); ImageReadParam param = reader.getDefaultReadParam(); BufferedImage img = reader.read(0, param); + iis.close(); // The orientation is about flipping and rotating. Here is what an 'F' looks like // on pictures with each orientation. diff --git a/webapp/WEB-INF/web.xml b/webapp/WEB-INF/web.xml index e3677c2..01424c2 100644 --- a/webapp/WEB-INF/web.xml +++ b/webapp/WEB-INF/web.xml @@ -19,6 +19,10 @@ base photos + + dbdir + db + From 9e840209cf31d0e49b37318292e4421d505629ea Mon Sep 17 00:00:00 2001 From: Knut Forkalsrud Date: Fri, 5 Feb 2010 21:20:24 -0800 Subject: [PATCH 2/3] using BerkeleyDB for thumbnail database --- .gitignore | 6 +- .../album/db/ThumbnailDatabase.java | 117 ++++++++++++++++++ 2 files changed, 120 insertions(+), 3 deletions(-) create mode 100644 src/org/forkalsrud/album/db/ThumbnailDatabase.java diff --git a/.gitignore b/.gitignore index 18aed1a..506d561 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,5 @@ -target +/target cache.properties -build -db +/build +/db diff --git a/src/org/forkalsrud/album/db/ThumbnailDatabase.java b/src/org/forkalsrud/album/db/ThumbnailDatabase.java new file mode 100644 index 0000000..18f5e61 --- /dev/null +++ b/src/org/forkalsrud/album/db/ThumbnailDatabase.java @@ -0,0 +1,117 @@ +/** + * + */ +package org.forkalsrud.album.db; + +import java.io.File; +import java.nio.charset.Charset; + +import org.forkalsrud.album.web.CachedImage; + +import com.sleepycat.bind.tuple.TupleBinding; +import com.sleepycat.bind.tuple.TupleInput; +import com.sleepycat.bind.tuple.TupleOutput; +import com.sleepycat.je.Database; +import com.sleepycat.je.DatabaseConfig; +import com.sleepycat.je.DatabaseEntry; +import com.sleepycat.je.Environment; +import com.sleepycat.je.EnvironmentConfig; +import com.sleepycat.je.OperationStatus; +import com.sleepycat.je.Transaction; + +/** + * @author knut + * + */ +public class ThumbnailDatabase extends TupleBinding { + + private static Charset UTF8 = Charset.forName("utf-8"); + + private Environment environment; + private Database db; + + public void init(File dir) { + + String dbname = "thumbnails"; + + EnvironmentConfig environmentConfig = new EnvironmentConfig(); + environmentConfig.setAllowCreate(true); + environmentConfig.setTransactional(true); + // perform other environment configurations + environment = new Environment(dir, environmentConfig); + DatabaseConfig databaseConfig = new DatabaseConfig(); + databaseConfig.setAllowCreate(true); + databaseConfig.setTransactional(true); + // perform other database configurations + this.db = environment.openDatabase(null, dbname, databaseConfig); + } + + public void destroy() { + db.close(); + environment.close(); + } + + public long size() { + return db.count(); + } + + public CachedImage load(String key) { + + DatabaseEntry data = new DatabaseEntry(); + Transaction txn = environment.beginTransaction(null, null); + OperationStatus status = db.get(txn, key(key), data, null); + txn.commitNoSync(); + if (OperationStatus.SUCCESS.equals(status)) { + return entryToObject(data); + } else { + return null; + } + } + + public void store(String key, CachedImage img) { + + DatabaseEntry data = new DatabaseEntry(); + objectToEntry(img, data); + DatabaseEntry binKey = key(key); + Transaction txn = environment.beginTransaction(null, null); + db.delete(txn, binKey); + db.put(txn, binKey, data); + txn.commitSync(); + } + + + private DatabaseEntry key(String key) { + DatabaseEntry returnValue = new DatabaseEntry(); + returnValue.setData(key.getBytes(UTF8)); + return returnValue; + } + + + @Override + public CachedImage entryToObject(TupleInput in) { + + CachedImage img = new CachedImage(); + int version = in.readInt(); + if (version != 1) { + throw new RuntimeException("I only understand version 1"); + } + img.lastModified = in.readLong(); + img.mimeType = in.readString(); + int lobLen = in.readInt(); + img.bits = new byte[lobLen]; + in.read(img.bits, 0, lobLen); + return img; + } + + + @Override + public void objectToEntry(CachedImage img, TupleOutput out) { + + out.writeInt(1); // version 1 + out.writeLong(img.lastModified); + out.writeString(img.mimeType); + out.writeInt(img.bits.length); + out.write(img.bits); + } + +} From 96c874404451fa8e45ca130b9e749a28467e4740 Mon Sep 17 00:00:00 2001 From: Knut Forkalsrud Date: Fri, 5 Feb 2010 21:29:08 -0800 Subject: [PATCH 3/3] no more commons-logging --- src/org/forkalsrud/album/web/AlbumServlet.java | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/org/forkalsrud/album/web/AlbumServlet.java b/src/org/forkalsrud/album/web/AlbumServlet.java index b96f9fd..b029e42 100644 --- a/src/org/forkalsrud/album/web/AlbumServlet.java +++ b/src/org/forkalsrud/album/web/AlbumServlet.java @@ -12,7 +12,6 @@ import javax.servlet.http.HttpServlet; import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; -import org.apache.commons.logging.LogFactory; import org.apache.log4j.PropertyConfigurator; import org.forkalsrud.album.db.ThumbnailDatabase; import org.forkalsrud.album.exif.DirectoryEntry; @@ -40,8 +39,6 @@ public class AlbumServlet base = new File(getServletConfig().getInitParameter("base")).getAbsoluteFile(); basePrefix = "/" + base.getName(); - LogFactory.getFactory().setAttribute("org.apache.commons.logging.Log", "org.apache.commons.logging.impl.Log4JLogger"); - String dbDirName = getServletConfig().getInitParameter("dbdir"); File dbDir = dbDirName != null ? new File(dbDirName) : new File(System.getProperty("java.io.tmpdir"), "album"); dbDir.mkdir();