# HG changeset patch # User IRI # Date 1269409475 -3600 # Node ID 4aea9df5515d644ed3079953f25a94ae3bd492b5 # Parent 616b76cdb13a4ebbdab4216341ad2ae311917a94 Unit test and debug diff -r 616b76cdb13a -r 4aea9df5515d src/fr/iri/thd/sonyengine/core/DbEnv.java --- a/src/fr/iri/thd/sonyengine/core/DbEnv.java Tue Mar 23 18:51:44 2010 +0100 +++ b/src/fr/iri/thd/sonyengine/core/DbEnv.java Wed Mar 24 06:44:35 2010 +0100 @@ -18,14 +18,16 @@ // The setup() method opens the environment and store // for us. - public void setup(File envHome, boolean readOnly) + public void setup(File envHome, boolean readOnly, boolean transactional) throws DatabaseException { EnvironmentConfig myEnvConfig = new EnvironmentConfig(); StoreConfig storeConfig = new StoreConfig(); myEnvConfig.setReadOnly(readOnly); + myEnvConfig.setTransactional(transactional); storeConfig.setReadOnly(readOnly); + storeConfig.setTransactional(transactional); // If the environment is opened for write, then we want to be // able to create the environment and entity store if @@ -72,6 +74,11 @@ } } } + + public void sync() { + this.store.sync(); + this.env.sync(); + } } diff -r 616b76cdb13a -r 4aea9df5515d src/fr/iri/thd/sonyengine/core/MovieFragment.java --- a/src/fr/iri/thd/sonyengine/core/MovieFragment.java Tue Mar 23 18:51:44 2010 +0100 +++ b/src/fr/iri/thd/sonyengine/core/MovieFragment.java Wed Mar 24 06:44:35 2010 +0100 @@ -2,8 +2,12 @@ import com.sleepycat.persist.model.Entity; import com.sleepycat.persist.model.PrimaryKey; +import javax.xml.bind.annotation.XmlRootElement; + + @Entity +@XmlRootElement public class MovieFragment { public String getId() { diff -r 616b76cdb13a -r 4aea9df5515d src/fr/iri/thd/sonyengine/core/Tag.java --- a/src/fr/iri/thd/sonyengine/core/Tag.java Tue Mar 23 18:51:44 2010 +0100 +++ b/src/fr/iri/thd/sonyengine/core/Tag.java Wed Mar 24 06:44:35 2010 +0100 @@ -6,9 +6,11 @@ import com.sleepycat.persist.model.PrimaryKey; import com.sleepycat.persist.model.Relationship; import com.sleepycat.persist.model.SecondaryKey; +import javax.xml.bind.annotation.XmlRootElement; @Entity +@XmlRootElement public class Tag { @PrimaryKey @@ -24,6 +26,10 @@ public String getId() { return id; } + + public void setId(String id) { + this.id = id; + } public void setName(String name) { this.name = name; @@ -41,6 +47,12 @@ return segment; } + public Tag(String id, String name, String segment) { + this.setId(id); + this.setName(name); + this.setSegment(segment); + } + public Tag(String name, String segment) { this.setName(name); this.setSegment(segment); diff -r 616b76cdb13a -r 4aea9df5515d src/fr/iri/thd/sonyengine/test/TestEngine.java --- a/src/fr/iri/thd/sonyengine/test/TestEngine.java Tue Mar 23 18:51:44 2010 +0100 +++ b/src/fr/iri/thd/sonyengine/test/TestEngine.java Wed Mar 24 06:44:35 2010 +0100 @@ -1,17 +1,28 @@ package fr.iri.thd.sonyengine.test; import java.io.File; +import java.io.IOException; +import java.io.StringReader; +import java.util.ArrayList; +import java.util.List; import javax.ws.rs.core.MediaType; +import javax.xml.parsers.DocumentBuilderFactory; +import javax.xml.parsers.ParserConfigurationException; import org.junit.After; import org.junit.Assert; import org.junit.Before; import org.junit.Test; +import org.w3c.dom.Document; +import org.w3c.dom.NodeList; +import org.xml.sax.InputSource; +import org.xml.sax.SAXException; import com.sleepycat.persist.EntityCursor; import com.sun.jersey.api.client.Client; import com.sun.jersey.api.client.WebResource; +import com.sun.jersey.api.representation.Form; import fr.iri.thd.sonyengine.core.DataAccessor; import fr.iri.thd.sonyengine.core.DbEnv; @@ -109,7 +120,7 @@ engine.reTrain(); engine.showAllTagValues(); - Neighbor neighbor = engine.findMore("ref1", "enfant", .05f, true); + //Neighbor neighbor = engine.findMore("ref1", "enfant", .05f, true); while (neighbor != null) { System.out.println(neighbor.getMovieRef() + " distance: " @@ -190,38 +201,77 @@ public static final String DATABASE_PATH = "/Users/ymh/dev/tmp/db"; private DbEnv dbenv = null; - - + private File databasePathFile = null; + @Before public void setup() { String databasepath = System.getProperty("database_path"); if(databasepath == null) databasepath = DATABASE_PATH; - File databasePathFile = new File(databasepath); - dbenv = new DbEnv(); - dbenv.setup(databasePathFile, true); + databasePathFile = new File(databasepath); } @After public void teardown() { - dbenv.close(); + + if(dbenv != null) + dbenv.close(); } @Test public void testCreateMovieFragment() { - - DataAccessor da = new DataAccessor(this.dbenv.getEntityStore()); - + String movieName = "movie" + System.currentTimeMillis(); Client c = Client.create(); WebResource res = c.resource(BASE_URL + "segment/create/" + movieName); - String response = res.accept(MediaType.TEXT_PLAIN_TYPE).post(String.class); + String response = res.accept(MediaType.APPLICATION_XML_TYPE).post(String.class); + + Assert.assertNotNull(response); + + + dbenv = new DbEnv(); + dbenv.setup(databasePathFile, true, true); + + DataAccessor da = new DataAccessor(this.dbenv.getEntityStore()); + + + EntityCursor cursor = da.movieFragmentById.keys(); + + for (String key : cursor) { + Assert.assertNotNull(key); + } + + cursor.close(); + + Assert.assertTrue(da.movieFragmentById.contains(movieName)); + } + + @Test + public void testCreateMovieFragmentForm() { + + String movieName = "movie" + System.currentTimeMillis(); + + Client c = Client.create(); + + Form form = new Form(); + form.add("id", movieName); + + WebResource res = c.resource(BASE_URL + "segment/create"); + + String response = res.type(MediaType.APPLICATION_FORM_URLENCODED).post(String.class, form); + Assert.assertNotNull(response); + dbenv = new DbEnv(); + dbenv.setup(databasePathFile, true, true); + + DataAccessor da = new DataAccessor(this.dbenv.getEntityStore()); + + EntityCursor cursor = da.movieFragmentById.keys(); for (String key : cursor) { @@ -235,5 +285,258 @@ } + @Test + public void testCreateTag() { + + String movieName = "movie" + System.currentTimeMillis(); + String tagName = "tag" + System.currentTimeMillis();; + + Client c = Client.create(); + WebResource res = c.resource(BASE_URL + "segment/create/" + movieName); + String response = res.accept(MediaType.APPLICATION_JSON_TYPE).post(String.class); + + Assert.assertNotNull(response); + + res = c.resource(BASE_URL).path("tag").path("add").path(tagName).path(movieName); + response = res.accept(MediaType.APPLICATION_JSON_TYPE).post(String.class); + + Assert.assertNotNull(response); + + dbenv = new DbEnv(); + dbenv.setup(databasePathFile, true, true); + + DataAccessor da = new DataAccessor(this.dbenv.getEntityStore()); + + Assert.assertTrue(da.tagByName.contains(tagName)); + + } + + @Test + public void testCreateTagList() { + + String movieName = "movie" + System.currentTimeMillis(); + + List tagNames = new ArrayList(); + + for(int i=0; i< 10; i++) { + try { + Thread.sleep(1); + } catch (InterruptedException e) { + e.printStackTrace(); + } + tagNames.add("tag" + System.currentTimeMillis()); + } + + Client c = Client.create(); + WebResource res = c.resource(BASE_URL + "segment/create/" + movieName); + String response = res.accept(MediaType.APPLICATION_XML_TYPE).post(String.class); + + Assert.assertNotNull(response); + + String xml = ""; + for(String tagName: tagNames) { + xml += String.format("", tagName, movieName); + } + xml += ""; + + res = c.resource(BASE_URL).path("tag").path("add"); + Form form = new Form(); + form.add("xml", xml); + response = res.type(MediaType.APPLICATION_FORM_URLENCODED).accept(MediaType.APPLICATION_XML_TYPE).post(String.class, form); + + Assert.assertNotNull(response); + + dbenv = new DbEnv(); + dbenv.setup(databasePathFile, true, true); + + DataAccessor da = new DataAccessor(this.dbenv.getEntityStore()); + + for(String tagName: tagNames) { + Assert.assertTrue(da.tagByName.contains(tagName)); + } + + } + + @Test + public void testCreateSequenceList() { + + List movieIds = new ArrayList(); + + for(int i=0; i< 10; i++) { + try { + Thread.sleep(1); + } catch (InterruptedException e) { + e.printStackTrace(); + } + movieIds.add("movie" + System.currentTimeMillis()); + } + + String xml = ""; + for(String sequenceId: movieIds) { + xml += String.format("", sequenceId); + } + xml += ""; + + Client c = Client.create(); + WebResource res = c.resource(BASE_URL).path("segment").path("createall"); + Form form = new Form(); + form.add("xml", xml); + String response = res.type(MediaType.APPLICATION_FORM_URLENCODED).accept(MediaType.APPLICATION_XML_TYPE).post(String.class, form); + + Assert.assertNotNull(response); + + dbenv = new DbEnv(); + dbenv.setup(databasePathFile, true, true); + + DataAccessor da = new DataAccessor(this.dbenv.getEntityStore()); + + for(String sequenceId: movieIds) { + Assert.assertTrue(da.movieFragmentById.contains(sequenceId)); + } + + } + + + @Test + public void testReset() { + Client c = Client.create(); + WebResource res = c.resource(BASE_URL); + String response = res.path("engine").path("reset").accept(MediaType.TEXT_PLAIN).post(String.class); + + Assert.assertEquals(response, "ok"); + + dbenv = new DbEnv(); + dbenv.setup(databasePathFile, true, true); + + DataAccessor da = new DataAccessor(this.dbenv.getEntityStore()); + + Assert.assertEquals(0, da.movieFragmentById.count()); + Assert.assertEquals(0, da.tagById.count()); + Assert.assertEquals(0, da.tagByName.count()); + Assert.assertEquals(0, da.tagBySegment.count()); + + } + + @Test + public void testTrain() { + + Client c = Client.create(); + WebResource res = c.resource(BASE_URL); + String response = res.path("engine").path("reset").accept(MediaType.TEXT_PLAIN).post(String.class); + + String[] ids = new String[]{"ref1","ref2","ref3","ref4","ref5","ref6", "ref7"}; + + String xmlSegment = ""; + for(String sequenceId: ids) { + xmlSegment += String.format("", sequenceId); + } + xmlSegment += ""; + + c = Client.create(); + res = c.resource(BASE_URL).path("segment").path("createall"); + Form form = new Form(); + form.add("xml", xmlSegment); + response = res.type(MediaType.APPLICATION_FORM_URLENCODED).accept(MediaType.APPLICATION_XML_TYPE).post(String.class, form); + + Assert.assertNotNull(response); + + + ArrayList tags = new ArrayList(); + tags.add(new String[]{"crayon", "ref1"}); + tags.add(new String[]{"carte", "ref1"}); + tags.add(new String[]{"bateau", "ref1"}); + tags.add(new String[]{"voiture", "ref1"}); + tags.add(new String[]{"car", "ref1"}); + + tags.add(new String[]{"enfant", "ref2"}); + tags.add(new String[]{"famille", "ref2"}); + tags.add(new String[]{"violon", "ref2"}); + tags.add(new String[]{"maison", "ref2"}); + tags.add(new String[]{"hopital", "ref2"}); + + tags.add(new String[]{"enfant", "ref3"}); + tags.add(new String[]{"famille", "ref3"}); + tags.add(new String[]{"violon", "ref3"}); + tags.add(new String[]{"maison", "ref3"}); + tags.add(new String[]{"ordinateur", "ref3"}); + tags.add(new String[]{"fenetre", "ref3"}); + tags.add(new String[]{"voiture", "ref3"}); + tags.add(new String[]{"guitare", "ref3"}); + tags.add(new String[]{"carte", "ref3"}); + + tags.add(new String[]{"enfant", "ref4"}); + tags.add(new String[]{"famille", "ref4"}); + tags.add(new String[]{"violon", "ref4"}); + tags.add(new String[]{"bateau", "ref4"}); + tags.add(new String[]{"ordinateur", "ref4"}); + tags.add(new String[]{"fenetre", "ref4"}); + tags.add(new String[]{"stylo", "ref4"}); + tags.add(new String[]{"perroquet", "ref4"}); + tags.add(new String[]{"crayon", "ref4"}); + tags.add(new String[]{"tele", "ref4"}); + tags.add(new String[]{"bateau", "ref4"}); + + tags.add(new String[]{"ordinateur", "ref5"}); + tags.add(new String[]{"crayon", "ref5"}); + tags.add(new String[]{"fenetre", "ref5"}); + tags.add(new String[]{"hopital", "ref5"}); + tags.add(new String[]{"carte", "ref5"}); + tags.add(new String[]{"tele", "ref5"}); + tags.add(new String[]{"stylo", "ref5"}); + tags.add(new String[]{"perroquet", "ref5"}); + tags.add(new String[]{"bateau", "ref5"}); + tags.add(new String[]{"guitare", "ref5"}); + + tags.add(new String[]{"maison", "ref6"}); + tags.add(new String[]{"perroquet", "ref6"}); + tags.add(new String[]{"stylo", "ref6"}); + tags.add(new String[]{"guitare", "ref6"}); + tags.add(new String[]{"hopital", "ref6"}); + tags.add(new String[]{"tele", "ref6"}); + tags.add(new String[]{"car", "ref6"}); + tags.add(new String[]{"voiture", "ref6"}); + tags.add(new String[]{"fenetre", "ref6"}); + + tags.add(new String[]{"car", "ref7"}); + tags.add(new String[]{"voiture", "ref7"}); + + String xmlTags = ""; + for(String[] tagdef: tags) { + xmlTags += String.format("", (Object[])tagdef); + } + xmlTags += ""; + + res = c.resource(BASE_URL).path("tag").path("add"); + form = new Form(); + form.add("xml", xmlTags); + response = res.type(MediaType.APPLICATION_FORM_URLENCODED).accept(MediaType.APPLICATION_XML_TYPE).post(String.class, form); + + Assert.assertNotNull(response); + + //"engine/find/{segment}/{tag}/{percent}/{more}" + //Neighbor neighbor = engine.findMore("ref1", "enfant", .05f, true); + res = c.resource(BASE_URL).path("engine").path("find").path("ref1").path("enfant").path("0.05").path("true"); + response = res.accept(MediaType.APPLICATION_XML_TYPE).get(String.class); + Assert.assertNotNull(response); + + DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance(); + InputSource source = new InputSource(new StringReader(response)); + Document doc = null; + try { + doc = factory.newDocumentBuilder().parse(source); + } catch (SAXException e) { + e.printStackTrace(); + } catch (IOException e) { + e.printStackTrace(); + } catch (ParserConfigurationException e) { + e.printStackTrace(); + } + + NodeList nodes = doc.getElementsByTagName("neighbor"); + + Assert.assertEquals(1, nodes.getLength()); + + } + } diff -r 616b76cdb13a -r 4aea9df5515d src/fr/iri/thd/sonyengine/web/EngineResource.java --- a/src/fr/iri/thd/sonyengine/web/EngineResource.java Tue Mar 23 18:51:44 2010 +0100 +++ b/src/fr/iri/thd/sonyengine/web/EngineResource.java Wed Mar 24 06:44:35 2010 +0100 @@ -1,13 +1,17 @@ package fr.iri.thd.sonyengine.web; import java.io.StringWriter; +import java.util.ArrayList; +import java.util.List; import javax.servlet.ServletContext; import javax.ws.rs.GET; import javax.ws.rs.Path; import javax.ws.rs.POST; import javax.ws.rs.PathParam; +import javax.ws.rs.Produces; import javax.ws.rs.core.Context; +import javax.ws.rs.core.MediaType; import javax.xml.parsers.DocumentBuilderFactory; import javax.xml.parsers.ParserConfigurationException; import javax.xml.transform.OutputKeys; @@ -22,6 +26,7 @@ import org.w3c.dom.Element; import com.sleepycat.je.Transaction; +import com.sleepycat.persist.EntityCursor; import fr.iri.thd.sonyengine.core.DataAccessor; import fr.iri.thd.sonyengine.core.DbEnv; @@ -50,13 +55,14 @@ @GET @Path("find/{segment}/{tag}/{percent}/{more}") + @Produces(MediaType.APPLICATION_XML) public String find( @PathParam("segment") String segment, @PathParam("tag") String tag, @PathParam("percent") String percent, @PathParam("more") String more) { - float percentf = (float)Integer.parseInt(percent) / 100.0f; + float percentf = Float.parseFloat(percent); boolean moreb = Boolean.parseBoolean(more); Neighbor n = ThdEngine.getEngine().findMore(segment, tag, percentf, moreb); @@ -103,24 +109,47 @@ @POST @Path("reset") + @Produces(MediaType.TEXT_PLAIN) public String ResetDatabase(@Context ServletContext context) { DbEnv dbenv = (DbEnv)context.getAttribute(ServletContainer.DB_ENV_ATTRIBUTE); DataAccessor da = new DataAccessor(dbenv.getEntityStore()); - Transaction trans = dbenv.getEntityStore().getEnvironment().beginTransaction(null,null); + EntityCursor keyCursor = da.tagById.keys(); + List keysTag = new ArrayList(); + for (String key : keyCursor) { + keysTag.add(key); + } + keyCursor.close(); + + List keysMovie = new ArrayList(); - for (String key : da.tagById.keys()) { - da.tagById.delete(trans, key); + keyCursor = da.movieFragmentById.keys(); + + for (String key : keyCursor) { + keysMovie.add(key); } - for (String key : da.movieFragmentById.keys()) { - da.movieFragmentById.delete(trans, key); - } + keyCursor.close(); + + Transaction trans = dbenv.getEnv().beginTransaction(null,null); - trans.commit(); + for(String key : keysTag) { + da.tagById.delete(trans,key); + } + + //trans.commitSync(); + //trans = dbenv.getEnv().beginTransaction(null,null); - dbenv.getEntityStore().sync(); + for(String key : keysMovie) { + da.movieFragmentById.delete(trans,key); + } + trans.commitSync(); + + dbenv.sync(); + + ThdEngine.getEngine().clearAll(); + ThdEngine.getEngine().reTrain(); return "ok"; } diff -r 616b76cdb13a -r 4aea9df5515d src/fr/iri/thd/sonyengine/web/MovieFragmentResource.java --- a/src/fr/iri/thd/sonyengine/web/MovieFragmentResource.java Tue Mar 23 18:51:44 2010 +0100 +++ b/src/fr/iri/thd/sonyengine/web/MovieFragmentResource.java Wed Mar 24 06:44:35 2010 +0100 @@ -1,24 +1,39 @@ package fr.iri.thd.sonyengine.web; +import java.io.IOException; +import java.io.StringReader; +import java.util.ArrayList; +import java.util.List; + import javax.servlet.ServletContext; +import javax.ws.rs.Consumes; +import javax.ws.rs.FormParam; import javax.ws.rs.POST; import javax.ws.rs.Path; import javax.ws.rs.PathParam; +import javax.ws.rs.Produces; import javax.ws.rs.core.Context; +import javax.ws.rs.core.MediaType; +import javax.xml.parsers.DocumentBuilderFactory; +import javax.xml.parsers.ParserConfigurationException; +import org.w3c.dom.Document; +import org.w3c.dom.Element; +import org.w3c.dom.NodeList; +import org.xml.sax.InputSource; +import org.xml.sax.SAXException; + +import thd.ThdEngine; import fr.iri.thd.sonyengine.core.DataAccessor; import fr.iri.thd.sonyengine.core.DbEnv; import fr.iri.thd.sonyengine.core.MovieFragment; -import thd.ThdEngine; - @Path ("segment") public class MovieFragmentResource { - @POST - @Path("create/{id}") - public String create(@PathParam("id") String id, @Context ServletContext context) { - + + private MovieFragment create(String id, ServletContext context) { + ThdEngine.getEngine().createMovieFragment(id); DbEnv dbenv = (DbEnv)context.getAttribute(ServletContainer.DB_ENV_ATTRIBUTE); @@ -27,14 +42,73 @@ MovieFragment fragment = new MovieFragment(id); da.movieFragmentById.put(fragment); - dbenv.getEntityStore().sync(); - + dbenv.sync(); ThdEngine.getEngine().reTrain(); - return "ok"; + return fragment; + } + + @POST + @Path("create/{id}") + @Produces({MediaType.APPLICATION_XML, MediaType.APPLICATION_JSON}) + public MovieFragment createPath(@PathParam("id") String id, @Context ServletContext context) { + + return this.create(id, context); } + + @POST + @Path("create") + @Produces({MediaType.APPLICATION_XML, MediaType.APPLICATION_JSON}) + @Consumes(MediaType.APPLICATION_FORM_URLENCODED) + public MovieFragment createForm(@FormParam("id") String id, @Context ServletContext context) { + return this.create(id, context); + } + + @POST + @Path("createall") + @Produces({MediaType.APPLICATION_XML, MediaType.APPLICATION_JSON}) + @Consumes(MediaType.APPLICATION_FORM_URLENCODED) + public List createAll(@FormParam("xml") String xmldocstr, @Context ServletContext context) { + + DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance(); + InputSource source = new InputSource(new StringReader(xmldocstr)); + Document doc = null; + try { + doc = factory.newDocumentBuilder().parse(source); + } catch (SAXException e) { + e.printStackTrace(); + } catch (IOException e) { + e.printStackTrace(); + } catch (ParserConfigurationException e) { + e.printStackTrace(); + } + DbEnv dbenv = (DbEnv)context.getAttribute(ServletContainer.DB_ENV_ATTRIBUTE); + DataAccessor da = new DataAccessor(dbenv.getEntityStore()); + + + NodeList tags = doc.getElementsByTagName("segment"); + + List list = new ArrayList(); + + for(int i = 0; i add(@FormParam("xml") String xmldocstr, @Context ServletContext context) { DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance(); InputSource source = new InputSource(new StringReader(xmldocstr)); @@ -70,6 +78,9 @@ NodeList tags = doc.getElementsByTagName("tag"); + + List list = new ArrayList(); + for(int i = 0; i