From 5be5fd3daa0d50980fb3791e904e035cdbca254f Mon Sep 17 00:00:00 2001 From: RncLsn Date: Thu, 7 May 2015 19:26:24 +0100 Subject: Making the output machine-readable (JSON). --- src/uk/ac/ox/cs/pagoda/query/AnswerTuple.java | 41 +++-- src/uk/ac/ox/cs/pagoda/query/QueryRecord.java | 191 +++++++++++++++------ src/uk/ac/ox/cs/pagoda/reasoner/QueryReasoner.java | 67 ++++---- src/uk/ac/ox/cs/pagoda/util/Utility.java | 39 +---- 4 files changed, 213 insertions(+), 125 deletions(-) (limited to 'src/uk/ac/ox/cs/pagoda') diff --git a/src/uk/ac/ox/cs/pagoda/query/AnswerTuple.java b/src/uk/ac/ox/cs/pagoda/query/AnswerTuple.java index 8d7e0b1..183cbe1 100644 --- a/src/uk/ac/ox/cs/pagoda/query/AnswerTuple.java +++ b/src/uk/ac/ox/cs/pagoda/query/AnswerTuple.java @@ -1,13 +1,10 @@ package uk.ac.ox.cs.pagoda.query; -import java.util.HashMap; -import java.util.Map; - +import com.google.gson.*; import org.semanticweb.HermiT.model.Constant; import org.semanticweb.HermiT.model.Individual; import org.semanticweb.HermiT.model.Term; import org.semanticweb.HermiT.model.Variable; - import uk.ac.ox.cs.JRDFox.JRDFStoreException; import uk.ac.ox.cs.JRDFox.model.Datatype; import uk.ac.ox.cs.JRDFox.model.GroundTerm; @@ -15,6 +12,10 @@ import uk.ac.ox.cs.JRDFox.model.Literal; import uk.ac.ox.cs.JRDFox.store.TupleIterator; import uk.ac.ox.cs.pagoda.util.Namespace; +import java.lang.reflect.Type; +import java.util.HashMap; +import java.util.Map; + public class AnswerTuple { public static final String SEPARATOR = "\t"; @@ -40,17 +41,21 @@ public class AnswerTuple { m_tuple = new GroundTerm[arity]; for (int i = 0; i < arity; ++i) m_tuple[i] = sup.m_tuple[i]; } + + private AnswerTuple(String m_str) { + this.m_str = m_str; + } public int getArity() { return m_tuple.length; } public int hashCode() { -// return toString().hashCode(); - int code = 0; - for (int i = 0; i < m_tuple.length; ++i) - code = code * 1997 + m_tuple[i].hashCode(); - return code; + return toString().hashCode(); +// int code = 0; +// for (int i = 0; i < m_tuple.length; ++i) +// code = code * 1997 + m_tuple[i].hashCode(); +// return code; } public boolean equals(Object obj) { @@ -131,5 +136,21 @@ public class AnswerTuple { if (length == extendedTuple.getArity()) return extendedTuple; else return new AnswerTuple(extendedTuple, length); } - + + + public static class AnswerTupleSerializer implements JsonSerializer { + + public JsonElement serialize(AnswerTuple src, Type typeOfSrc, JsonSerializationContext context) { + return new JsonPrimitive(src.toString()); + } + + } + + public class AnswerTupleDeserializer implements JsonDeserializer { + public AnswerTuple deserialize(JsonElement json, Type typeOfT, JsonDeserializationContext context) + throws JsonParseException { + return new AnswerTuple(json.getAsJsonPrimitive().getAsString()); + } + } + } diff --git a/src/uk/ac/ox/cs/pagoda/query/QueryRecord.java b/src/uk/ac/ox/cs/pagoda/query/QueryRecord.java index ce92a67..4b55046 100644 --- a/src/uk/ac/ox/cs/pagoda/query/QueryRecord.java +++ b/src/uk/ac/ox/cs/pagoda/query/QueryRecord.java @@ -1,37 +1,9 @@ package uk.ac.ox.cs.pagoda.query; -import java.io.BufferedWriter; -import java.io.FileNotFoundException; -import java.io.FileOutputStream; -import java.io.IOException; -import java.io.OutputStreamWriter; -import java.util.Collection; -import java.util.HashMap; -import java.util.HashSet; -import java.util.Iterator; -import java.util.LinkedList; -import java.util.Map; -import java.util.Set; - -import org.semanticweb.HermiT.model.Atom; -import org.semanticweb.HermiT.model.AtomicConcept; -import org.semanticweb.HermiT.model.AtomicRole; -import org.semanticweb.HermiT.model.DLClause; -import org.semanticweb.HermiT.model.DLPredicate; -import org.semanticweb.HermiT.model.Variable; -import org.semanticweb.owlapi.model.OWLAxiom; -import org.semanticweb.owlapi.model.OWLClass; -import org.semanticweb.owlapi.model.OWLClassAssertionAxiom; -import org.semanticweb.owlapi.model.OWLDataProperty; -import org.semanticweb.owlapi.model.OWLDataPropertyAssertionAxiom; -import org.semanticweb.owlapi.model.OWLIndividual; -import org.semanticweb.owlapi.model.OWLLiteral; -import org.semanticweb.owlapi.model.OWLObjectProperty; -import org.semanticweb.owlapi.model.OWLObjectPropertyAssertionAxiom; -import org.semanticweb.owlapi.model.OWLOntology; -import org.semanticweb.owlapi.model.OWLOntologyManager; -import org.semanticweb.owlapi.model.OWLOntologyStorageException; - +import com.google.gson.*; +import com.google.gson.reflect.TypeToken; +import org.semanticweb.HermiT.model.*; +import org.semanticweb.owlapi.model.*; import uk.ac.ox.cs.pagoda.hermit.DLClauseHelper; import uk.ac.ox.cs.pagoda.reasoner.light.RDFoxAnswerTuples; import uk.ac.ox.cs.pagoda.rules.GeneralProgram; @@ -39,6 +11,10 @@ import uk.ac.ox.cs.pagoda.util.ConjunctiveQueryHelper; import uk.ac.ox.cs.pagoda.util.Namespace; import uk.ac.ox.cs.pagoda.util.Utility; +import java.io.*; +import java.lang.reflect.Type; +import java.util.*; + public class QueryRecord { public static final String botQueryText = "SELECT ?X WHERE { ?X }"; @@ -47,11 +23,12 @@ public class QueryRecord { private String queryText; private int queryID = -1; - private String[][] answerVariables = null; - private Set soundAnswerTuples = new HashSet(); + + private String[][] answerVariables = null; + private Set soundAnswerTuples = new HashSet(); private Set gapAnswerTuples = null; - private QueryManager m_manager; + private QueryManager m_manager; public QueryRecord(QueryManager manager, String text, int id, int subID) { m_manager =manager; @@ -178,7 +155,7 @@ public class QueryRecord { // str.contains("internal:def"); // } - boolean processed = false; + boolean processed = false; public void markAsProcessed() { processed = true; @@ -205,7 +182,7 @@ public class QueryRecord { return queryText; } - String stringQueryID = null; + String stringQueryID = null; public String getQueryID() { return stringQueryID; @@ -233,27 +210,27 @@ public class QueryRecord { writer.newLine(); writer.write(queryText); writer.newLine(); - StringBuilder space = new StringBuilder(); - int arity = getArity(), varSpace = 0; + StringBuilder space = new StringBuilder(); + int arity = getArity(), varSpace = 0; for (int i = 0; i < arity; ++i) - varSpace += answerVariables[0][i].length(); + varSpace += answerVariables[0][i].length(); for (int i = 0; i < (SEPARATOR.length() - varSpace) / (arity + 1); ++i) - space.append(" "); + space.append(" "); for (int i = 0; i < getArity(); ++i) { writer.write(space.toString()); writer.write(answerVariables[0][i]); } writer.newLine(); writer.write(SEPARATOR); - writer.newLine(); + writer.newLine(); for (AnswerTuple tuple: soundAnswerTuples) { - writer.write(tuple.toString()); + writer.write(tuple.toString()); writer.newLine(); } if (!processed()) for (AnswerTuple tuple: gapAnswerTuples) { writer.write("*"); - writer.write(tuple.toString()); + writer.write(tuple.toString()); writer.newLine(); } // writer.write(SEPARATOR); @@ -261,6 +238,38 @@ public class QueryRecord { } } + + public void outputAnswerStatistics() { + + int answerCounter = soundAnswerTuples.size(); + if (!processed()) answerCounter += gapAnswerTuples.size(); + + Utility.logInfo("The number of answer tuples: " + answerCounter); +// if (jsonAnswers != null) { +// JSONObject jsonAnswer = new JSONObject(); +// +// jsonAnswer.put("queryID", queryID); +// jsonAnswer.put("queryText", queryText); +// +// JSONArray answerVars = new JSONArray(); +// int arity = getArity(), varSpace = 0; +// for (int i = 0; i < getArity(); i++) +// answerVars.add(answerVariables[0][i]); +// jsonAnswer.put("answerVars", answerVars); +// +// JSONArray answerTuples = new JSONArray(); +// soundAnswerTuples.stream().forEach(t -> answerTuples.add(t)); +// jsonAnswer.put("answerTuples", answerTuples); +// +// if (!processed) { +// JSONArray gapAnswerTuples = new JSONArray(); +// gapAnswerTuples.stream().forEach(t -> gapAnswerTuples.add(t)); +// } +// jsonAnswer.put("gapAnswerTuples", gapAnswerTuples); +// +// jsonAnswers.put(Integer.toString(queryID), jsonAnswer); +// } + } public void outputTimes() { for (Step step: Step.values()) { @@ -290,8 +299,8 @@ public class QueryRecord { return diffculty; } - OWLOntology relevantOntology = null; - Set relevantClauses = new HashSet(); + OWLOntology relevantOntology = null; + Set relevantClauses = new HashSet(); public void setRelevantOntology(OWLOntology knowledgebase) { relevantOntology = knowledgebase; @@ -355,7 +364,7 @@ public class QueryRecord { public enum Step {LowerBound, UpperBound, ELLowerBound, Fragment, FragmentRefinement, Summarisation, Dependency, FullReasoning}; - double[] timer; + double[] timer; public void addProcessingTime(Step step, double time) { timer[step.ordinal()] += time; @@ -431,14 +440,14 @@ public class QueryRecord { } } - int subID; + int subID; public void updateSubID() { ++subID; stringQueryID = String.valueOf(queryID) + "_" + subID; } - DLClause queryClause = null; + DLClause queryClause = null; public DLClause getClause() { if (queryClause != null) @@ -557,6 +566,88 @@ public class QueryRecord { public boolean hasNonAnsDistinguishedVariables() { return answerVariables[1].length > answerVariables[0].length; } - + public static class QueryRecordSerializer implements JsonSerializer { + + public JsonElement serialize(QueryRecord src, Type typeOfSrc, JsonSerializationContext context) { + Gson gson = new GsonBuilder().setPrettyPrinting().create(); + JsonObject object = new JsonObject(); + object.addProperty("queryID", src.queryID); + object.addProperty("queryText", src.queryText); + object.addProperty("difficulty", src.diffculty.toString()); + + object.add("answerVariables", context.serialize(src.getAnswerVariables())); + object.add("answers", context.serialize(src.soundAnswerTuples)); + object.add("gapAnswers", context.serialize(src.gapAnswerTuples)); + + return object; + } + } + + private QueryRecord() { } + + public class QueryRecordDeserializer implements JsonDeserializer { + + public QueryRecord deserialize(JsonElement json, Type typeOfT, JsonDeserializationContext context) + throws JsonParseException { + + QueryRecord record = new QueryRecord(); + JsonObject object = json.getAsJsonObject(); + record.queryID = object.getAsJsonPrimitive("queryID").getAsInt(); + record.queryText = object.getAsJsonPrimitive("queryText").getAsString(); + + JsonArray answerVariablesJson = object.getAsJsonArray("answerVariables"); + record.answerVariables = new String[2][]; + record.answerVariables[0] = new String[answerVariablesJson.size()]; + for(int i = 0; i < answerVariablesJson.size(); i++) + record.answerVariables[0][i] = answerVariablesJson.get(i).getAsString(); + + record.soundAnswerTuples = context.deserialize(object.getAsJsonObject("answers"), + new TypeToken>() {}.getType()); + + record.gapAnswerTuples = context.deserialize(object.getAsJsonObject("gapAnswers"), + new TypeToken>() {}.getType()); + return null; + } + } + + /* + * Two QueryRecords are equal iff + * they have the same queryText and + * their AnswerTuples have the same string representation + * */ + @Override + public boolean equals(Object o) { + if(!o.getClass().equals(getClass())) return false; + QueryRecord that = (QueryRecord) o; + + if(!this.queryText.equals(that.queryText)) return false; + + if(soundAnswerTuples.size() != that.soundAnswerTuples.size()) return false; + if(gapAnswerTuples.size() != that.gapAnswerTuples.size()) return false; + + ArrayList thisSoundAnswers = new ArrayList<>(soundAnswerTuples); + Collections.sort(thisSoundAnswers, (AnswerTuple t1, AnswerTuple t2) -> t1.m_str.compareTo(t2.m_str)); + + ArrayList thatSoundAnswers = new ArrayList<>(that.soundAnswerTuples); + Collections.sort(thatSoundAnswers, (AnswerTuple t1, AnswerTuple t2) -> t1.m_str.compareTo(t2.m_str)); + + Iterator soundIt1 = this.soundAnswerTuples.iterator(); + Iterator soundIt2 = that.soundAnswerTuples.iterator(); + while(soundIt1.hasNext() && soundIt2.hasNext()) + if(!soundIt1.next().m_str.equals(soundIt2.next().m_str)) return false; + + ArrayList thisGapAnswers = new ArrayList<>(gapAnswerTuples); + Collections.sort(thisGapAnswers, (AnswerTuple t1, AnswerTuple t2) -> t1.m_str.compareTo(t2.m_str)); + + ArrayList thatGapAnswers = new ArrayList<>(that.gapAnswerTuples); + Collections.sort(thatGapAnswers, (AnswerTuple t1, AnswerTuple t2) -> t1.m_str.compareTo(t2.m_str)); + + Iterator gapIt1 = this.gapAnswerTuples.iterator(); + Iterator gapIt2 = that.gapAnswerTuples.iterator(); + while(gapIt1.hasNext() && gapIt2.hasNext()) + if(!gapIt1.next().m_str.equals(gapIt2.next().m_str)) return false; + + return true; + } } diff --git a/src/uk/ac/ox/cs/pagoda/reasoner/QueryReasoner.java b/src/uk/ac/ox/cs/pagoda/reasoner/QueryReasoner.java index f2b7251..1f08fdf 100644 --- a/src/uk/ac/ox/cs/pagoda/reasoner/QueryReasoner.java +++ b/src/uk/ac/ox/cs/pagoda/reasoner/QueryReasoner.java @@ -1,16 +1,10 @@ package uk.ac.ox.cs.pagoda.reasoner; -import java.io.BufferedWriter; -import java.io.File; -import java.io.FileNotFoundException; -import java.io.FileOutputStream; -import java.io.IOException; -import java.io.OutputStreamWriter; -import java.util.Collection; - +import com.google.gson.Gson; +import com.google.gson.GsonBuilder; import org.semanticweb.owlapi.model.OWLOntology; - import uk.ac.ox.cs.pagoda.owl.OWLHelper; +import uk.ac.ox.cs.pagoda.query.AnswerTuple; import uk.ac.ox.cs.pagoda.query.AnswerTuples; import uk.ac.ox.cs.pagoda.query.QueryManager; import uk.ac.ox.cs.pagoda.query.QueryRecord; @@ -18,6 +12,13 @@ import uk.ac.ox.cs.pagoda.util.Properties; import uk.ac.ox.cs.pagoda.util.Timer; import uk.ac.ox.cs.pagoda.util.Utility; +import java.io.BufferedWriter; +import java.io.File; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Paths; +import java.util.Collection; + public abstract class QueryReasoner { // protected boolean forSemFacet = false; @@ -179,28 +180,33 @@ public abstract class QueryReasoner { } +// public void evaluate(Collection queryRecords) { +// evaluate(queryRecords); +// } + + BufferedWriter answerWriter = null; + public void evaluate(Collection queryRecords) { - evaluate(queryRecords, null); - } - - BufferedWriter answerWriter = null; - - public void evaluate(Collection queryRecords, String answerFile) { if (!isConsistent()) { Utility.logDebug("The ontology and dataset is inconsistent."); return ; } - - if (answerWriter == null && answerFile != null) { + + if(properties.getAnswerPath() != null && answerWriter == null) { try { - answerWriter = new BufferedWriter(new OutputStreamWriter(new FileOutputStream(answerFile))); - } catch (FileNotFoundException e) { - Utility.logInfo("The answer file not found! " + answerFile); - return ; - } + answerWriter = Files.newBufferedWriter(Paths.get(properties.getAnswerPath())); + } catch (IOException e) { + Utility.logError("The answer path is not valid!"); + e.printStackTrace(); + } } - Timer t = new Timer(); + Timer t = new Timer(); + Gson gson = new GsonBuilder() + .registerTypeAdapter(AnswerTuple.class, new AnswerTuple.AnswerTupleSerializer()) + .registerTypeAdapter(QueryRecord.class, new QueryRecord.QueryRecordSerializer()) + .setPrettyPrinting() + .create(); for (QueryRecord record: queryRecords) { // if (Integer.parseInt(record.getQueryID()) != 218) continue; Utility.logInfo("---------- start evaluating Query " + record.getQueryID() + " ----------", @@ -215,24 +221,23 @@ public abstract class QueryReasoner { continue; } } - // FIXME: change the argument below - try { - record.outputAnswers(answerWriter); - } catch (IOException e) { - Utility.logInfo("Error in outputing answers " + answerFile); - } + record.outputAnswerStatistics(); record.outputTimes(); - record.dispose(); } + // TODO it can handle one call only + // if you call twice, you will end up with a json file with multiple roots + gson.toJson(queryRecords, answerWriter); + queryRecords.stream().forEach(record -> record.dispose()); } public void dispose() { - if (answerWriter != null) + if (answerWriter != null) { try { answerWriter.close(); } catch (IOException e) { e.printStackTrace(); } + } Utility.cleanup(); } diff --git a/src/uk/ac/ox/cs/pagoda/util/Utility.java b/src/uk/ac/ox/cs/pagoda/util/Utility.java index 7138ca9..6d50ee0 100644 --- a/src/uk/ac/ox/cs/pagoda/util/Utility.java +++ b/src/uk/ac/ox/cs/pagoda/util/Utility.java @@ -1,32 +1,13 @@ package uk.ac.ox.cs.pagoda.util; -import java.io.BufferedReader; -import java.io.BufferedWriter; -import java.io.File; -import java.io.FileInputStream; -import java.io.FileNotFoundException; -import java.io.FileOutputStream; -import java.io.IOException; -import java.io.InputStreamReader; -import java.io.OutputStreamWriter; -import java.io.PrintStream; -import java.nio.file.Files; -import java.nio.file.Paths; -import java.text.SimpleDateFormat; -import java.time.LocalDateTime; -import java.time.format.DateTimeFormatter; -import java.util.Collection; -import java.util.Date; -import java.util.HashSet; -import java.util.LinkedList; -import java.util.Scanner; -import java.util.Set; -import java.util.Stack; - import org.apache.log4j.Logger; import org.semanticweb.HermiT.model.Atom; -import static java.nio.file.StandardCopyOption.REPLACE_EXISTING; +import java.io.*; +import java.text.SimpleDateFormat; +import java.time.LocalDateTime; +import java.time.format.DateTimeFormatter; +import java.util.*; public class Utility { @@ -259,14 +240,4 @@ public class Utility { return iri.replace(FILE_SEPARATOR, JAVA_FILE_SEPARATOR).replace(" ", "%20"); } - public static String combinePaths(String path1, String path2) { - File file1 = new File(path1); - File file2 = new File(file1, path2); - return file2.getPath(); - } - - public static void copyFile(String src, String dst) throws IOException { - Files.copy(Paths.get(src), Paths.get(dst), REPLACE_EXISTING); - } - } -- cgit v1.2.3