From 90cb6032058ad3fc16b895922823b5a700121b1b Mon Sep 17 00:00:00 2001 From: RncLsn Date: Mon, 24 Aug 2015 15:54:05 +0100 Subject: Incremental Skolemised store (seems completed). --- src/resources/pagoda.properties | 6 ++- .../pagoda/multistage/MultiStageQueryEngine.java | 37 ++++++++++------ .../ac/ox/cs/pagoda/reasoner/MyQueryReasoner.java | 50 +++++++++++++++++++--- .../cs/pagoda/reasoner/light/BasicQueryEngine.java | 7 +++ src/uk/ac/ox/cs/pagoda/util/PagodaProperties.java | 12 ++++++ 5 files changed, 93 insertions(+), 19 deletions(-) (limited to 'src') diff --git a/src/resources/pagoda.properties b/src/resources/pagoda.properties index 34b3d7a..aa08593 100644 --- a/src/resources/pagoda.properties +++ b/src/resources/pagoda.properties @@ -2,8 +2,12 @@ debug=true useAlwaysSimpleUpperBound=false #skolemUpperBound=DISABLED #skolemUpperBound=BEFORE_SUMMARISATION -skolemUpperBound=AFTER_SUMMARISATION +skolemUpperBound=BEFORE_SUMMARISATION skolemDepth=10 + +# seems ok for -Xmx6g +maxTriplesInSkolemStore=2500000 + toCallHermit=true statisticsDir=/home/alessandro/Dropbox/Oxford/PAGOdA/statistics \ No newline at end of file diff --git a/src/uk/ac/ox/cs/pagoda/multistage/MultiStageQueryEngine.java b/src/uk/ac/ox/cs/pagoda/multistage/MultiStageQueryEngine.java index f3a78f6..209e1d7 100644 --- a/src/uk/ac/ox/cs/pagoda/multistage/MultiStageQueryEngine.java +++ b/src/uk/ac/ox/cs/pagoda/multistage/MultiStageQueryEngine.java @@ -35,7 +35,9 @@ public class MultiStageQueryEngine extends StageQueryEngine { private HashMap statistics = new HashMap<>(); private Set> oversizedSkolemisedFacts; private RDFoxTripleManager rdFoxTripleManager; + private int lastMaxTermDepth = -1; + private boolean firstCall = true; public MultiStageQueryEngine(String name, boolean checkValidity) { super(name, checkValidity); @@ -79,11 +81,12 @@ public class MultiStageQueryEngine extends StageQueryEngine { public int materialiseSkolemly(DatalogProgram dProgram, GapByStore4ID gap, int maxTermDepth) { if(isDisposed()) throw new DisposedException(); - if(maxTermDepth <= lastMaxTermDepth) + if(!firstCall && maxTermDepth <= lastMaxTermDepth) throw new IllegalArgumentException("maxTermDepth must be greater than " + lastMaxTermDepth); + if(firstCall) + materialise("lower program", dProgram.getLower().toString()); lastMaxTermDepth = maxTermDepth; - materialise("lower program", dProgram.getLower().toString()); Program generalProgram = dProgram.getGeneral(); LimitedSkolemisationApplication program = new LimitedSkolemisationApplication(generalProgram, @@ -91,7 +94,9 @@ public class MultiStageQueryEngine extends StageQueryEngine { maxTermDepth); rdFoxTripleManager = new RDFoxTripleManager(store, true); Treatment treatment = new Pick4NegativeConceptNaive(this, program, rdFoxTripleManager); - return materialise(program, treatment, gap, maxTermDepth); + int result = materialise(program, treatment, gap, maxTermDepth); + firstCall = false; + return result; } public int materialise4SpecificQuery(Program generalProgram, QueryRecord record, BottomStrategy upperBottom) { @@ -117,7 +122,11 @@ public class MultiStageQueryEngine extends StageQueryEngine { } private int materialise(MultiStageUpperProgram program, Treatment treatment, GapByStore4ID gap, int maxTermDepth) { - boolean actuallyCleaned = cleanStoreFromOversizedSkolemisedFacts(); + if(!firstCall) + cleanStoreFromOversizedSkolemisedFacts(); + + boolean isSkolemising = maxTermDepth > 0; +// boolean isSkolemising = true; if(gap != null) treatment.addAdditionalGapTuples(); @@ -157,11 +166,11 @@ public class MultiStageQueryEngine extends StageQueryEngine { } } else { - if(!incrementally) { + if(!incrementally && firstCall) { // store.addRules(new String[] {datalogProgram}); store.importRules(datalogProgram); } - store.applyReasoning(incrementally || actuallyCleaned); + store.applyReasoning(incrementally || !firstCall); } // Utility.logInfo("The number of sameAs assertions in the current store: " + getSameAsNumber()); @@ -181,15 +190,17 @@ public class MultiStageQueryEngine extends StageQueryEngine { Utility.logDebug("Time to materialise datalog-rules: " + subTimer.duration()); subTimer.reset(); - if((violations = program.isIntegrated(this, incrementally)) == null || violations.size() == 0) { - store.clearRulesAndMakeFactsExplicit(); + if((violations = program.isIntegrated(this, !isSkolemising && incrementally)) == null || violations.size() == 0) { + if(!isSkolemising) + store.clearRulesAndMakeFactsExplicit(); Utility.logDebug(name + " store after materialising " + programName + ": " + tripleCount + " (" + (tripleCount - tripleCountBeforeMat) + " new)"); Utility.logDebug(name + " store is DONE for multi-stage materialising in " + t.duration() + " seconds."); return isValid() ? 1 : 0; } Utility.logDebug("Time to detect violations: " + subTimer.duration()); -// store.makeFactsExplicit(); + if(!isSkolemising) + store.makeFactsExplicit(); subTimer.reset(); oldTripleCount = store.getTriplesCount(); @@ -274,8 +285,8 @@ public class MultiStageQueryEngine extends StageQueryEngine { result.add(new TupleBuilder().append(idTriple[0]).append(idTriple[1]) .append(idTriple[2]).build()); } - else if(!(atom.getArgument(0) instanceof Individual)) - throw new IllegalArgumentException("No individuals: " + atom); +// else if(!(atom.getArgument(0) instanceof Individual)) +// throw new IllegalArgumentException("No individuals: " + atom); } else { if((atom.getArgument(0) instanceof Individual && termsManager.getDepthOf((Individual) atom.getArgument(0)) >= maxDepth) @@ -284,8 +295,8 @@ public class MultiStageQueryEngine extends StageQueryEngine { result.add(new TupleBuilder().append(idTriple[0]).append(idTriple[1]) .append(idTriple[2]).build()); } - else if(!(atom.getArgument(0) instanceof Individual) && !(atom.getArgument(1) instanceof Individual)) - throw new IllegalArgumentException("No individuals: " + atom); +// else if(!(atom.getArgument(0) instanceof Individual) && !(atom.getArgument(1) instanceof Individual)) +// throw new IllegalArgumentException("No individuals: " + atom); } } diff --git a/src/uk/ac/ox/cs/pagoda/reasoner/MyQueryReasoner.java b/src/uk/ac/ox/cs/pagoda/reasoner/MyQueryReasoner.java index a393474..8a90a26 100644 --- a/src/uk/ac/ox/cs/pagoda/reasoner/MyQueryReasoner.java +++ b/src/uk/ac/ox/cs/pagoda/reasoner/MyQueryReasoner.java @@ -2,6 +2,7 @@ package uk.ac.ox.cs.pagoda.reasoner; import org.semanticweb.karma2.profile.ELHOProfile; import org.semanticweb.owlapi.model.OWLOntology; +import uk.ac.ox.cs.JRDFox.JRDFStoreException; import uk.ac.ox.cs.pagoda.multistage.MultiStageQueryEngine; import uk.ac.ox.cs.pagoda.owl.EqualitiesEliminator; import uk.ac.ox.cs.pagoda.owl.OWLHelper; @@ -18,6 +19,7 @@ import uk.ac.ox.cs.pagoda.tracking.QueryTracker; import uk.ac.ox.cs.pagoda.tracking.TrackingRuleEncoder; import uk.ac.ox.cs.pagoda.tracking.TrackingRuleEncoderDisjVar1; import uk.ac.ox.cs.pagoda.tracking.TrackingRuleEncoderWithGap; +import uk.ac.ox.cs.pagoda.util.ExponentialInterpolation; import uk.ac.ox.cs.pagoda.util.PagodaProperties; import uk.ac.ox.cs.pagoda.util.Timer; import uk.ac.ox.cs.pagoda.util.Utility; @@ -25,6 +27,7 @@ import uk.ac.ox.cs.pagoda.util.disposable.DisposedException; import uk.ac.ox.cs.pagoda.util.tuples.Tuple; import java.util.Collection; +import java.util.LinkedList; class MyQueryReasoner extends QueryReasoner { @@ -399,8 +402,32 @@ class MyQueryReasoner extends QueryReasoner { relevantStore.materialise("Mark original individuals", relevantOriginalMarkProgram); boolean isFullyProcessed = false; - for (int currentMaxTermDepth = 1; - currentMaxTermDepth <= properties.getSkolemDepth() && !isFullyProcessed; currentMaxTermDepth++) { + LinkedList> lastTwoTriplesCounts = new LinkedList<>(); + for (int currentMaxTermDepth = 1; !isFullyProcessed; currentMaxTermDepth++) { + + if(currentMaxTermDepth > properties.getSkolemDepth()) { + Utility.logInfo("Maximum term depth reached"); + break; + } + + if(lastTwoTriplesCounts.size() == 2) { + if(lastTwoTriplesCounts.get(0).get(1).equals(lastTwoTriplesCounts.get(1).get(1))) + break; + + ExponentialInterpolation interpolation = new ExponentialInterpolation(lastTwoTriplesCounts.get(0).get(0), + lastTwoTriplesCounts.get(0).get(1), + lastTwoTriplesCounts.get(1).get(0), + lastTwoTriplesCounts.get(1).get(1)); + double triplesEstimate = interpolation.computeValue(currentMaxTermDepth); + + Utility.logDebug("Estimate of the number of triples:" + triplesEstimate); + + // exit condition if the query is not fully answered + if(triplesEstimate > properties.getMaxTriplesInSkolemStore()) { + Utility.logInfo("Interrupting Semi-Skolemisation because of triples count limit"); + break; + } + } Utility.logInfo("Trying with maximum depth " + currentMaxTermDepth); @@ -408,18 +435,31 @@ class MyQueryReasoner extends QueryReasoner { currentMaxTermDepth); queryRecord.addProcessingTime(Step.SKOLEM_UPPER_BOUND, t.duration()); if(materialisationTag == -1) { + relevantStore.dispose(); throw new Error("A consistent ontology has turned out to be " + "inconsistent in the Skolemises-relevant-upper-store"); } else if(materialisationTag != 1) { Utility.logInfo("Semi-Skolemised relevant upper store cannot be employed"); - return false; + break; } Utility.logInfo("Querying semi-Skolemised upper store..."); isFullyProcessed = queryUpperStore(relevantStore, queryRecord, - queryRecord.getExtendedQueryText(), - Step.SKOLEM_UPPER_BOUND); + queryRecord.getExtendedQueryText(), + Step.SKOLEM_UPPER_BOUND); + + try { + lastTwoTriplesCounts.add + (new Tuple<>((long) currentMaxTermDepth, relevantStore.getStoreSize())); + } catch (JRDFStoreException e) { + e.printStackTrace(); + break; + } + if(lastTwoTriplesCounts.size() > 2) + lastTwoTriplesCounts.remove(); + + Utility.logInfo("Last two triples counts:" + lastTwoTriplesCounts); } relevantStore.dispose(); diff --git a/src/uk/ac/ox/cs/pagoda/reasoner/light/BasicQueryEngine.java b/src/uk/ac/ox/cs/pagoda/reasoner/light/BasicQueryEngine.java index 107d3ca..034827e 100644 --- a/src/uk/ac/ox/cs/pagoda/reasoner/light/BasicQueryEngine.java +++ b/src/uk/ac/ox/cs/pagoda/reasoner/light/BasicQueryEngine.java @@ -32,6 +32,13 @@ public class BasicQueryEngine extends RDFoxQueryEngine { parameters.m_useBushy = true; } + /*** + * @return Overall number of triples. + */ + public long getStoreSize() throws JRDFStoreException { + return store.getTriplesCount(); + } + public void materialiseFoldedly(DatalogProgram dProgram, GapByStore4ID gap) { if(isDisposed()) throw new DisposedException(); if(gap != null) { diff --git a/src/uk/ac/ox/cs/pagoda/util/PagodaProperties.java b/src/uk/ac/ox/cs/pagoda/util/PagodaProperties.java index 18469ff..0f9ad4e 100644 --- a/src/uk/ac/ox/cs/pagoda/util/PagodaProperties.java +++ b/src/uk/ac/ox/cs/pagoda/util/PagodaProperties.java @@ -20,6 +20,7 @@ public class PagodaProperties { private static final int DEFAULT_SKOLEM_DEPTH; private static final boolean DEFAULT_TO_CALL_HERMIT; private static final Path DEFAULT_STATISTICS_DIR; + private static final long DEFAULT_MAX_TRIPLES_IN_SKOLEM_STORE; public static boolean shellModeDefault = false; private static boolean debug = DEFAULT_DEBUG; @@ -30,6 +31,7 @@ public class PagodaProperties { int defaultSkolemDepth = 1; boolean toCallHermit = true; Path defaultStatisticsDir = null; + long defaultMaxTriplesInSkolemStore = 1000000; try (InputStream in = PagodaProperties.class.getClassLoader().getResourceAsStream(CONFIG_FILE)) { Properties config = new Properties(); @@ -77,6 +79,10 @@ public class PagodaProperties { defaultSkolemDepth = Integer.parseInt(config.getProperty("skolemDepth")); logger.debug("By default the max skolemisation depth is " + defaultSkolemDepth); } + if (config.containsKey("maxTriplesInSkolemStore")) { + defaultMaxTriplesInSkolemStore = Long.parseLong(config.getProperty("maxTriplesInSkolemStore")); + logger.debug("By default the maximum number of triples in the Skolem store is " + defaultMaxTriplesInSkolemStore); + } } catch (IOException e) { e.printStackTrace(); @@ -86,6 +92,7 @@ public class PagodaProperties { DEFAULT_TO_CALL_HERMIT = toCallHermit; DEFAULT_STATISTICS_DIR = defaultStatisticsDir; DEFAULT_SKOLEM_DEPTH = defaultSkolemDepth; + DEFAULT_MAX_TRIPLES_IN_SKOLEM_STORE = defaultMaxTriplesInSkolemStore; } String dataPath = null; @@ -108,6 +115,7 @@ public class PagodaProperties { private boolean useAlwaysSimpleUpperBound = DEFAULT_USE_ALWAYS_SIMPLE_UPPER_BOUND; private SkolemUpperBoundOptions skolemUpperBound = DEFAULT_SKOLEM_UPPER_BOUND; private Path statisticsDir = DEFAULT_STATISTICS_DIR; + private long maxTriplesInSkolemStore = DEFAULT_MAX_TRIPLES_IN_SKOLEM_STORE; public PagodaProperties(String path) { java.util.Properties m_properties = new java.util.Properties(); @@ -233,4 +241,8 @@ public class PagodaProperties { public void setStatisticsDir(Path statisticsDir) { this.statisticsDir = statisticsDir; } + + public long getMaxTriplesInSkolemStore() { + return maxTriplesInSkolemStore; + } } -- cgit v1.2.3