Incremental Skolemised store (seems completed).

5 files changed, 93 insertions, 19 deletions

diff --git a/src/resources/pagoda.properties b/src/resources/pagoda.properties
index 34b3d7a..aa08593 100644
--- a/src/resources/pagoda.properties
+++ b/src/resources/pagoda.properties
@@ -2,8 +2,12 @@ debug=true
 useAlwaysSimpleUpperBound=false
 #skolemUpperBound=DISABLED
 #skolemUpperBound=BEFORE_SUMMARISATION
-skolemUpperBound=AFTER_SUMMARISATION
+skolemUpperBound=BEFORE_SUMMARISATION
 skolemDepth=10
+
+# seems ok for -Xmx6g
+maxTriplesInSkolemStore=2500000
+
 toCallHermit=true
 
 statisticsDir=/home/alessandro/Dropbox/Oxford/PAGOdA/statistics
\ No newline at end of file
diff --git a/src/uk/ac/ox/cs/pagoda/multistage/MultiStageQueryEngine.java b/src/uk/ac/ox/cs/pagoda/multistage/MultiStageQueryEngine.java
index f3a78f6..209e1d7 100644
--- a/src/uk/ac/ox/cs/pagoda/multistage/MultiStageQueryEngine.java
+++ b/src/uk/ac/ox/cs/pagoda/multistage/MultiStageQueryEngine.java
@@ -35,7 +35,9 @@ public class MultiStageQueryEngine extends StageQueryEngine {
     private HashMap<String, List> statistics = new HashMap<>();
     private Set<Tuple<Integer>> oversizedSkolemisedFacts;
     private RDFoxTripleManager rdFoxTripleManager;
+
     private int lastMaxTermDepth = -1;
+    private boolean firstCall = true;
 
     public MultiStageQueryEngine(String name, boolean checkValidity) {
         super(name, checkValidity);
@@ -79,11 +81,12 @@ public class MultiStageQueryEngine extends StageQueryEngine {
     public int materialiseSkolemly(DatalogProgram dProgram, GapByStore4ID gap, int maxTermDepth) {
         if(isDisposed()) throw new DisposedException();
 
-        if(maxTermDepth <= lastMaxTermDepth)
+        if(!firstCall && maxTermDepth <= lastMaxTermDepth)
             throw new IllegalArgumentException("maxTermDepth must be greater than " + lastMaxTermDepth);
+        if(firstCall)
+            materialise("lower program", dProgram.getLower().toString());
         lastMaxTermDepth = maxTermDepth;
 
-        materialise("lower program", dProgram.getLower().toString());
         Program generalProgram = dProgram.getGeneral();
         LimitedSkolemisationApplication program =
                 new LimitedSkolemisationApplication(generalProgram,
@@ -91,7 +94,9 @@ public class MultiStageQueryEngine extends StageQueryEngine {
                                                     maxTermDepth);
         rdFoxTripleManager = new RDFoxTripleManager(store, true);
         Treatment treatment = new Pick4NegativeConceptNaive(this, program, rdFoxTripleManager);
-        return materialise(program, treatment, gap, maxTermDepth);
+        int result = materialise(program, treatment, gap, maxTermDepth);
+        firstCall = false;
+        return result;
     }
 
     public int materialise4SpecificQuery(Program generalProgram, QueryRecord record, BottomStrategy upperBottom) {
@@ -117,7 +122,11 @@ public class MultiStageQueryEngine extends StageQueryEngine {
     }
 
     private int materialise(MultiStageUpperProgram program, Treatment treatment, GapByStore4ID gap, int maxTermDepth) {
-        boolean actuallyCleaned = cleanStoreFromOversizedSkolemisedFacts();
+        if(!firstCall)
+            cleanStoreFromOversizedSkolemisedFacts();
+
+        boolean isSkolemising = maxTermDepth > 0;
+//        boolean isSkolemising = true;
 
         if(gap != null)
             treatment.addAdditionalGapTuples();
@@ -157,11 +166,11 @@ public class MultiStageQueryEngine extends StageQueryEngine {
                     }
                 }
                 else {
-                    if(!incrementally) {
+                    if(!incrementally && firstCall) {
 //                                              store.addRules(new String[] {datalogProgram});
                         store.importRules(datalogProgram);
                     }
-                    store.applyReasoning(incrementally || actuallyCleaned);
+                    store.applyReasoning(incrementally || !firstCall);
                 }
 
 //                              Utility.logInfo("The number of sameAs assertions in the current store: " + getSameAsNumber());
@@ -181,15 +190,17 @@ public class MultiStageQueryEngine extends StageQueryEngine {
                 Utility.logDebug("Time to materialise datalog-rules: " + subTimer.duration());
 
                 subTimer.reset();
-                if((violations = program.isIntegrated(this, incrementally)) == null || violations.size() == 0) {
-                    store.clearRulesAndMakeFactsExplicit();
+                if((violations = program.isIntegrated(this, !isSkolemising && incrementally)) == null || violations.size() == 0) {
+                    if(!isSkolemising)
+                        store.clearRulesAndMakeFactsExplicit();
                     Utility.logDebug(name + " store after materialising " + programName + ": " + tripleCount + " (" + (tripleCount - tripleCountBeforeMat) + " new)");
                     Utility.logDebug(name + " store is DONE for multi-stage materialising in " + t.duration() + " seconds.");
                     return isValid() ? 1 : 0;
                 }
                 Utility.logDebug("Time to detect violations: " + subTimer.duration());
 
-//                store.makeFactsExplicit();
+                if(!isSkolemising)
+                    store.makeFactsExplicit();
                 subTimer.reset();
                 oldTripleCount = store.getTriplesCount();
 
@@ -274,8 +285,8 @@ public class MultiStageQueryEngine extends StageQueryEngine {
                     result.add(new TupleBuilder<Integer>().append(idTriple[0]).append(idTriple[1])
                             .append(idTriple[2]).build());
                 }
-                else if(!(atom.getArgument(0) instanceof Individual))
-                    throw new IllegalArgumentException("No individuals: " + atom);
+//                else if(!(atom.getArgument(0) instanceof Individual))
+//                    throw new IllegalArgumentException("No individuals: " + atom);
             }
             else {
                 if((atom.getArgument(0) instanceof Individual && termsManager.getDepthOf((Individual) atom.getArgument(0)) >= maxDepth)
@@ -284,8 +295,8 @@ public class MultiStageQueryEngine extends StageQueryEngine {
                     result.add(new TupleBuilder<Integer>().append(idTriple[0]).append(idTriple[1])
                             .append(idTriple[2]).build());
                 }
-                else if(!(atom.getArgument(0) instanceof Individual) && !(atom.getArgument(1) instanceof Individual))
-                    throw new IllegalArgumentException("No individuals: " + atom);
+//                else if(!(atom.getArgument(0) instanceof Individual) && !(atom.getArgument(1) instanceof Individual))
+//                    throw new IllegalArgumentException("No individuals: " + atom);
             }
 
         }
diff --git a/src/uk/ac/ox/cs/pagoda/reasoner/MyQueryReasoner.java b/src/uk/ac/ox/cs/pagoda/reasoner/MyQueryReasoner.java
index a393474..8a90a26 100644
--- a/src/uk/ac/ox/cs/pagoda/reasoner/MyQueryReasoner.java
+++ b/src/uk/ac/ox/cs/pagoda/reasoner/MyQueryReasoner.java
@@ -2,6 +2,7 @@ package uk.ac.ox.cs.pagoda.reasoner;
 
 import org.semanticweb.karma2.profile.ELHOProfile;
 import org.semanticweb.owlapi.model.OWLOntology;
+import uk.ac.ox.cs.JRDFox.JRDFStoreException;
 import uk.ac.ox.cs.pagoda.multistage.MultiStageQueryEngine;
 import uk.ac.ox.cs.pagoda.owl.EqualitiesEliminator;
 import uk.ac.ox.cs.pagoda.owl.OWLHelper;
@@ -18,6 +19,7 @@ import uk.ac.ox.cs.pagoda.tracking.QueryTracker;
 import uk.ac.ox.cs.pagoda.tracking.TrackingRuleEncoder;
 import uk.ac.ox.cs.pagoda.tracking.TrackingRuleEncoderDisjVar1;
 import uk.ac.ox.cs.pagoda.tracking.TrackingRuleEncoderWithGap;
+import uk.ac.ox.cs.pagoda.util.ExponentialInterpolation;
 import uk.ac.ox.cs.pagoda.util.PagodaProperties;
 import uk.ac.ox.cs.pagoda.util.Timer;
 import uk.ac.ox.cs.pagoda.util.Utility;
@@ -25,6 +27,7 @@ import uk.ac.ox.cs.pagoda.util.disposable.DisposedException;
 import uk.ac.ox.cs.pagoda.util.tuples.Tuple;
 
 import java.util.Collection;
+import java.util.LinkedList;
 
 class MyQueryReasoner extends QueryReasoner {
 
@@ -399,8 +402,32 @@ class MyQueryReasoner extends QueryReasoner {
         relevantStore.materialise("Mark original individuals", relevantOriginalMarkProgram);
 
         boolean isFullyProcessed = false;
-        for (int currentMaxTermDepth = 1;
-             currentMaxTermDepth <= properties.getSkolemDepth() && !isFullyProcessed; currentMaxTermDepth++) {
+        LinkedList<Tuple<Long>> lastTwoTriplesCounts = new LinkedList<>();
+        for (int currentMaxTermDepth = 1; !isFullyProcessed; currentMaxTermDepth++) {
+
+            if(currentMaxTermDepth > properties.getSkolemDepth()) {
+                Utility.logInfo("Maximum term depth reached");
+                break;
+            }
+
+            if(lastTwoTriplesCounts.size() == 2) {
+                if(lastTwoTriplesCounts.get(0).get(1).equals(lastTwoTriplesCounts.get(1).get(1)))
+                    break;
+
+                ExponentialInterpolation interpolation = new ExponentialInterpolation(lastTwoTriplesCounts.get(0).get(0),
+                        lastTwoTriplesCounts.get(0).get(1),
+                        lastTwoTriplesCounts.get(1).get(0),
+                        lastTwoTriplesCounts.get(1).get(1));
+                double triplesEstimate = interpolation.computeValue(currentMaxTermDepth);
+
+                Utility.logDebug("Estimate of the number of triples:" + triplesEstimate);
+
+                // exit condition if the query is not fully answered
+                if(triplesEstimate > properties.getMaxTriplesInSkolemStore()) {
+                    Utility.logInfo("Interrupting Semi-Skolemisation because of triples count limit");
+                    break;
+                }
+            }
 
             Utility.logInfo("Trying with maximum depth " + currentMaxTermDepth);
 
@@ -408,18 +435,31 @@ class MyQueryReasoner extends QueryReasoner {
                     currentMaxTermDepth);
             queryRecord.addProcessingTime(Step.SKOLEM_UPPER_BOUND, t.duration());
             if(materialisationTag == -1) {
+                relevantStore.dispose();
                 throw new Error("A consistent ontology has turned out to be " +
                                         "inconsistent in the Skolemises-relevant-upper-store");
             }
             else if(materialisationTag != 1) {
                 Utility.logInfo("Semi-Skolemised relevant upper store cannot be employed");
-                return false;
+                break;
             }
 
             Utility.logInfo("Querying semi-Skolemised upper store...");
             isFullyProcessed = queryUpperStore(relevantStore, queryRecord,
-                                                       queryRecord.getExtendedQueryText(),
-                                                       Step.SKOLEM_UPPER_BOUND);
+                    queryRecord.getExtendedQueryText(),
+                    Step.SKOLEM_UPPER_BOUND);
+
+            try {
+                lastTwoTriplesCounts.add
+                        (new Tuple<>((long) currentMaxTermDepth, relevantStore.getStoreSize()));
+            } catch (JRDFStoreException e) {
+                e.printStackTrace();
+                break;
+            }
+            if(lastTwoTriplesCounts.size() > 2)
+                lastTwoTriplesCounts.remove();
+
+            Utility.logInfo("Last two triples counts:" + lastTwoTriplesCounts);
         }
 
         relevantStore.dispose();
diff --git a/src/uk/ac/ox/cs/pagoda/reasoner/light/BasicQueryEngine.java b/src/uk/ac/ox/cs/pagoda/reasoner/light/BasicQueryEngine.java
index 107d3ca..034827e 100644
--- a/src/uk/ac/ox/cs/pagoda/reasoner/light/BasicQueryEngine.java
+++ b/src/uk/ac/ox/cs/pagoda/reasoner/light/BasicQueryEngine.java
@@ -32,6 +32,13 @@ public class BasicQueryEngine extends RDFoxQueryEngine {
         parameters.m_useBushy = true;
     }
 
+    /***
+     * @return Overall number of triples.
+     */
+    public long getStoreSize() throws JRDFStoreException {
+        return store.getTriplesCount();
+    }
+
     public void materialiseFoldedly(DatalogProgram dProgram, GapByStore4ID gap) {
         if(isDisposed()) throw new DisposedException();
         if(gap != null) {
diff --git a/src/uk/ac/ox/cs/pagoda/util/PagodaProperties.java b/src/uk/ac/ox/cs/pagoda/util/PagodaProperties.java
index 18469ff..0f9ad4e 100644
--- a/src/uk/ac/ox/cs/pagoda/util/PagodaProperties.java
+++ b/src/uk/ac/ox/cs/pagoda/util/PagodaProperties.java
@@ -20,6 +20,7 @@ public class PagodaProperties {
     private static final int DEFAULT_SKOLEM_DEPTH;
     private static final boolean DEFAULT_TO_CALL_HERMIT;
     private static final Path DEFAULT_STATISTICS_DIR;
+    private static final long DEFAULT_MAX_TRIPLES_IN_SKOLEM_STORE;
 
     public static boolean shellModeDefault = false;
     private static boolean debug = DEFAULT_DEBUG;
@@ -30,6 +31,7 @@ public class PagodaProperties {
         int defaultSkolemDepth = 1;
         boolean toCallHermit = true;
         Path defaultStatisticsDir = null;
+        long defaultMaxTriplesInSkolemStore = 1000000;
 
         try (InputStream in = PagodaProperties.class.getClassLoader().getResourceAsStream(CONFIG_FILE)) {
             Properties config = new Properties();
@@ -77,6 +79,10 @@ public class PagodaProperties {
                 defaultSkolemDepth = Integer.parseInt(config.getProperty("skolemDepth"));
                 logger.debug("By default the max skolemisation depth is " + defaultSkolemDepth);
             }
+            if (config.containsKey("maxTriplesInSkolemStore")) {
+                defaultMaxTriplesInSkolemStore = Long.parseLong(config.getProperty("maxTriplesInSkolemStore"));
+                logger.debug("By default the maximum number of triples in the Skolem store is " + defaultMaxTriplesInSkolemStore);
+            }
 
         } catch (IOException e) {
             e.printStackTrace();
@@ -86,6 +92,7 @@ public class PagodaProperties {
         DEFAULT_TO_CALL_HERMIT = toCallHermit;
         DEFAULT_STATISTICS_DIR = defaultStatisticsDir;
         DEFAULT_SKOLEM_DEPTH = defaultSkolemDepth;
+        DEFAULT_MAX_TRIPLES_IN_SKOLEM_STORE = defaultMaxTriplesInSkolemStore;
     }
 
     String dataPath = null;
@@ -108,6 +115,7 @@ public class PagodaProperties {
     private boolean useAlwaysSimpleUpperBound = DEFAULT_USE_ALWAYS_SIMPLE_UPPER_BOUND;
     private SkolemUpperBoundOptions skolemUpperBound = DEFAULT_SKOLEM_UPPER_BOUND;
     private Path statisticsDir = DEFAULT_STATISTICS_DIR;
+    private long maxTriplesInSkolemStore = DEFAULT_MAX_TRIPLES_IN_SKOLEM_STORE;
 
     public PagodaProperties(String path) {
         java.util.Properties m_properties = new java.util.Properties();
@@ -233,4 +241,8 @@ public class PagodaProperties {
     public void setStatisticsDir(Path statisticsDir) {
         this.statisticsDir = statisticsDir;
     }
+
+    public long getMaxTriplesInSkolemStore() {
+        return maxTriplesInSkolemStore;
+    }
 }