From ad58eea444d9f1f16a2498a32777719911203a23 Mon Sep 17 00:00:00 2001 From: Federico Igne Date: Tue, 5 Jan 2021 17:00:29 +0000 Subject: Add external script to execute benchmarks and gather results. --- run_tests.bash | 203 +++++++++++++++++++++ src/main/scala/uk/ac/ox/cs/rsacomb/Main.scala | 2 +- .../scala/uk/ac/ox/cs/rsacomb/RSAOntology.scala | 16 +- .../scala/uk/ac/ox/cs/rsacomb/util/Logger.scala | 2 +- 4 files changed, 214 insertions(+), 9 deletions(-) create mode 100755 run_tests.bash diff --git a/run_tests.bash b/run_tests.bash new file mode 100755 index 0000000..17cbf01 --- /dev/null +++ b/run_tests.bash @@ -0,0 +1,203 @@ +#!/usr/bin/env bash + +NC='\033[0m' +RED='\033[0;31m' +GREEN='\033[0;32m' +YELLOW='\033[0;33m' + +msg_info() { + echo -e "${GREEN}$1${NC}" +} + +msg_warn() { + echo -e "${YELLOW}$1${NC}" +} + +msg_error() { + echo -e "${RED}$1${NC}" +} + +print_help() { + echo + echo "testRSA - a quick script to run tests all night long" + echo + echo "USAGE:" + echo " testRSA OPTION [...]" + echo + echo "OPTIONs are:" + echo " -o | --ontology :" + echo " path to ontology." + echo " -d | --data :" + echo " path to a folder containing data for the ontology." + echo " -q | --queries :" + echo " path to a folder containing SPARQL query files to be" + echo " executed against the ontology and data." + echo " -p | --prefix :" + echo " provides a folder to prefix to the output files." + echo " Defaults to './results'." + echo " -h | -? | --help:" + echo " print this help" + echo +} + +ONTOLOGY="" +DATA="" +QUERIES="" +PREFIX="./results" + +while [[ $# -gt 0 ]] +do + case $1 in + -o|--ontology) + shift + ONTOLOGY="$1" + [ ! -r "$ONTOLOGY" ] && \ + msg_error "Unable to read '$ONTOLOGY'" && \ + print_help && \ + exit 2 + ;; + -d|--data) + shift + DATA="$1" + [ ! -d "$DATA" ] && \ + msg_error "'$DATA' is not a directory" && \ + print_help && \ + exit 2 + ;; + -q|--queries) + shift + QUERIES="$1" + [ ! -d "$QUERIES" ] && \ + msg_error "'$QUERIES' is not a directory" && \ + print_help && \ + exit 2 + ;; + -p|--prefix) + shift + PREFIX="$1" + ;; + -h|-?|--help) + print_help + exit 0 + ;; + *) + msg_error "$OPTION: invalid option" + print_help + exit 1 + ;; + esac + shift +done + +[ -z "$ONTOLOGY" ] && \ + msg_error "Use -o | --ontology to provide an ontology file" && \ + print_help && \ + exit 3 + +[ -z "$DATA" ] && \ + msg_error "Use -d | --data to provide a data folder" && \ + print_help && \ + exit 3 + +[ -z "$QUERIES" ] && \ + msg_error "Use -q | --queries to provide a query folder" && \ + print_help && \ + exit 3 + + +DATAS=`\ls $DATA/*` +mkdir -p "$PREFIX" +for QUERY in "$QUERIES"/*.sparql +do + sbt "run $QUERY $ONTOLOGY $DATAS" 2>&1 | tee "$PREFIX/answers_$(basename $QUERY .sparql).txt" +done + +OUTPUT="$PREFIX/results.csv" +echo "NAME, TBOX, RBOX, ABOX, \ + CANONICAL MODEL GENERATION, \ + CANONICAL MODEL RULES, CANONICAL MODEL RULES LOADING, \ + CANONICAL MODEL FACTS, CANONICAL MODEL FACTS LOADING, \ + CANONICAL MODEL IDB, CANONICAL MODEL EDB, \ + FILTERING PROGRAM GENERATION, \ + FILTERING PROGRAM RULES, FILTERING PROGRAM RULES LOADING, \ + FILTERING PROGRAM FACTS, FILTERING PROGRAM FACTS LOADING, \ + FILTERING PROGRAM IDB, FILTERING PROGRAM EDB, \ + ANSWERING TIME, #ANSWERS, #UNFILTERED, #SPURIOUS, %SPURIOUS" > "$OUTPUT" + +for RESULT in "$PREFIX"/*.txt +do + awk -v filename="$RESULT" ' + BEGIN { + OFS = ", " + name = filename + sub("^.*answers_", "", name) + sub(".txt$", "", name) + } + /Original TBox/ { tbox_size = $NF } + /Original RBox/ { rbox_size = $NF } + /Original ABox/ { abox_size = $NF } + /Generating canonical model program \(END\)/ { canon_gen_time = $NF } + /Generating filtering program \(END\)/ { filter_gen_time = $NF } + /Canonical model rules/ { + canon_rules = $NF + canon = 1 + } + /Canonical model facts/ { + canon_facts = $NF + canon = 1 + } + /Filtering program rules/ { + filter_rules = $NF + } + /Filtering program facts/ { + filter_facts = $NF + } + /Loading rules \(END\)/ { + if (canon) { + canon_rules_load = $NF + } else { + filter_rules_load = $NF + } + } + /Loading facts/ { + if (canon) { + canon_facts_load = $NF + } else { + filter_facts_load = $NF + } + } + /Aggregate number of IDB facts/ { + sub("^.*=", "") + sub(",$", "") + if (canon) { + canon_idb = $0 + } else { + filter_idb = $0 + } + } + /Aggregate number of EDB facts/ { + sub("^.*=", "") + sub(",$", "") + if (canon) { + canon_edb = $0 + canon = 0 + } else { + filter_edb = $0 + } + } + /Answers computation \(END\)/ { answers_time = $NF } + /Number of answers/ { answers = $(NF-1) } + /Number of unfiltered answers/ { unfiltered = $(NF-1) } + /Number of spurious answers/ { spurious = $(NF-1) } + /Percentage of spurious answers/ { spurious_perc = $NF } + END { print name, tbox_size, rbox_size, abox_size, \ + canon_gen_time, canon_rules, canon_rules_load, canon_facts, canon_facts_load, \ + canon_idb, canon_edb, \ + filter_gen_time, filter_rules, filter_rules_load, filter_facts, filter_facts_load, \ + filter_idb, filter_edb, \ + answers_time, answers, unfiltered, spurious, spurious_perc + } + ' "$RESULT" >> "$OUTPUT" +done + +exit 0 diff --git a/src/main/scala/uk/ac/ox/cs/rsacomb/Main.scala b/src/main/scala/uk/ac/ox/cs/rsacomb/Main.scala index 0554dbc..bf96a31 100644 --- a/src/main/scala/uk/ac/ox/cs/rsacomb/Main.scala +++ b/src/main/scala/uk/ac/ox/cs/rsacomb/Main.scala @@ -61,7 +61,7 @@ object RSAComb extends App { case Some(query) => { val answers = ontology ask query Logger.print(s"$answers", Logger.QUIET) - Logger print s"Number of answer: ${answers.length} (${answers.lengthWithMultiplicity})" + Logger print s"Number of answers: ${answers.length} (${answers.lengthWithMultiplicity})" /* Additional DEBUG information */ if (Logger.level >= Logger.DEBUG) { diff --git a/src/main/scala/uk/ac/ox/cs/rsacomb/RSAOntology.scala b/src/main/scala/uk/ac/ox/cs/rsacomb/RSAOntology.scala index 4ac5a77..8d5bf4c 100644 --- a/src/main/scala/uk/ac/ox/cs/rsacomb/RSAOntology.scala +++ b/src/main/scala/uk/ac/ox/cs/rsacomb/RSAOntology.scala @@ -114,9 +114,9 @@ class RSAOntology(val ontology: OWLOntology) { val axioms: List[OWLLogicalAxiom] = abox ::: tbox ::: rbox - Logger.print(s"Original TBox: ${tbox.length} axioms", Logger.DEBUG) - Logger.print(s"Original RBox: ${rbox.length} axioms", Logger.DEBUG) - Logger.print(s"Original ABox: ${abox.length} axioms", Logger.DEBUG) + Logger.print(s"Original TBox: ${tbox.length}", Logger.DEBUG) + Logger.print(s"Original RBox: ${rbox.length}", Logger.DEBUG) + Logger.print(s"Original ABox: ${abox.length}", Logger.DEBUG) /* Retrieve individuals in the original ontology */ @@ -358,16 +358,18 @@ class RSAOntology(val ontology: OWLOntology) { //data.beginTransaction(TransactionType.READ_WRITE) - Logger print s"Canonical model: ${canon.rules.length} rules" + Logger print s"Canonical model rules: ${canon.rules.length}" RDFoxUtil.addRules(data, this.canonicalModel.rules) - Logger print s"Canonical model: ${canon.facts.length} facts" + Logger print s"Canonical model facts: ${canon.facts.length}" RDFoxUtil.addFacts(data, this.canonicalModel.facts) - Logger print s"Filtering program: ${filter.facts.length} facts" + RDFoxUtil printStatisticsFor data + + Logger print s"Filtering program facts: ${filter.facts.length}" RDFoxUtil.addFacts(data, filter.facts) - Logger print s"Filtering program: ${filter.rules.length} rules" + Logger print s"Filtering program rules: ${filter.rules.length}" RDFoxUtil.addRules(data, filter.rules) //data.commitTransaction() diff --git a/src/main/scala/uk/ac/ox/cs/rsacomb/util/Logger.scala b/src/main/scala/uk/ac/ox/cs/rsacomb/util/Logger.scala index 74797a2..56e9de0 100644 --- a/src/main/scala/uk/ac/ox/cs/rsacomb/util/Logger.scala +++ b/src/main/scala/uk/ac/ox/cs/rsacomb/util/Logger.scala @@ -38,7 +38,7 @@ object Logger { print(s"$desc (START)", lvl) val result = expr val t1 = System.currentTimeMillis() - print(s"$desc (END): ${(t1 - t0).toFloat / 1000}s", lvl) + print(s"$desc (END): ${(t1 - t0).toFloat / 1000}", lvl) result } -- cgit v1.2.3