1 files changed, 320 insertions, 0 deletions

diff --git a/external/uk/ac/ox/cs/data/sample/DataSampling.java b/external/uk/ac/ox/cs/data/sample/DataSampling.java
new file mode 100644
index 0000000..1a788e3
--- /dev/null
+++ b/external/uk/ac/ox/cs/data/sample/DataSampling.java
@@ -0,0 +1,320 @@
+package uk.ac.ox.cs.data.sample;
+
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileNotFoundException;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.util.*;
+import java.util.Map.Entry;
+
+import org.openrdf.model.Resource;
+import org.openrdf.model.Statement;
+import org.openrdf.model.URI;
+import org.openrdf.model.Value;
+import org.openrdf.model.impl.StatementImpl;
+import org.openrdf.model.impl.URIImpl;
+import org.openrdf.rio.RDFHandler;
+import org.openrdf.rio.RDFHandlerException;
+import org.openrdf.rio.RDFParseException;
+import org.openrdf.rio.RDFParser;
+import org.openrdf.rio.ntriples.NTriplesParser;
+import org.openrdf.rio.turtle.*;
+
+import uk.ac.ox.cs.pagoda.owl.OWLHelper;
+import uk.ac.ox.cs.pagoda.util.Namespace;
+import uk.ac.ox.cs.pagoda.util.Utility;
+
+public class DataSampling {
+        
+        File[] m_list;
+        RDFGraph m_graph;
+        double m_percentage;
+        Set<String> excludeEntities = new HashSet<String>(); 
+        
+        public DataSampling(String prefix, String fileName, String excludeFile, double percentage, boolean inTurtle) {
+                if (excludeFile != null) {
+                        try {
+                                Scanner scanner = new Scanner(new File(excludeFile));
+                                while (scanner.hasNextLine())
+                                        excludeEntities.add(OWLHelper.removeAngles(scanner.nextLine().trim())); 
+                                scanner.close(); 
+                        } catch (FileNotFoundException e1) {
+                                // TODO Auto-generated catch block
+                                e1.printStackTrace();
+                        }
+                }
+                excludeEntities.add("http://www.w3.org/2002/07/owl#imports"); 
+
+                File file = new File(fileName); 
+                if (file.isDirectory()) m_list = file.listFiles();
+                else m_list = new File[] {file};
+                m_percentage = percentage; 
+                
+                RDFParser parser = inTurtle ? new TurtleParser() : new NTriplesParser();
+                
+                GraphRDFHandler handler = new GraphRDFHandler(excludeEntities); 
+                parser.setRDFHandler(handler);
+                
+                FileInputStream istream;
+                try {
+                        for (File tFile: m_list) {
+                                parser.parse(istream = new FileInputStream(tFile), prefix);
+                                istream.close();
+                        }
+                } catch (IOException e) {
+                        e.printStackTrace();
+                } catch (RDFParseException e) {
+                        e.printStackTrace();
+                } catch (RDFHandlerException e) {
+                        e.printStackTrace();
+                }
+                
+                m_graph = handler.getGraph(); 
+        }
+        
+        public void sample(String outputFile, boolean multiStart) {
+                try {
+                        FileOutputStream ostream = new FileOutputStream(outputFile); 
+                        TurtleWriter writer = new TurtleWriter(ostream);
+                        writer.startRDF();
+                        
+                        if (m_percentage < 100) {
+                                Sampler sam = multiStart ? 
+                                                new RandomWalkMulti(m_graph, writer) : 
+                                                        new RandomWalk(m_graph, writer);
+                                sam.setLimit((int) (m_graph.numberOfStatement / 100 * m_percentage));
+                                System.out.println("Statement limit: " + (m_graph.numberOfStatement / 100 * m_percentage)); 
+                                sam.sample();
+                                sam.dispose(); 
+                        }
+                        else {
+                                m_graph.visit(writer);
+                        }
+                        writer.endRDF();
+                        ostream.close();
+                } catch (IOException e) {
+                        e.printStackTrace(); 
+                } catch (RDFHandlerException e) {
+                        // TODO Auto-generated catch block
+                        e.printStackTrace();
+                }
+        }
+        
+        public static void main(String[] args) {
+                sampleReactome();
+//              sampleChEMBL(); 
+        }
+        
+        static void sampleReactome() {
+//              double[] ps = {40, 70, 100}; 
+                double[] ps = {0.25, 0.5, 0.75}; 
+                for (double per: ps) {
+                DataSampling sampling = new DataSampling(
+                                "http://www.biopax.org/release/biopax-level3.owl#", 
+//                              "/home/yzhou/krr-nas-share/Yujiao/ontologies/bio2rdf/reactome/data/data.ttl", 
+                                "/home/yzhou/krr-nas-share/Yujiao/ontologies/bio2rdf/reactome/data/simplified.nt",
+//                              "/home/yzhou/krr-nas-share/Yujiao/ontologies/bio2rdf/reactome/graph sampling/reactome_sample_40.ttl", 
+                                "/home/yzhou/krr-nas-share/Yujiao/ontologies/bio2rdf/reactome/graph sampling/exclude",
+                                per, 
+                                true); 
+                sampling.sample("/home/yzhou/krr-nas-share/Yujiao/ontologies/bio2rdf/reactome/graph sampling/sample_test_" + per + ".ttl", true);
+//              sampling.sample("/home/yzhou/krr-nas-share/Yujiao/ontologies/bio2rdf/reactome/graph sampling/simplifed_sample_test_" + per + ".ttl", true);
+//              sampling.sample("output/sample_reactome_multi.ttl", true);
+                }
+        }
+
+        static void sampleChEMBL() {
+                DataSampling sampling = new DataSampling(
+                                "http://rdf.ebi.ac.uk/terms/chembl", 
+                                "/home/yzhou/RDFdata/ChEMBL/facts/chembl_kbfile.nt", 
+                                null,
+                                100, 
+                                false);
+                
+                sampling.sample("output/sample_chembl_multi.ttl", true); 
+                sampling.sample("output/sample_chembl.ttl", false); 
+        }
+        
+}
+
+class RDFGraph {
+        
+        Map<Value, Integer> index = new HashMap<Value, Integer>();
+        Map<Integer, Value> inverseIndex = new HashMap<Integer, Value>(); 
+        MapToList<Integer> labels = new MapToList<Integer>(); 
+        
+        MapToList<RDFEdge> edges = new MapToList<RDFEdge>();
+        Set<String> excludeEntities; 
+        
+        int numberOfIndividuals = 0, numberOfProperties = 0; 
+        
+        public RDFGraph(Set<String> exclude) {
+                excludeEntities = exclude; 
+                for (String str: excludeEntities) 
+                        System.out.println(str); 
+                System.out.println("---------------"); 
+        }
+
+        public void visit(TurtleWriter writer) throws RDFHandlerException {
+                Integer key; 
+                for (Entry<Integer, LinkedList<Integer>> entry: labels.entrySet()) {
+                        key = entry.getKey(); 
+                        for (Integer type: entry.getValue())
+                                writer.handleStatement(getStatement(key, type)); 
+                }
+                
+                for (Entry<Integer, LinkedList<RDFEdge>> entry: edges.entrySet()) {
+                        key = entry.getKey();
+                        if ((inverseIndex.get(key) instanceof URI) && 
+                                        ((URI) inverseIndex.get(key)).toString().equals("http://www.reactome.org/biopax/46/879693#UnificationXref9"))
+                                System.out.println("Here");
+
+                        for (RDFEdge edge: entry.getValue())
+                                writer.handleStatement(getStatement(key, edge.m_label, edge.m_dst));
+                }
+        }
+
+        private int getID(Value v, boolean isIndividual) {
+                if (v.toString().contains("imports"))
+                        System.out.println(v.toString()); 
+                if (excludeEntities.contains(v.toString())) {
+                        return 0; 
+                }
+                
+                Integer id = index.get(v); 
+                if (id == null)
+                        if (isIndividual) {
+                                index.put(v, id = ++numberOfIndividuals);
+                                inverseIndex.put(id, v);
+                        }
+                        else {
+                                index.put(v, id = --numberOfProperties);
+                                inverseIndex.put(id, v); 
+                        }
+                return id; 
+        }
+        
+        int numberOfStatement = 0;
+        int counter = 0; 
+        
+        public void addTriple(Resource s, URI p, Value o) {
+                ++numberOfStatement; 
+                if (numberOfStatement % 1000000 == 0) {
+                        Utility.logInfo("No.of statements: " + numberOfStatement, "\tNo.of individuals: " + numberOfIndividuals, "\tNo.of predicates: " + (-numberOfProperties));
+                }
+                
+                if (p.equals(rdftype)) {
+                        int type = getID(o, false), i = getID(s, true);
+                        if (i == 0) {
+//                              System.out.println("<" + s + "> <" + p + "> <" + o + ">"); 
+                                return ; 
+                        }
+                        labels.add(i, type); 
+                }
+                else { 
+                        int i = getID(s, true), j = getID(o, true), prop = getID(p, false) ;
+                        if (i == 0 || j == 0 || prop == 0) {
+//                              System.out.println("<" + s + "> <" + p + "> <" + o + ">"); 
+                                return ; 
+                        }
+                        edges.add(i, new RDFEdge(prop, j)); 
+                }
+        }
+        
+        URI rdftype = new URIImpl(Namespace.RDF_TYPE); 
+
+        public Statement getStatement(int... args) {
+                if (args.length == 2) 
+                        return new StatementImpl((Resource) inverseIndex.get(args[0]), rdftype, (Value) inverseIndex.get(args[1]));
+                else if (args.length == 3) 
+                        return new StatementImpl((Resource) inverseIndex.get(args[0]), (URI) inverseIndex.get(args[1]), (Value) inverseIndex.get(args[2]));
+                return null;
+        }
+
+        public String getRawString(int id) {
+                return inverseIndex.get(id).toString();
+        }
+        
+}
+
+class MapToList<T> {
+        
+        private Map<Integer, LinkedList<T>> map = new HashMap<Integer, LinkedList<T>>();
+        
+        public void add(int key, T value) {
+                LinkedList<T> list = map.get(key); 
+                if (list == null) 
+                        map.put(key, list = new LinkedList<T>()); 
+                list.add(value); 
+        }
+
+        public Set<Map.Entry<Integer, LinkedList<T>>> entrySet() {
+                return map.entrySet();
+        }
+
+        public void shuffle() {
+                for (List<T> list: map.values())
+                        Collections.shuffle(list);
+        }
+
+        public LinkedList<T> get(int key) {
+                return map.get(key); 
+        }
+
+}
+
+class RDFEdge {
+        
+        int m_label, m_dst; 
+
+        public RDFEdge(int label, int dst) {
+                m_label = label; 
+                m_dst = dst; 
+        }
+
+}
+
+class GraphRDFHandler implements RDFHandler {
+        
+        RDFGraph m_graph;
+        Set<String> m_exclude; 
+        
+        public GraphRDFHandler(Set<String> excludeEntities) {
+                m_exclude = excludeEntities; 
+        }
+        
+        @Override
+        public void startRDF() throws RDFHandlerException {
+                m_graph = new RDFGraph(m_exclude);              
+        }
+
+        public RDFGraph getGraph() {
+                return m_graph;
+        }
+
+        @Override
+        public void endRDF() throws RDFHandlerException {
+                // TODO Auto-generated method stub
+                
+        }
+
+        @Override
+        public void handleNamespace(String prefix, String uri)
+                        throws RDFHandlerException {
+                // TODO Auto-generated method stub
+                
+        }
+
+        @Override
+        public void handleStatement(Statement st) throws RDFHandlerException {
+                m_graph.addTriple(st.getSubject(), st.getPredicate(), st.getObject());
+        }
+
+        @Override
+        public void handleComment(String comment) throws RDFHandlerException {
+                // TODO Auto-generated method stub
+                
+        }
+        
+}
\ No newline at end of file