From 9ae2b9e018e17eafeb86b445f403308918daf4d3 Mon Sep 17 00:00:00 2001 From: Rob Court Date: Sat, 31 Aug 2024 18:55:31 +0100 Subject: [PATCH 1/7] adding limit --- src/main/java/ebi/spot/neo4j2owl/N2OProcedure.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/java/ebi/spot/neo4j2owl/N2OProcedure.java b/src/main/java/ebi/spot/neo4j2owl/N2OProcedure.java index 0353185..f63f005 100644 --- a/src/main/java/ebi/spot/neo4j2owl/N2OProcedure.java +++ b/src/main/java/ebi/spot/neo4j2owl/N2OProcedure.java @@ -73,10 +73,10 @@ public Stream exportOWLNodes(@Name("skip") Long skip, @Name("lim @SuppressWarnings("unused") @Procedure(mode = Mode.WRITE) - public Stream exportOWLEdges() { + public Stream exportOWLEdges(@Name("skip") Long skip, @Name("limit") Long limit) { logger.resetTimer(); N2OExportService exportService = new N2OExportService(db); - N2OReturnValue result = exportService.owl2ExportEdges(); + N2OReturnValue result = exportService.owl2ExportEdges(skip, limit); return Stream.of(result); } From 0528551a96feda3df24a65ded9070050772e21c7 Mon Sep 17 00:00:00 2001 From: Rob Court Date: Sat, 31 Aug 2024 19:02:34 +0100 Subject: [PATCH 2/7] Update N2OExportService.java --- .../java/ebi/spot/neo4j2owl/exporter/N2OExportService.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/java/ebi/spot/neo4j2owl/exporter/N2OExportService.java b/src/main/java/ebi/spot/neo4j2owl/exporter/N2OExportService.java index 659c345..71e65e6 100644 --- a/src/main/java/ebi/spot/neo4j2owl/exporter/N2OExportService.java +++ b/src/main/java/ebi/spot/neo4j2owl/exporter/N2OExportService.java @@ -115,7 +115,7 @@ public N2OReturnValue owl2ExportNodes(Long skip, Long limit) { return returnValue; } - public N2OReturnValue owl2ExportEdges() { + public N2OReturnValue owl2ExportEdges(Long skip, Long limit) { n2OEntityManager = new N2OExportManager(); qsls_with_no_matching_properties = new HashSet<>(); logger.resetTimer(); @@ -125,7 +125,7 @@ public N2OReturnValue owl2ExportEdges() { OWLOntologyManager man = OWLManager.createOWLOntologyManager(); OWLOntology o = man.createOntology(); - findEntities(0L, Long.MAX_VALUE); + findEntities(o, skip, limit); addRelation(o, N2OStatic.RELTYPE_SUBCLASSOF); addRelation(o, N2OStatic.RELTYPE_INSTANCEOF); for (String rel_qsl : getRelations(OWLAnnotationProperty.class)) { From d72471ab5832bdd2d6e05cd96f2fd98d154d04f3 Mon Sep 17 00:00:00 2001 From: Rob Court Date: Sat, 31 Aug 2024 20:44:36 +0100 Subject: [PATCH 3/7] typo fix --- src/main/java/ebi/spot/neo4j2owl/exporter/N2OExportService.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/ebi/spot/neo4j2owl/exporter/N2OExportService.java b/src/main/java/ebi/spot/neo4j2owl/exporter/N2OExportService.java index 71e65e6..e5febde 100644 --- a/src/main/java/ebi/spot/neo4j2owl/exporter/N2OExportService.java +++ b/src/main/java/ebi/spot/neo4j2owl/exporter/N2OExportService.java @@ -125,7 +125,7 @@ public N2OReturnValue owl2ExportEdges(Long skip, Long limit) { OWLOntologyManager man = OWLManager.createOWLOntologyManager(); OWLOntology o = man.createOntology(); - findEntities(o, skip, limit); + findEntities(skip, limit); addRelation(o, N2OStatic.RELTYPE_SUBCLASSOF); addRelation(o, N2OStatic.RELTYPE_INSTANCEOF); for (String rel_qsl : getRelations(OWLAnnotationProperty.class)) { From bbf11e03841c1eba75ba4ec9ff72224c67222014 Mon Sep 17 00:00:00 2001 From: hkir-dev Date: Wed, 21 Aug 2024 15:57:48 +0100 Subject: [PATCH 4/7] csv file postfixes added for sideloaded entities --- pom.xml | 1 - .../spot/neo4j2owl/importer/N2OCSVWriter.java | 17 +++++++++++++---- .../neo4j2owl/importer/N2OImportService.java | 6 +++--- .../neo4j2owl/importer/N2OImporterRunner.java | 6 +++++- .../ebi/spot/neo4j2owl/importer/N2OUtils.java | 16 +++++++++------- 5 files changed, 30 insertions(+), 16 deletions(-) diff --git a/pom.xml b/pom.xml index a3625e7..5fb8ac2 100644 --- a/pom.xml +++ b/pom.xml @@ -104,7 +104,6 @@ com.github.ben-manes.caffeine caffeine 2.3.3 - provided diff --git a/src/main/java/ebi/spot/neo4j2owl/importer/N2OCSVWriter.java b/src/main/java/ebi/spot/neo4j2owl/importer/N2OCSVWriter.java index 49063c5..c32e710 100644 --- a/src/main/java/ebi/spot/neo4j2owl/importer/N2OCSVWriter.java +++ b/src/main/java/ebi/spot/neo4j2owl/importer/N2OCSVWriter.java @@ -17,6 +17,7 @@ public class N2OCSVWriter { private final RelationTypeCounter relationTypeCounter; private final N2OLog log = N2OLog.getInstance(); private final N2OImportCSVConfig n2OImportCSVConfig = new N2OImportCSVConfig(); + private final String csvPostfix; public enum CSV_TYPE { NODES("nodes"), RELATIONSHIPS("relationship"); @@ -26,10 +27,15 @@ public enum CSV_TYPE } } - N2OCSVWriter(N2OImportManager manager, RelationTypeCounter relationTypeCounter, File dir) { + N2OCSVWriter(N2OImportManager manager, RelationTypeCounter relationTypeCounter, File dir, String csvPostfix) { this.manager = manager; this.dir = dir; this.relationTypeCounter = relationTypeCounter; + if(csvPostfix == null) { + this.csvPostfix = ""; + }else { + this.csvPostfix = csvPostfix; + } } void exportOntologyToCSV() throws N2OException { @@ -49,20 +55,20 @@ private void processExportForRelationships() throws N2OException { Map> relationships = indexRelationshipsByType(); Map> dataout_rel = prepareRelationCSVsForExport(relationships); prepareCyperQueries(dataout_rel, CSV_TYPE.RELATIONSHIPS); - N2OUtils.writeToFile(getImportDir(), dataout_rel, CSV_TYPE.RELATIONSHIPS); + N2OUtils.writeToFile(getImportDir(), dataout_rel, CSV_TYPE.RELATIONSHIPS, csvPostfix); } private void processExportForNodes() throws N2OException { Map> entities = indexEntitiesByType(); Map> dataout = prepareNodeCSVsForExport(entities); prepareCyperQueries(dataout, CSV_TYPE.NODES); - N2OUtils.writeToFile(dir, dataout, CSV_TYPE.NODES); + N2OUtils.writeToFile(dir, dataout, CSV_TYPE.NODES, csvPostfix); } private void prepareCyperQueries(Map> dataout, CSV_TYPE csv_type) { for(String type: dataout.keySet()) { - File f = N2OUtils.constructFileHandle(dir, csv_type.name, type); + File f = N2OUtils.constructFileHandle(dir, csv_type.name, type, csvPostfix); String cypher = constructCypherQuery(csv_type, f); this.n2OImportCSVConfig.putImport(cypher, f.getName()); } @@ -72,6 +78,9 @@ private String constructCypherQuery(CSV_TYPE csv_type, File f) { String filename = f.getName(); String type = filename.substring(filename.indexOf("_") + 1).replaceAll(N2OStatic.CSV_EXTENSION, ""); + if(!csvPostfix.isEmpty()) { + type = type.replace(csvPostfix + "_" , ""); + } Integer periodic_commit = N2OConfig.getInstance().getPeriodicCommit(); String cypher = "USING PERIODIC COMMIT "+periodic_commit+"\n" + "LOAD CSV WITH HEADERS FROM \"file:/"+filename+"\" AS cl\n"; diff --git a/src/main/java/ebi/spot/neo4j2owl/importer/N2OImportService.java b/src/main/java/ebi/spot/neo4j2owl/importer/N2OImportService.java index 9f6e921..3872330 100644 --- a/src/main/java/ebi/spot/neo4j2owl/importer/N2OImportService.java +++ b/src/main/java/ebi/spot/neo4j2owl/importer/N2OImportService.java @@ -25,10 +25,10 @@ public void prepareConfig(String config, File importdir) throws IOException, N2O } public N2OCSVWriter prepareCSVFilesForImport(String url, File importdir, N2OImportResult importResults) throws OWLOntologyCreationException, IOException, InterruptedException, ExecutionException, N2OException { - return prepareCSVFilesForImport(url, importdir, importResults, true, null); + return prepareCSVFilesForImport(url, importdir, importResults, true, null, null); } - public N2OCSVWriter prepareCSVFilesForImport(String url, File importdir, N2OImportResult importResults, Boolean enableReasoning, String annotation_iri) throws OWLOntologyCreationException, IOException, InterruptedException, ExecutionException, N2OException { + public N2OCSVWriter prepareCSVFilesForImport(String url, File importdir, N2OImportResult importResults, Boolean enableReasoning, String annotation_iri, String csvPostfix) throws OWLOntologyCreationException, IOException, InterruptedException, ExecutionException, N2OException { logger.log("Loading Ontology"); OWLOntology o = OWLManager.createOWLOntologyManager().loadOntologyFromOntologyDocument(getOntologyIRI(url, importdir)); logger.log("Size ontology: " + o.getAxiomCount()); @@ -38,7 +38,7 @@ public N2OCSVWriter prepareCSVFilesForImport(String url, File importdir, N2OImpo logger.log("Loading in Database: " + importdir.getAbsolutePath()); - N2OCSVWriter csvWriter = new N2OCSVWriter(ontologyImporter.getImportManager(), ontologyImporter.getRelationTypeCounter(), importdir); + N2OCSVWriter csvWriter = new N2OCSVWriter(ontologyImporter.getImportManager(), ontologyImporter.getRelationTypeCounter(), importdir, csvPostfix); csvWriter.exportOntologyToCSV(); return csvWriter; } diff --git a/src/main/java/ebi/spot/neo4j2owl/importer/N2OImporterRunner.java b/src/main/java/ebi/spot/neo4j2owl/importer/N2OImporterRunner.java index 26e20f8..aa194a4 100644 --- a/src/main/java/ebi/spot/neo4j2owl/importer/N2OImporterRunner.java +++ b/src/main/java/ebi/spot/neo4j2owl/importer/N2OImporterRunner.java @@ -14,11 +14,15 @@ public static void main(String[] args) { File importdir = new File(args[2]); Boolean enableReasoning = true; String annotation_iri = null; + String csvPostfix = null; if (args.length > 3) { enableReasoning = Boolean.parseBoolean(args[3]); annotation_iri = args[4]; } + if (args.length > 5) { + csvPostfix = args[5]; + } if (config.equals("none")) { config = null; @@ -28,7 +32,7 @@ public static void main(String[] args) { N2OImportResult importResults = new N2OImportResult(); try { importService.prepareConfig(config, importdir); - N2OCSVWriter csvWriter = importService.prepareCSVFilesForImport(url, importdir, importResults, enableReasoning, annotation_iri); + N2OCSVWriter csvWriter = importService.prepareCSVFilesForImport(url, importdir, importResults, enableReasoning, annotation_iri, csvPostfix); File cypherDir = new File(importdir, "transactions"); if (!cypherDir.isDirectory()) { boolean created = cypherDir.mkdir(); diff --git a/src/main/java/ebi/spot/neo4j2owl/importer/N2OUtils.java b/src/main/java/ebi/spot/neo4j2owl/importer/N2OUtils.java index e885e3b..42592d0 100644 --- a/src/main/java/ebi/spot/neo4j2owl/importer/N2OUtils.java +++ b/src/main/java/ebi/spot/neo4j2owl/importer/N2OUtils.java @@ -62,11 +62,10 @@ public static Object extractValueFromOWLAnnotationValue(OWLAnnotationValue aval) } return "neo4j2owl_UnknownValue"; } - - - public static void writeToFile(File dir, Map> dataout, N2OCSVWriter.CSV_TYPE nodeclass) throws N2OException { + + public static void writeToFile(File dir, Map> dataout, N2OCSVWriter.CSV_TYPE nodeclass, String csvPostfix) throws N2OException { for (String type : dataout.keySet()) { - File f = constructFileHandle(dir, nodeclass.name, type); + File f = constructFileHandle(dir, nodeclass.name, type, csvPostfix); try { FileUtils.writeLines(f, dataout.get(type)); } catch (IOException e) { @@ -74,9 +73,12 @@ public static void writeToFile(File dir, Map> dataout, N2OC } } } - - static File constructFileHandle(File dir, String nodeclass, String type) { - return new File(dir, nodeclass + "_" + type + N2OStatic.CSV_EXTENSION); + + static File constructFileHandle(File dir, String nodeclass, String type, String namePostfix) { + if(!namePostfix.isEmpty() && !namePostfix.startsWith("_")) { + namePostfix = "_" + namePostfix; + } + return new File(dir, nodeclass + namePostfix + "_" + type + N2OStatic.CSV_EXTENSION); } public static String render(OWLClassExpression ce) { From 5356dc9869191aec69f607202543152e45017c13 Mon Sep 17 00:00:00 2001 From: hkir-dev Date: Fri, 23 Aug 2024 13:53:24 +0100 Subject: [PATCH 5/7] exluded caffeine jar from jar dependencies --- pom.xml | 1 + 1 file changed, 1 insertion(+) diff --git a/pom.xml b/pom.xml index 5fb8ac2..a3625e7 100644 --- a/pom.xml +++ b/pom.xml @@ -104,6 +104,7 @@ com.github.ben-manes.caffeine caffeine 2.3.3 + provided From fa4e24d240a450bc6ec2b5ec6c777ac46fedb41b Mon Sep 17 00:00:00 2001 From: hkir-dev Date: Sat, 31 Aug 2024 21:54:38 +0100 Subject: [PATCH 6/7] relation pagination by type --- .../java/ebi/spot/neo4j2owl/N2OProcedure.java | 4 +-- .../neo4j2owl/exporter/N2OExportService.java | 31 +++++++++++++------ 2 files changed, 24 insertions(+), 11 deletions(-) diff --git a/src/main/java/ebi/spot/neo4j2owl/N2OProcedure.java b/src/main/java/ebi/spot/neo4j2owl/N2OProcedure.java index f63f005..4a8ae69 100644 --- a/src/main/java/ebi/spot/neo4j2owl/N2OProcedure.java +++ b/src/main/java/ebi/spot/neo4j2owl/N2OProcedure.java @@ -73,10 +73,10 @@ public Stream exportOWLNodes(@Name("skip") Long skip, @Name("lim @SuppressWarnings("unused") @Procedure(mode = Mode.WRITE) - public Stream exportOWLEdges(@Name("skip") Long skip, @Name("limit") Long limit) { + public Stream exportOWLEdges(@Name("relationType") String relationType) { logger.resetTimer(); N2OExportService exportService = new N2OExportService(db); - N2OReturnValue result = exportService.owl2ExportEdges(skip, limit); + N2OReturnValue result = exportService.owl2ExportEdges(relationType); return Stream.of(result); } diff --git a/src/main/java/ebi/spot/neo4j2owl/exporter/N2OExportService.java b/src/main/java/ebi/spot/neo4j2owl/exporter/N2OExportService.java index e5febde..c7a2187 100644 --- a/src/main/java/ebi/spot/neo4j2owl/exporter/N2OExportService.java +++ b/src/main/java/ebi/spot/neo4j2owl/exporter/N2OExportService.java @@ -50,7 +50,12 @@ public class N2OExportService { // static IRIManager iriManager = new IRIManager(); private N2OExportManager n2OEntityManager; private Set qsls_with_no_matching_properties; - + + private final static String SUBCLASS_OF = "subclassOf"; + private final static String INSTANCE_OF = "instanceOf"; + private final static String ANNOTATION_PROPERTY = "annotationProperty"; + private final static String OBJECT_PROPERTY = "objectProperty"; + public N2OExportService(GraphDatabaseService db) { this.db = db; } @@ -115,7 +120,7 @@ public N2OReturnValue owl2ExportNodes(Long skip, Long limit) { return returnValue; } - public N2OReturnValue owl2ExportEdges(Long skip, Long limit) { + public N2OReturnValue owl2ExportEdges(String relationType) { n2OEntityManager = new N2OExportManager(); qsls_with_no_matching_properties = new HashSet<>(); logger.resetTimer(); @@ -125,14 +130,22 @@ public N2OReturnValue owl2ExportEdges(Long skip, Long limit) { OWLOntologyManager man = OWLManager.createOWLOntologyManager(); OWLOntology o = man.createOntology(); - findEntities(skip, limit); - addRelation(o, N2OStatic.RELTYPE_SUBCLASSOF); - addRelation(o, N2OStatic.RELTYPE_INSTANCEOF); - for (String rel_qsl : getRelations(OWLAnnotationProperty.class)) { - addRelation(o, rel_qsl); + findEntities(0L, Long.MAX_VALUE); + if (relationType == null || relationType.isEmpty() || relationType.equals(SUBCLASS_OF)) { + addRelation(o, N2OStatic.RELTYPE_SUBCLASSOF); } - for (String rel_qsl : getRelations(OWLObjectProperty.class)) { - addRelation(o, rel_qsl); + if (relationType == null || relationType.isEmpty() || relationType.equals(INSTANCE_OF)) { + addRelation(o, N2OStatic.RELTYPE_INSTANCEOF); + } + if (relationType == null || relationType.isEmpty() || relationType.equals(ANNOTATION_PROPERTY)) { + for (String rel_qsl : getRelations(OWLAnnotationProperty.class)) { + addRelation(o, rel_qsl); + } + } + if (relationType == null || relationType.isEmpty() || relationType.equals(OBJECT_PROPERTY)) { + for (String rel_qsl : getRelations(OWLObjectProperty.class)) { + addRelation(o, rel_qsl); + } } ByteArrayOutputStream os = new ByteArrayOutputStream(); // new FileOutputStream(new File(fileName)) man.saveOntology(o, new RDFXMLDocumentFormat(), os); From 73fea975b90bdbbea0e7d269f680fa48a75af746 Mon Sep 17 00:00:00 2001 From: hkir-dev Date: Sun, 1 Sep 2024 10:47:28 +0100 Subject: [PATCH 7/7] extra chunking for object relations --- .../java/ebi/spot/neo4j2owl/N2OProcedure.java | 4 +-- .../neo4j2owl/exporter/N2OExportService.java | 27 +++++++++++++++++-- 2 files changed, 27 insertions(+), 4 deletions(-) diff --git a/src/main/java/ebi/spot/neo4j2owl/N2OProcedure.java b/src/main/java/ebi/spot/neo4j2owl/N2OProcedure.java index 4a8ae69..a1a8d12 100644 --- a/src/main/java/ebi/spot/neo4j2owl/N2OProcedure.java +++ b/src/main/java/ebi/spot/neo4j2owl/N2OProcedure.java @@ -73,10 +73,10 @@ public Stream exportOWLNodes(@Name("skip") Long skip, @Name("lim @SuppressWarnings("unused") @Procedure(mode = Mode.WRITE) - public Stream exportOWLEdges(@Name("relationType") String relationType) { + public Stream exportOWLEdges(@Name("relationType") String relationType, @Name("currentChunk") int currentChunk, @Name("limit") int chunkCount) { logger.resetTimer(); N2OExportService exportService = new N2OExportService(db); - N2OReturnValue result = exportService.owl2ExportEdges(relationType); + N2OReturnValue result = exportService.owl2ExportEdges(relationType, currentChunk, chunkCount); return Stream.of(result); } diff --git a/src/main/java/ebi/spot/neo4j2owl/exporter/N2OExportService.java b/src/main/java/ebi/spot/neo4j2owl/exporter/N2OExportService.java index c7a2187..d9ab03d 100644 --- a/src/main/java/ebi/spot/neo4j2owl/exporter/N2OExportService.java +++ b/src/main/java/ebi/spot/neo4j2owl/exporter/N2OExportService.java @@ -120,7 +120,7 @@ public N2OReturnValue owl2ExportNodes(Long skip, Long limit) { return returnValue; } - public N2OReturnValue owl2ExportEdges(String relationType) { + public N2OReturnValue owl2ExportEdges(String relationType, int currentChunk, int chunkCount) { n2OEntityManager = new N2OExportManager(); qsls_with_no_matching_properties = new HashSet<>(); logger.resetTimer(); @@ -143,7 +143,9 @@ public N2OReturnValue owl2ExportEdges(String relationType) { } } if (relationType == null || relationType.isEmpty() || relationType.equals(OBJECT_PROPERTY)) { - for (String rel_qsl : getRelations(OWLObjectProperty.class)) { + Set objRelations = getRelations(OWLObjectProperty.class); + List> chunks = splitSet(objRelations, chunkCount); + for (String rel_qsl : chunks.get(currentChunk)) { addRelation(o, rel_qsl); } } @@ -492,4 +494,25 @@ private int safeAdd(int number, int increment, int maxValue) { // test < number on integer overflow return (test < number || test > maxValue) ? maxValue : test; } + + + /** + * Splits given set into given number of chunks. + * @param set to split + * @param numChunks number of chunks + * @return List of subsets + */ + public static List> splitSet(Set set, int numChunks) { + List> chunks = new ArrayList<>(); + List list = new ArrayList<>(set); + int chunkSize = (int) Math.ceil((double) list.size() / numChunks); + + for (int i = 0; i < list.size(); i += chunkSize) { + int end = Math.min(list.size(), i + chunkSize); + chunks.add(new HashSet<>(list.subList(i, end))); + } + + return chunks; + } + }