removing duplicates

VirtualFlyBrain · Oct 22, 2024 · e5f3a84 · e5f3a84
1 parent 83ed322
commit e5f3a84
Showing 1 changed file with 10 additions and 3 deletions.
diff --git a/src/uk/ac/ebi/vfb/neo4j/flybase2neo/add_refs_for_anat.py b/src/uk/ac/ebi/vfb/neo4j/flybase2neo/add_refs_for_anat.py
@@ -190,8 +190,15 @@ def gen_pub_links(self):
                             self.write_pub_link(d['short_form'], s, type=k)
 
     def commit(self):
-        self.node_writer.commit(chunk_length=1000, verbose=True)
-        self.edge_witer.commit(chunk_length=1000, verbose=True)
+    # Remove duplicates from the node writer statements
+    self.node_writer.statements = list(set(self.node_writer.statements))
+
+    # Remove duplicates from the edge writer statements
+    self.edge_writer.statements = list(set(self.edge_writer.statements))
+
+    # Commit the unique statements
+    self.node_writer.commit(chunk_length=1000, verbose=True)
+    self.edge_witer.commit(chunk_length=1000, verbose=True)
 
 def __main__():
     pl = pubLink(endpoint=args.endpoint,
@@ -201,4 +208,4 @@ def __main__():
     pl.gen_pub_links()
     pl.commit()
 
-__main__()
+__main__()