Make PubMed fetch incremental and idempotent

dspinellis · Feb 1, 2024 · 7f677b3 · 7f677b3
1 parent 7b43f05
commit 7f677b3
Showing 1 changed file with 3 additions and 1 deletion.
diff --git a/examples/common/fetch-pubmed.sh b/examples/common/fetch-pubmed.sh
@@ -8,7 +8,7 @@ set -eu
 
 BASE=https://ftp.ncbi.nlm.nih.gov/pubmed/baseline/
 
-mkdir pubmed-data
+mkdir -p pubmed-data
 cd pubmed-data
 
 # Obtain last baseline file from README.txt
@@ -18,5 +18,7 @@ last=$(curl --silent https://ftp.ncbi.nlm.nih.gov/pubmed/baseline/README.txt |
 for n in $(seq 1 $last) ; do
   file_name=$(printf 'pubmed24n%04d.xml.gz' $n)
 
+  test -r $file_name && continue
+
   curl --silent $BASE/$file_name >$file_name
 done