[feat] add couch2git, converts databases to https://github.com/NeuroJ…

…SON-io/

bin/couch2git

@@ -0,0 +1,53 @@
+#!/bin/bash
+
+serverurl=https://neurojson.io:7777
+bowserurl=https://neurojson.org/db
+
+dbname=$1
+
+[[ $# -gt 1 ]] && serverurl=$2
+
+mkdir -p "$dbname"
+cd "$dbname"
+
+git init .
+
+currdate=$(date '+%Y-%m-%d_%H:%M:%S')
+
+cat << EOF > README.md
+# NeuroJSON database: $dbname
+- Database URL: $bowserurl/$dbname
+- REST URL: $serverurl/$dbname
+- Snapshot time: $currdate
+EOF
+
+git add README.md
+git commit -a -m "Initial commit"
+
+mkdir orig
+mkdir json
+cd json
+
+curl -s -X GET "$serverurl/$dbname/_changes"  | jq -c -r '.results | .[] | "\(.id)\t\(.changes | .[] | .rev)\t\(.deleted)"' | \
+while read doc rev isdel
+do
+  echo "processing: $doc $rev $isdel"
+
+  [[ "$isdel" == "true"  ]] && continue;
+
+  maxrev=`echo $rev | grep -o '^[0-9]*'`
+  curl -s -X GET "$serverurl/$dbname/$doc?revs_info=true"  | flatjson > "_${doc}_revs.json"
+  allrev=`cat "_${doc}_revs.json" | jq -r '._revs_info | .[] | .rev' | tac -r`
+
+  for revidx in $allrev
+  do
+    if [[ $maxrev -eq 1 ]]; then
+      mv "_${doc}_revs.json" "${doc}.json"
+    else
+      curl -s -X GET "$serverurl/$dbname/$doc?rev=$revidx" | flatjson > "${doc}.json"
+    fi
+
+    git add "${doc}.json"
+    git commit -a -m "Update $doc to $revidx"
+  done
+done

bin/neuroj

@@ -12,14 +12,26 @@
 #
 # Dependencies:
 #
-# For Linux and Mac OS:
+# For Ubuntu Linux:
 #   jq, curl, octave, jbids (https://github.com/NeuroJSON/jbids - including 4
 #   submodules under tools), libparallel-forkmanager-perl (for Parallel::ForkManager)
-#   libwww-perl (for LWP::UserAgent), libjson-xs-perl (for JSON::XS)
+#   libwww-perl (for LWP::UserAgent, only needed if curl is not installed), libjson-xs-perl (for JSON::XS)
+#
+# For MacOS: please run (you could prepend sudo to install for all users, or without for current user only)
+#   brew install jq curl octave
+#   perl -MCPAN -e 'install Parallel::ForkManager'
+#   perl -MCPAN -e 'install JSON::XS'
+#   perl -MCPAN -e 'install LWP::UserAgent'  # no need if curl is installed
 #
 # For Windows: please install cygwin64 (http://cygwin.com/) or MSYS2 (http://msys2.org/)
 #   please also install jq, curl, octave, jbids (https://github.com/NeuroJSON/jbids)
 #
+#   If using cygwin64, please choose to install the follow packages in the installer:
+#        jq, curl, perl-libwww-perl, perl-JSON-XS, octave
+#
+#   If using mingw64/msys2, please run
+#        pacman -S mingw64/mingw-w64-x86_64-jq curl perl-libwww perl-Parallel-ForkManager
+#
 # Specifically, this script requires Perl 5.x and curl (if curl is not install
 # in the path, it looks for the LWP::UserAgent perl modules).
 #
@@ -42,7 +54,7 @@ if ( @ARGV == 0 || grep( /^--help$/, @ARGV ) || grep( /^-h$/, @ARGV ) ) {
 }
 
 my ( $infolder, $outfolder, $dbname, $dsname, $threads, $dbuser, $dbpass );
-my %action = ();
+my %action    = ();
 my $serverurl = ( exists $ENV{"NEUROJSON_IO"} ) ? $ENV{"NEUROJSON_IO"} : "https://neurojson.io:7777";
 
 $threads = 4;
@@ -195,7 +207,7 @@ if ( exists( $action{"push"} ) && $dbname ne '' ) {
         } else {
             next;
         }
-        my $url = "$serverurl/$dbname/$ds";
+        my $url     = "$serverurl/$dbname/$ds";
         my $content = webpost( $url, "\@$outfolder/${ds}.json", $dbuser, $dbpass, '-X PUT' );
         if ( $content ne '' ) {
             print $content;
@@ -204,7 +216,7 @@ if ( exists( $action{"push"} ) && $dbname ne '' ) {
 }
 
 if ( exists( $action{"find"} ) && $dbname ne '' ) {
-    my $url = "$serverurl/$dbname/_find";
+    my $url     = "$serverurl/$dbname/_find";
     my $content = webpost( $url, $action{'find'}, $dbuser, $dbpass );
     if ( $content ne '' ) {
         print $content;

bin/tsv2json

@@ -22,7 +22,7 @@ if ( @ARGV < 1 ) {
 }
 
 $iscompress = ( defined $ENV{'NJPREP_ZIP_TSV'} ) ? $ENV{'NJPREP_ZIP_TSV'} : 0;
-$keymeta = 0;
+$keymeta    = 0;
 
 if ( @ARGV > 2 ) {
     $iscompress = $ARGV[2];
@@ -75,13 +75,13 @@ if ($iscompress) {
         if ( !( $keymeta == 1 && $hd =~ /age|sex|gender/i ) ) {
             $keylen = keys %{ $enum{$hd} };
             if ( $keylen < 256 ) {
-                $buf = pack( 'C*', @{ $data{$hd} } );
+                $buf      = pack( 'C*', @{ $data{$hd} } );
                 $datatype = 'uint8';
             } elsif ( $keylen < 65536 ) {
-                $buf = pack( 'S*', @{ $data{$hd} } );
+                $buf      = pack( 'S*', @{ $data{$hd} } );
                 $datatype = 'uint16';
             } else {
-                $buf = pack( 'L*', @{ $data{$hd} } );
+                $buf      = pack( 'L*', @{ $data{$hd} } );
                 $datatype = 'uint32';
             }
             eval("use Compress::Zlib; 1")
@@ -129,7 +129,7 @@ if (@header) {
 
 sub csvsplit {
     my $line = shift;
-    my $sep = ( shift or ',' );
+    my $sep  = ( shift or ',' );
 
     return () unless $line;