consolidated changes for tests on dydralxcbase : -u format, path test…

…s, successor tests, ...

accounts-api/accounts/openrdf-sesame/authorization/authorization.sh

@@ -9,44 +9,44 @@ set -e
 queryURL="${STORE_URL}/system/accounts/${STORE_ACCOUNT}/authorization"
 
 
-${CURL} -s -w "%{http_code}\n" -f -s -S -X PATCH --user "${STORE_TOKEN}:" \
+${CURL} -s -w "%{http_code}\n" -f -s -S -X PATCH --user ":${STORE_TOKEN}" \
   --data-binary @authorization.trig \
   -H "Content-Type: application/trig" \
   "${STORE_URL}/${STORE_ACCOUNT}/system/service" \
   | test_patch_success
 
 
-${CURL} -s -H "Accept: application/sparql-results+json"  --user "${STORE_TOKEN}:" \
+${CURL} -s -H "Accept: application/sparql-results+json"  --user ":${STORE_TOKEN}" \
   "${queryURL}?role=http://dydra.com/account/other&target=http://dydra.com/jhacker/request-repository" \
   | fgrep -q Read
 
-${CURL} -s -H "Accept: application/sparql-results+json"  --user "${STORE_TOKEN}:" \
+${CURL} -s -H "Accept: application/sparql-results+json"  --user ":${STORE_TOKEN}" \
   "${queryURL}?role=http://dydra.com/user/authenticated&target=http://dydra.com/jhacker/request-repository" \
   | fgrep -q Read
 
 ## anonymous access to the anonymous repository only
-${CURL} -s -H "Accept: application/sparql-results+json"  --user "${STORE_TOKEN}:" \
+${CURL} -s -H "Accept: application/sparql-results+json"  --user ":${STORE_TOKEN}" \
   "${queryURL}?role=http://xmlns.com/foaf/0.1/Agent&target=http://dydra.com/jhacker/request-repository" \
   | fgrep -q "[ ]"
-${CURL} -s -H "Accept: application/sparql-results+json"  --user "${STORE_TOKEN}:" \
+${CURL} -s -H "Accept: application/sparql-results+json"  --user ":${STORE_TOKEN}" \
   "${queryURL}?role=http://xmlns.com/foaf/0.1/Agent&target=http://dydra.com/jhacker/anonymous-repository" \
   | fgrep -q Read
 
 ## access through a view
-${CURL} -s -H "Accept: application/sparql-results+json"  --user "${STORE_TOKEN}:" \
+${CURL} -s -H "Accept: application/sparql-results+json"  --user ":${STORE_TOKEN}" \
   "${queryURL}?role=http://xmlns.com/foaf/0.1/Agent&target=http://dydra.com/jhacker/request-repository/anonview&mode=Execute" \
   | fgrep -q Execute
 
-${CURL} -s -H "Accept: application/sparql-results+json"  --user "${STORE_TOKEN}:" \
+${CURL} -s -H "Accept: application/sparql-results+json"  --user ":${STORE_TOKEN}" \
   "${queryURL}?role=http://dydra.com/jhacker/request-repository/anonview&target=http://dydra.com/jhacker/request-repository" \
   | fgrep -q Read
 
 ## access via a repository
-${CURL} -s -H "Accept: application/sparql-results+json"  --user "${STORE_TOKEN}:" \
+${CURL} -s -H "Accept: application/sparql-results+json"  --user ":${STORE_TOKEN}" \
   "${queryURL}?role=http://dydra.com/other/other&target=http://dydra.com/jhacker/request-repository" \
   | fgrep -q Read
 
 ## access via another view
-${CURL} -s -H "Accept: application/sparql-results+json"  --user "${STORE_TOKEN}:" \
+${CURL} -s -H "Accept: application/sparql-results+json"  --user ":${STORE_TOKEN}" \
   "${queryURL}?role=http://dydra.com/jhacker/request-repository/view&target=http://dydra.com/jhacker/request-repository" \
   | fgrep -q Read

define.sh

@@ -56,10 +56,10 @@ fi
 
 export STORE_SITE="dydra.com"           # the abstract site name
 if [[ "" == "${STORE_ACCOUNT}" ]]
-then export STORE_ACCOUNT="openrdf-sesame"
+then export STORE_ACCOUNT="test"
 fi
 if [[ "" == "${STORE_REPOSITORY}" ]]
-then export STORE_REPOSITORY="mem-rdf"
+then export STORE_REPOSITORY="test"
 fi
 if [[ "" == "${STORE_COLLABORATOR}" ]]
 then export STORE_COLLABORATOR="jhacker"
@@ -70,8 +70,6 @@ export STORE_REPOSITORY_WRITABLE="${STORE_REPOSITORY}-write"
 export STORE_REPOSITORY_PROVENANCE="${STORE_REPOSITORY}-provenance"
 export STORE_REPOSITORY_PUBLIC="${STORE_REPOSITORY}-public"
 export STORE_REPOSITORY_REVISIONED="${STORE_REPOSITORY}-revisioned"
-export STORE_REPOSITORY_CLASS_DEFAULT="lmdb-quad-repository"
-export STORE_REVISIONED_REPOSITORY_CLASS_DEFAULT="lmdb-revisioned-repository"
 if [[ "" == "${GRAPH_STORE_PATCH_LEGACY}" ]]
 then export GRAPH_STORE_PATCH_LEGACY=true
 fi
@@ -290,7 +288,7 @@ function initialize_account () {
 ${CURL} -w "%{http_code}\n" -L -f -s -X POST \
      -H "Content-Type: application/n-quads" --data-binary @- \
      -u ":${STORE_TOKEN}" \
-     ${STORE_URL}/${STORE_ACCOUNT}/system <<EOF
+     ${STORE_URL}/${STORE_ACCOUNT}/system/service <<EOF
 <http://dydra.com/accounts/openrdf-sesame> <urn:dydra:baseIRI> <http://dydra.com/accounts/openrdf-sesame> <http://dydra.com/accounts/openrdf-sesame> .
 EOF
 }
@@ -315,7 +313,7 @@ function initialize_repository_configuration () {
 ${CURL} -w "%{http_code}\n" -L -f -s -X POST \
      -H "Content-Type: application/n-quads" --data-binary @- \
      -u ":${STORE_TOKEN}" \
-     ${STORE_URL}/${STORE_ACCOUNT}/system <<EOF
+     ${STORE_URL}/${STORE_ACCOUNT}/system/service <<EOF
 <http://${STORE_SITE}/accounts/openrdf-sesame/repositories/mem-rdf> <urn:dydra:baseIRI> <http://www.openrdf.org/mem-rdf> <http://${STORE_SITE}/accounts/openrdf-sesame/repositories/mem-rdf> .
 <http://${STORE_SITE}/accounts/openrdf-sesame/repositories/mem-rdf> <urn:dydra:skolemize> "false"^^<http://www.w3.org/2001/XMLSchema#boolean> <http://${STORE_SITE}/accounts/openrdf-sesame/repositories/mem-rdf> .
 <http://${STORE_SITE}/accounts/openrdf-sesame/repositories/mem-rdf> <urn:dydra:defaultContextTerm> <urn:dydra:default> <http://${STORE_SITE}/accounts/openrdf-sesame/repositories/mem-rdf> .
@@ -484,7 +482,7 @@ function curl_sparql_request () {
 
   echo ${CURL} -L -f -s "${curl_args[@]}" ${curl_url} > $ECHO_OUTPUT
   mkdir -p /tmp/test/
-  ${CURL} -L -f -s "${curl_args[@]}" ${curl_url}
+  ${CURL} -L -f -s  "${curl_args[@]}" ${curl_url}
 }
 
 
@@ -519,6 +517,7 @@ function curl_sparql_view () {
     case "$1" in
       --account) account="${2}"; shift 2;;
       default|DEFAULT) graph="default"; shift 1;;
+      -D) curl_args+=("${1}" "${2}"); shift 2;;
       --graph) if [[ "" == "${2}" ]] ; then graph=""; else graph="graph=${2}"; fi;  shift 2;;
       -H) case "$2" in
           Accept:*) accept_media_type[1]="${2}"; shift 2;;
@@ -833,8 +832,9 @@ function create_repository() {
   local -a curl_args=()
   local -a account="${STORE_ACCOUNT}"
   local -a repository="new"
-  local -a class="${STORE_REPOSITORY_CLASS_DEFAULT}"
+  local -a class="${STORE_REPOSITORY_CLASS}"
   local -a temporal_properties=""
+  local -a event_properties=""
   local -a time_series_properties=""
   while [[ "$#" > 0 ]] ; do
     case "$1" in
@@ -847,7 +847,7 @@ function create_repository() {
     esac
   done
   local -a URL="${STORE_URL}/system/accounts/${account}/repositories"
-  echo "create repository: ${account}/${repository}, class: ${class}" > $ECHO_OUTPUT
+  echo "create repository: ${account}/${repository}, class: ${class}" > /dev/tty > $ECHO_OUTPUT
   ${CURL} -w "%{http_code}\n" -f -s -X POST "${curl_args[@]}" \
      -H "Content-Type: application/json" \
      -H "Accept: application/n-quads" \
@@ -1033,6 +1033,15 @@ function set_store_features () {
     then export STORE_INDEXED_EVENTS="true"
     else export STORE_INDEXED_EVENTS="false"
   fi
+
+  if [[ "true" == ${STORE_INDEXED_TIMES} ]]
+  then
+    export STORE_REVISIONED_REPOSITORY_CLASS=lmdb-revisioned-repository
+    export STORE_REPOSITORY_CLASS=lmdb-quad-repository
+  else
+    export STORE_REVISIONED_REPOSITORY_CLASS=revisioned-lmdb-repository
+    export STORE_REPOSITORY_CLASS=lmdb-repository
+  fi
 }
 
 export -f create_account

extensions/admin/revisions/project-history.sh

@@ -4,11 +4,14 @@ source test-if-revisioned-and-common-functions.sh
 
 echo "initial test after delete revisions" > ${INFO_OUTPUT}
 delete_revisions --repository ${repository} | fgrep -x 200 > ${GREP_OUTPUT}
+echo "deleted" > ${INFO_OUTPUT}
 repository_number_of_revisions --repository ${repository} | fgrep -x "1" > ${GREP_OUTPUT}
 
+echo "check_empty_repository top" > ${INFO_OUTPUT}
 check_empty_repository
 make_base_revision_ordinals
 
+echo "add_quad 1" > ${INFO_OUTPUT}
 add_quad 1
 repository_number_of_revisions --repository ${repository} | fgrep -x "2" > ${GREP_OUTPUT}
 curl_graph_store_get --repository ${repository} | tr -s '\n' '\t' \

extensions/admin/revisions/test-if-revisioned-and-common-functions.sh

@@ -79,8 +79,15 @@ function make_base_revision_ordinals() {
 }
 
 function check_empty_repository() {
+  echo "check empty repository" > ${INFO_OUTPUT}
   rev="HEAD"
   result=$(curl_graph_store_get_code_nofail --repository ${repository} revision-id=${rev} 2>&1 > /dev/null)
   echo "result: ${result}" > ${INFO_OUTPUT}
   test "$result" -eq "404"
 }
+
+function check_empty_repository() {
+  echo "check empty repository" > ${INFO_OUTPUT}
+  curl_graph_store_get -w '%{http_code}\n' --repository ${repository} revision-id=${rev} \
+  | test_not_found
+}

extensions/admin/revisions/trim-history/block2-and-block3.1-tests.sh

@@ -46,8 +46,13 @@ echo "check visibilities of quads in revisions ${rev}: empty revision" > ${INFO_
 # HEAD~4 is tail and gives 404 already
 #curl_graph_store_get --repository ${repository} revision-id=${rev}| tr -s '\n' '\t' \
     #    | fgrep -v "object-3.1" | fgrep -v "object-3.2" | fgrep -v "object-2.1" | fgrep -v "object-2.2" > ${GREP_OUTPUT}
-result=$(curl_graph_store_get_code_nofail --repository ${repository} revision-id=${rev} 2>&1 > /dev/null)
-test "$result" -eq "404"
+
+curl_graph_store_get -w '%{http_code}\n' --repository ${repository} revision-id=${rev} \
+  | test_not_found
+# fails on osx
+#result=$(curl_graph_store_get_code_nofail --repository ${repository} revision-id=${rev} 2>&1 > /dev/null)
+#test "$result" -eq "404"
+
 rev="HEAD~3"
 echo "check visibilities of quads in revisions ${rev}" > ${INFO_OUTPUT}
 curl_graph_store_get --repository ${repository} revision-id=${rev} | tr -s '\n' '\t' \

extensions/admin/revisions/trim-history/block3.2-tests.sh

@@ -51,8 +51,13 @@ echo "check visibilities of quads in revisions ${rev}: empty revision" > ${INFO_
 # HEAD~4 is tail and gives 404 already
 #curl_graph_store_get --repository ${repository} revision-id=${rev}| tr -s '\n' '\t' \
     #    | fgrep -v "object-3.1" | fgrep -v "object-3.2" | fgrep -v "object-2.1" | fgrep -v "object-2.2" > ${GREP_OUTPUT}
-result=$(curl_graph_store_get_code_nofail --repository ${repository} revision-id=${rev} 2>&1 > /dev/null)
-test "$result" -eq "404"
+
+curl_graph_store_get -w '%{http_code}\n' --repository ${repository} revision-id=${rev} \
+  | test_not_found
+# fails on osx
+#result=$(curl_graph_store_get_code_nofail --repository ${repository} revision-id=${rev} 2>&1 > /dev/null)
+#test "$result" -eq "404"
+
 rev="HEAD~4"
 echo "check visibilities of quads in revisions ${rev}" > ${INFO_OUTPUT}
 curl_graph_store_get --repository ${repository} revision-id=${rev} | tr -s '\n' '\t' \

extensions/admin/revisions/trim-history/block4.1-tests.sh

@@ -60,8 +60,13 @@ echo "check visibilities of quads in revisions ${rev}: empty revision" > ${INFO_
 # HEAD~4 is tail and gives 404 already
 #curl_graph_store_get --repository ${repository} revision-id=${rev} | tr -s '\n' '\t' \
     #    | fgrep -v "object-3.1" | fgrep -v "object-3.2" | fgrep -v "object-2.1" | fgrep -v "object-2.2" > ${GREP_OUTPUT}
-result=$(curl_graph_store_get_code_nofail --repository ${repository} revision-id=${rev} 2>&1 > /dev/null)
-test "$result" -eq "404"
+
+curl_graph_store_get -w '%{http_code}\n' --repository ${repository} revision-id=${rev} \
+  | test_not_found
+# fails on osx
+#result=$(curl_graph_store_get_code_nofail --repository ${repository} revision-id=${rev} 2>&1 > /dev/null)
+#test "$result" -eq "404"
+
 rev="HEAD~5"
 echo "check visibilities of quads in revisions ${rev}" > ${INFO_OUTPUT}
 curl_graph_store_get --repository ${repository} revision-id=${rev} | tr -s '\n' '\t' \

extensions/admin/revisions/trim-history/block4.2-tests.sh

@@ -54,8 +54,13 @@ echo "check visibilities of quads in revisions ${rev}: empty revision" > ${INFO_
 # HEAD~4 is tail and gives 404 already
 #curl_graph_store_get --repository ${repository} revision-id=${rev} | tr -s '\n' '\t' \
     #    | fgrep -v "object-3.1" | fgrep -v "object-3.2" | fgrep -v "object-2.1" | fgrep -v "object-2.2" > ${GREP_OUTPUT}
-result=$(curl_graph_store_get_code_nofail --repository ${repository} revision-id=${rev} 2>&1 > /dev/null)
-test "$result" -eq "404"
+
+curl_graph_store_get -w '%{http_code}\n' --repository ${repository} revision-id=${rev} \
+  | test_not_found
+# fails on osx
+#result=$(curl_graph_store_get_code_nofail --repository ${repository} revision-id=${rev} 2>&1 > /dev/null)
+#test "$result" -eq "404"
+
 rev="HEAD~4"
 echo "check visibilities of quads in revisions ${rev}" > ${INFO_OUTPUT}
 curl_graph_store_get --repository ${repository} revision-id=${rev} | tr -s '\n' '\t' \

extensions/admin/revisions/trim-history/block4.3-tests.sh

@@ -71,8 +71,13 @@ echo "check visibilities of quads in revisions ${rev}: empty revision" > ${INFO_
 # HEAD~4 is tail and results in HTTP 404 already
 #curl_graph_store_get --repository ${repository} revision-id=${rev} | tr -s '\n' '\t' \
 #    | fgrep -v "object-3.1" | fgrep -v "object-3.2" | fgrep -v "object-2.1" | fgrep -v "object-2.2" > ${GREP_OUTPUT}
-result=$(curl_graph_store_get_code_nofail --repository ${repository} revision-id=${rev} 2>&1 > /dev/null)
-test "$result" -eq "404"
+
+curl_graph_store_get -w '%{http_code}\n' --repository ${repository} revision-id=${rev} \
+  | test_not_found
+# fails on osx
+#result=$(curl_graph_store_get_code_nofail --repository ${repository} revision-id=${rev} 2>&1 > /dev/null)
+#test "$result" -eq "404"
+
 rev="HEAD~5"
 echo "check visibilities of quads in revisions ${rev}" > ${INFO_OUTPUT}
 curl_graph_store_get --repository ${repository} revision-id=${rev} | tr -s '\n' '\t' \

extensions/inference/README.md

@@ -14,7 +14,7 @@ It is intended to cover the following mechanisms:
 
 ## Pattern-Directed entailment
 
-The library's construct and n3 rules are are compiled into filter which activated
+The library's construct and n3 rules are are compiled into filters which activated
 BGP transformations. These tests cover
 
 - test different rule formats for simple rdfs entailment

extensions/quality-of-service/README.txt

@@ -12,24 +12,45 @@ The resource allocation is determined by the upstream host which is specified in
 The upstream hosts are recorded in the system/system repository.
 From these the upstream entries are generated and manually incorporated in
 
-    /opt/rails/etc/nginx/dydra/upstream.conf.
+    /opt/rails/etc/nginx/dydra/upstream/spocq-upstream.conf
 
 view specific locations are generated from the contents of the
 <http://dydra.com/quality-of-service/views> and <http://dydra.com/quality-of-service> graphs
 in an account's system repository.
-These are stored in the account's respective .conf file in the /opt/rails/etc/nginx/dydra/qos/ directory.
-The general /opt/rails/etc/nginx/dydra/upstream.conf file must include a reference to the account's location configuration file in order for them to be active.
-This file is generated as a side-effect of updating the account's quality-of-service repository
+These are stored in the account's respective .conf file in the directory
+
+    /opt/rails/etc/nginx/dydra/spocq/qos/
+
+These are included by a general qos route configuration file
+
+    /opt/rails/etc/nginx/dydra/spocq/qos.conf
+
+It must include a reference to the account's location configuration file in order for them to be active.
+The account configuration file is generated as a side-effect of updating the account's quality-of-service repository
+
+QOS for non-view locations is managed by recognizing a quality-of-service header.
+This is passed through an identity map to validate the header and to establish a default which falls back to the general pattern-based locations.
+
+  map $http_quality_of_service $upstream {
+    Administration Administration;
+    Queued Queued;
+    SPARQL SPARQL;
+    Scheduled Scheduled;
+    Service Service;
+    default spocq; # legacy
+  }
 
 #### Systemd configuration
 
-Each upstream processor is defined by a systemd entry which determines the executed binary, the initialization arguments and the configuration file.
+Each upstream processor is defined by a systemd .service file which determines the executed binary, the initialization arguments and the configuration file.
 The binaries are all spocq-server.
 The arguments and configurations allow for
 - varied heap sizes
 - maximum simultaneous request counts
 - maximum pending request queue lengths.
 
+each instance is associated with its own /opt/spocq/init-http-127-0-0-1-???.sxp initialization file.
+
 
 #### Account configuration
 

extensions/quality-of-service/update-qos-invalid.sh

@@ -18,8 +18,8 @@ curl_sparql_view -X PUT --account test --repository test -w "%{http_code}\n" \
 select * where {?s ?p ?o}
 EOF
 
-echo "invalid view specified" > $ECHO_OUTPUT
-curl_graph_store_update -X PUT -w "%{http_code}\n" --account test --repository quality-of-service -H "Content-Type: application/trig" <<EOF | test_bad_request
+echo "allow invalid view specified" > $ECHO_OUTPUT
+curl_graph_store_update -X PUT -w "%{http_code}\n" --account test --repository quality-of-service -H "Content-Type: application/trig" <<EOF | test_success
 <http://dydra.com/quality-of-service> {
     <http://dydra.com/quality-of-service/class/Test> <http://dydra.com/quality-of-service/class> <http://dydra.com/quality-of-service/class/Application> .
 }

extensions/quality-of-service/update-qos.sh

@@ -12,20 +12,24 @@ set -e
 # for background:
 # (spocq.i::REPOSITORY-VIEW-DEFINITIONS (repository "test/test"))
 
-echo "create repository" > $ECHO_OUTPUT
+echo "create qos repository" > $ECHO_OUTPUT
+create_repository --repository quality-of-service \
+  | test_success
+
+echo "create view" > $ECHO_OUTPUT
 curl_sparql_view -X PUT --account test --repository test -w "%{http_code}\n" \
     -H "Content-Type: application/sparql-query" \
     --data-binary @- test_qos <<EOF | test_put_success
 select * where {?s ?p ?o}
 EOF
 
-echo "SPARQL qos specified" > $ECHO_OUTPUT
+echo "Queued qos specified" > $ECHO_OUTPUT
 curl_graph_store_update -X PUT -w "%{http_code}\n" --account test --repository quality-of-service -H "Content-Type: application/trig" <<EOF | test_success
 <http://dydra.com/quality-of-service> {
     <http://dydra.com/quality-of-service/class/Test> <http://dydra.com/quality-of-service/class> <http://dydra.com/quality-of-service/class/Application> .
 }
 <http://dydra.com/quality-of-service> {
-    <http://dydra.com/quality-of-service/class/Application> <http://dydra.com/quality-of-service/quality> <http://dydra.com/quality-of-service/class/SPARQL> .
+    <http://dydra.com/quality-of-service/class/Application> <http://dydra.com/quality-of-service/quality> <http://dydra.com/quality-of-service/class/Queued> .
 }
 
 <http://dydra.com/quality-of-service/views> {
@@ -38,7 +42,7 @@ echo "retrieve SPARQL qos" > $ECHO_OUTPUT
 curl_sparql_view -X GET --account test --repository test -w "%{http_code}\n" \
     -H "Accept: application/sparql-results+json" \
     -D - test_qos \
-    | fgrep Service-Quality | fgrep -q SPARQL
+    | fgrep Service-Quality | fgrep -q Queued
 
 
 echo "Scheduled qos specified" > $ECHO_OUTPUT
@@ -68,12 +72,13 @@ echo "clear the specification" > $ECHO_OUTPUT
 curl_graph_store_update -X PUT -w "%{http_code}\n" --account test --repository quality-of-service -H "Content-Type: application/trig" <<EOF | test_success
 EOF
 
+# SPARQL is the default established by the nginx general view location
 sleep 10
 echo "retrieve no qos" > $ECHO_OUTPUT
 curl_sparql_view -X GET --account test --repository test -w "%{http_code}\n" \
     -H "Accept: application/sparql-results+json" \
     -D - test_qos \
-     | fgrep Service-Quality | fgrep -q Unspecified
+     | fgrep Service-Quality | fgrep -q SPARQL
 
 
 echo "delete view" > $ECHO_OUTPUT

extensions/sparql-protocol/describe/describe-properties.sh

@@ -29,7 +29,7 @@ EOF
 curl_sparql_request \
  --repository "${STORE_REPOSITORY}-write" \
  -H "Accept: application/sparql-results+json" <<EOF \
- jq '.results.bindings[].o'  | tr -s '\n' '\t'  \
+ | jq '.results.bindings[].o'  | tr -s '\n' '\t'  \
  | fgrep '"depth 0"' \
  | fgrep -v '"depth 1 by uri"' | fgrep -q '"depth 1 by blank node"'