add identity search component to e2e test

Signed-off-by: linus-sun <[email protected]>
sigstore · Nov 22, 2024 · 53f6743 · 53f6743
1 parent b281a00
commit 53f6743
Show file tree

Hide file tree

Showing 5 changed files with 273 additions and 84 deletions.
diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
@@ -67,6 +67,22 @@ jobs:
   e2e-tests:
     name: Run end-to-end tests
     runs-on: ubuntu-latest
+    services:
+      mysql:
+        # sudo systemctl enable mysql.service
+        # sudo systemctl start mysql
+        image: mysql
+        env:
+          DATABASE_HOST: 127.0.0.1
+          # MYSQL_ROOT_PASSWORD: zaphod
+          MYSQL_ROOT_USER: root
+          MYSQL_DATABASE: test
+          MYSQL_USER: root
+          # MYSQL_PASSWORD: zaphod
+        ports:
+          - 3306:3306
+        options: --health-cmd="mysqladmin ping" --health-interval=10s --health-timeout=5s --health-retries=3
+
 
     steps: 
       - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
@@ -88,6 +104,11 @@ jobs:
         with:
           go-version-file: './go.mod'
           check-latest: true
-      - name: run Rekor end-to-end test
-        run: ./pkg/test/rekor_e2e/rekor_monitor_e2e_test.sh
+      # - name: run Rekor end-to-end test
+      #   run: ./pkg/test/rekor_e2e/rekor_monitor_e2e_test.sh
+      # - name: Create mariadb
+      #   run: | 
+      #     sudo /etc/init.d/mysql start
+      - name: run CT end-to-end test
+        run: ./pkg/test/ct_e2e/ct_monitor_e2e_test_manual.sh
 
diff --git a/pkg/test/ct_e2e/ct_e2e_test.sh b/pkg/test/ct_e2e/ct_e2e_test.sh
diff --git a/pkg/test/ct_e2e/ct_e2e_test.go → pkg/test/ct_e2e/ct_monitor_e2e_test.go b/pkg/test/ct_e2e/ct_e2e_test.go → pkg/test/ct_e2e/ct_monitor_e2e_test.go
@@ -26,6 +26,7 @@ import (
 	ctclient "github.com/google/certificate-transparency-go/client"
 	"github.com/google/certificate-transparency-go/jsonclient"
 	"github.com/sigstore/rekor-monitor/pkg/ct"
+	"github.com/sigstore/rekor-monitor/pkg/identity"
 )
 
 const (
@@ -37,7 +38,7 @@ const (
 )
 
 func TestCTConsistencyCheck(t *testing.T) {
-	fulcioClient, err := ctclient.New("http://localhost:8080/testlog", http.DefaultClient, jsonclient.Options{})
+	fulcioClient, err := ctclient.New("http://127.0.0.1:8080/testlog", http.DefaultClient, jsonclient.Options{})
 	if err != nil {
 		t.Errorf("error instantiating ct client: %v", err)
 	}
@@ -50,8 +51,20 @@ func TestCTConsistencyCheck(t *testing.T) {
 	tempLogInfoFileName := tempLogInfoFile.Name()
 	defer os.Remove(tempLogInfoFileName)
 
-	err = ct.RunConsistencyCheck(fulcioClient, tempLogInfoFileName)
+	_, _, err = ct.RunConsistencyCheck(fulcioClient, tempLogInfoFileName)
 	if err != nil {
 		t.Errorf("failed to successfully complete consistency check: %v", err)
 	}
+
+	_, err = ct.IdentitySearch(fulcioClient, 0, 1, identity.MonitoredValues{
+		CertificateIdentities: []identity.CertificateIdentity{
+			{
+				CertSubject: "test-cert-subject",
+				Issuers:     []string{},
+			},
+		},
+	})
+	if err != nil {
+		t.Errorf("failed to successfully complete identity search: %v", err)
+	}
 }
diff --git a/pkg/test/ct_e2e/ct_monitor_e2e_test_ctfe.sh b/pkg/test/ct_e2e/ct_monitor_e2e_test_ctfe.sh
@@ -0,0 +1,127 @@
+#!/usr/bin/env bash
+#
+# Copyright 2024 The Sigstore Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -ex
+
+pushd $HOME
+
+echo "downloading service repos"
+for repo in certificate-transparency-go trillian; do
+    if [[ ! -d $repo ]]; then
+        git clone https://github.com/google/${repo}.git
+    fi
+done
+
+docker_compose="docker compose"
+
+pushd ./certificate-transparency-go/trillian/examples/deployment/docker/ctfe/
+docker compose up -d
+sleep 10
+popd
+
+
+die() {
+  echo "$*" > /dev/stderr
+  exit 1
+}
+
+collect_vars() {
+  # set unset environment variables to defaults
+  [ -z ${MYSQL_ROOT_USER+x} ] && MYSQL_ROOT_USER="root"
+  [ -z ${MYSQL_HOST+x} ] && MYSQL_HOST="127.0.0.1"
+  [ -z ${MYSQL_PORT+x} ] && MYSQL_PORT="3306"
+  [ -z ${MYSQL_DATABASE+x} ] && MYSQL_DATABASE="test"
+  [ -z ${MYSQL_USER+x} ] && MYSQL_USER="test"
+  [ -z ${MYSQL_PASSWORD+x} ] && MYSQL_PASSWORD="zaphod"
+  [ -z ${MYSQL_ROOT_PASSWORD+x} ] && MYSQL_ROOT_PASSWORD="zaphod"
+  [ -z ${MYSQL_USER_HOST+x} ] && MYSQL_USER_HOST="127.0.0.1"
+  FLAGS=()
+
+  # handle flags
+  FORCE=false
+  VERBOSE=false
+  while [[ $# -gt 0 ]]; do
+    case "$1" in
+      --force) FORCE=true ;;
+      --verbose) VERBOSE=true ;;
+      --help) usage; exit ;;
+      *) FLAGS+=("$1")
+    esac
+    shift 1
+  done
+
+  FLAGS+=(-u "${MYSQL_ROOT_USER}")
+  FLAGS+=(--host "${MYSQL_HOST}")
+  FLAGS+=(--port "${MYSQL_PORT}")
+
+  # Optionally print flags (before appending password)
+  [[ ${VERBOSE} = 'true' ]] && echo "- Using MySQL Flags: ${FLAGS[@]}"
+
+  # append password if supplied
+  [ -z ${MYSQL_ROOT_PASSWORD+x} ] || FLAGS+=(-p"${MYSQL_ROOT_PASSWORD}")
+}
+
+main() {
+  collect_vars "$@"
+
+  readonly TRILLIAN_PATH=$(go list -f '{{.Dir}}' github.com/google/trillian)
+
+  echo "Warning: about to destroy and reset database '${MYSQL_DATABASE}'"
+  echo "Resetting DB..."
+  mysql "${FLAGS[@]}" -e "DROP DATABASE IF EXISTS ${MYSQL_DATABASE};" || \
+  die "Error: Failed to drop database '${MYSQL_DATABASE}'."
+  mysql "${FLAGS[@]}" -e "CREATE DATABASE ${MYSQL_DATABASE};" || \
+  die "Error: Failed to create database '${MYSQL_DATABASE}'."
+  mysql "${FLAGS[@]}" -e "CREATE USER IF NOT EXISTS ${MYSQL_USER}@'${MYSQL_USER_HOST}' IDENTIFIED BY '${MYSQL_PASSWORD}';" || \
+  die "Error: Failed to create user '${MYSQL_USER}@${MYSQL_USER_HOST}'."
+  mysql "${FLAGS[@]}" -e "GRANT ALL ON ${MYSQL_DATABASE}.* TO ${MYSQL_USER}@'${MYSQL_USER_HOST}'" || \
+  die "Error: Failed to grant '${MYSQL_USER}' user all privileges on '${MYSQL_DATABASE}'."
+  # mysql "${FLAGS[@]}" -D ${MYSQL_DATABASE} < ./trillian/storage/mysql/schema/storage.sql || \
+  # die "Error: Failed to create tables in '${MYSQL_DATABASE}' database."
+  echo "Reset Complete"
+}
+
+main "$@"
+
+docker exec -i ctfe-db mariadb -pzaphod -Dtest < ./trillian/storage/mysql/schema/storage.sql
+docker exec -i ctfe-db mariadb -pzaphod -Dtest < ./certificate-transparency-go/trillian/ctfe/storage/mysql/schema.sql
+
+CTFE_CONF_DIR=/tmp/ctfedocker
+if [ ! -d $CTFE_CONF_DIR ]; then
+  mkdir ${CTFE_CONF_DIR}
+fi
+
+TREE_ID=$(go run github.com/google/trillian/cmd/createtree@master --admin_server=127.0.0.1:8090)
+sed "s/@TREE_ID@/${TREE_ID}/" ./certificate-transparency-go/trillian/examples/deployment/docker/ctfe/ct_server.cfg > ${CTFE_CONF_DIR}/ct_server.cfg
+cp ./certificate-transparency-go/trillian/testdata/fake-ca.cert ${CTFE_CONF_DIR}
+docker volume create --driver local --opt type=none --opt device=${CTFE_CONF_DIR} --opt o=bind ctfe_config
+
+pushd ./certificate-transparency-go/trillian/examples/deployment/docker/ctfe/
+docker compose down
+docker compose --profile frontend up -d
+sleep 30
+popd
+
+docker ps
+docker container logs ctfe-db
+
+popd
+go test -tags=ct_e2e -v -race ./pkg/test/ct_e2e/...
+
+pushd $HOME
+pushd ./certificate-transparency-go/trillian/examples/deployment/docker/ctfe/
+docker compose down
+popd
diff --git a/pkg/test/ct_e2e/ct_monitor_e2e_test_manual.sh b/pkg/test/ct_e2e/ct_monitor_e2e_test_manual.sh
@@ -0,0 +1,108 @@
+#!/usr/bin/env bash
+#
+# Copyright 2024 The Sigstore Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -ex
+
+pushd $HOME
+
+echo "downloading service repos"
+for repo in certificate-transparency-go trillian; do
+    if [[ ! -d $repo ]]; then
+        git clone https://github.com/google/${repo}.git
+    fi
+done
+
+die() {
+  echo "$*" > /dev/stderr
+  exit 1
+}
+
+collect_vars() {
+  # set unset environment variables to defaults
+  [ -z ${MYSQL_ROOT_USER+x} ] && MYSQL_ROOT_USER="test"
+  [ -z ${MYSQL_HOST+x} ] && MYSQL_HOST="127.0.0.1"
+  [ -z ${MYSQL_PORT+x} ] && MYSQL_PORT="3306"
+  [ -z ${MYSQL_DATABASE+x} ] && MYSQL_DATABASE="test"
+  [ -z ${MYSQL_USER+x} ] && MYSQL_USER="test"
+  [ -z ${MYSQL_PASSWORD+x} ] && MYSQL_PASSWORD="zaphod"
+  [ -z ${MYSQL_ROOT_PASSWORD+x} ] && MYSQL_ROOT_PASSWORD="zaphod"
+  [ -z ${MYSQL_USER_HOST+x} ] && MYSQL_USER_HOST="127.0.0.1"
+  FLAGS=()
+
+  # handle flags
+  FORCE=false
+  VERBOSE=false
+  while [[ $# -gt 0 ]]; do
+    case "$1" in
+      --force) FORCE=true ;;
+      --verbose) VERBOSE=true ;;
+      --help) usage; exit ;;
+      *) FLAGS+=("$1")
+    esac
+    shift 1
+  done
+
+  FLAGS+=(-u "${MYSQL_ROOT_USER}")
+  FLAGS+=(--host "${MYSQL_HOST}")
+  FLAGS+=(--port "${MYSQL_PORT}")
+
+  # Optionally print flags (before appending password)
+  [[ ${VERBOSE} = 'true' ]] && echo "- Using MySQL Flags: ${FLAGS[@]}"
+
+  # append password if supplied
+  [ -z ${MYSQL_ROOT_PASSWORD+x} ] || FLAGS+=(-p"${MYSQL_ROOT_PASSWORD}")
+}
+
+main() {
+#   collect_vars "$@"
+
+  readonly TRILLIAN_PATH=$(go list -f '{{.Dir}}' github.com/google/trillian)
+
+  echo "Warning: about to destroy and reset database '${MYSQL_DATABASE}'"
+  echo "Resetting DB..."
+  mysql "${FLAGS[@]}" -e "DROP DATABASE IF EXISTS ${MYSQL_DATABASE};" || \
+  die "Error: Failed to drop database '${MYSQL_DATABASE}'."
+  mysql "${FLAGS[@]}" -e "CREATE DATABASE ${MYSQL_DATABASE};" || \
+  die "Error: Failed to create database '${MYSQL_DATABASE}'."
+#   mysql "${FLAGS[@]}" -e "CREATE USER IF NOT EXISTS ${MYSQL_USER}@'${MYSQL_USER_HOST}' IDENTIFIED BY '${MYSQL_PASSWORD}';" || \
+#   die "Error: Failed to create user '${MYSQL_USER}@${MYSQL_USER_HOST}'."
+#   mysql "${FLAGS[@]}" -e "GRANT ALL ON ${MYSQL_DATABASE}.* TO ${MYSQL_USER}@'${MYSQL_USER_HOST}'" || \
+#   die "Error: Failed to grant '${MYSQL_USER}' user all privileges on '${MYSQL_DATABASE}'."
+  mysql "${FLAGS[@]}" -D ${MYSQL_DATABASE} < ./trillian/storage/mysql/schema/storage.sql || \
+  die "Error: Failed to create tables in '${MYSQL_DATABASE}' database."
+  mysql "${FLAGS[@]}" -D ${MYSQL_DATABASE} < ./certificate-transparency-go/trillian/ctfe/storage/mysql/schema.sql || \
+  die "Error: Failed to provision logs in '${MYSQL_DATABASE}' database."
+  echo "Reset Complete"
+}
+
+main "$@"
+
+pushd ./trillian
+go run github.com/google/trillian/cmd/trillian_log_server --mysql_uri="root@tcp(127.0.0.1:3306)/test" --rpc_endpoint=:8080 --http_endpoint=:8081 --logtostderr
+go run github.com/google/trillian/cmd/trillian_log_signer --mysql_uri="root@tcp(127.0.0.1:3306)/test" --force_master --rpc_endpoint=:8090 --http_endpoint=:8091 --logtostderr
+go run github.com/google/trillian/cmd/createtree --admin_server=:8080
+popd
+
+pushd ./certificate-transparency-go/trillian/examples/deployment/docker/ctfe/
+TRILLIAN_LOG_SERVER_RPC_ENDPOINT=127.0.0.1:8080
+go run github.com/google/certificate-transparency-go/trillian/ctfe/ct_server --log_config ./ct_server.cfg --http_endpoint=127.0.0.1:6966 --log_rpc_server ${TRILLIAN_LOG_SERVER_RPC_ENDPOINT} --logtostderr
+popd
+
+docker ps
+docker container logs ctfe-db
+
+popd
+go test -tags=ct_e2e -v -race ./pkg/test/ct_e2e/...