From cde6617a673220165110904737e46a0918c93695 Mon Sep 17 00:00:00 2001 From: Sergio Andres Rodriguez Orama Date: Fri, 27 Sep 2024 17:52:03 -0400 Subject: [PATCH] [Draft] Snapshot e2e test. --- .github/workflows/presubmit.yaml | 2 + e2etests/orchestration/BUILD.bazel | 60 +++--- e2etests/orchestration/common.go | 63 ++++++ e2etests/orchestration/snapshot_test.go | 243 ++++++++++++++++++++++++ 4 files changed, 346 insertions(+), 22 deletions(-) create mode 100644 e2etests/orchestration/snapshot_test.go diff --git a/.github/workflows/presubmit.yaml b/.github/workflows/presubmit.yaml index e18026700c..d11e66970a 100644 --- a/.github/workflows/presubmit.yaml +++ b/.github/workflows/presubmit.yaml @@ -72,6 +72,8 @@ jobs: e2e-tests-orchestration: runs-on: ubuntu-22.04 steps: + - name: Free space + run: rm -rf /opt/hostedtoolcache - name: Check kvm run: | ls /dev/kvm diff --git a/e2etests/orchestration/BUILD.bazel b/e2etests/orchestration/BUILD.bazel index f6e47b1bef..f3356b5811 100644 --- a/e2etests/orchestration/BUILD.bazel +++ b/e2etests/orchestration/BUILD.bazel @@ -34,11 +34,11 @@ go_library( ], ) -create_single_instance_test( - name="create_fixed_build_id_and_target", - build_id="11510808", - build_target="aosp_cf_x86_64_phone-trunk_staging-userdebug", -) +# create_single_instance_test( +# name="create_fixed_build_id_and_target", +# build_id="11510808", +# build_target="aosp_cf_x86_64_phone-trunk_staging-userdebug", +# ) aosp_artifact( name = "cvd_host_package", @@ -56,25 +56,41 @@ aosp_artifact( out_name = "images.zip", ) -go_test( - name = "create_local_image", - srcs = ["createlocalimage_test.go"], - data = [ - ":images_zip", - ":cvd_host_package", - "@images//docker:orchestration_image_tar", - ], - deps = [ - ":e2etesting", - "@com_github_google_android_cuttlefish_frontend_src_liboperator//api/v1:api", - "@com_github_google_cloud_android_orchestration//pkg/client", - "@com_github_google_go_cmp//cmp", - ], -) +# go_test( +# name = "create_local_image", +# srcs = ["createlocalimage_test.go"], +# data = [ +# ":images_zip", +# ":cvd_host_package", +# "@images//docker:orchestration_image_tar", +# ], +# deps = [ +# ":e2etesting", +# "@com_github_google_android_cuttlefish_frontend_src_liboperator//api/v1:api", +# "@com_github_google_cloud_android_orchestration//pkg/client", +# "@com_github_google_go_cmp//cmp", +# ], +# ) +# +# go_test( +# name = "create_from_images_zip", +# srcs = ["createfromimageszip_test.go"], +# data = [ +# ":images_zip", +# ":cvd_host_package", +# "@images//docker:orchestration_image_tar", +# ], +# deps = [ +# ":e2etesting", +# "@com_github_google_android_cuttlefish_frontend_src_liboperator//api/v1:api", +# "@com_github_google_cloud_android_orchestration//pkg/client", +# "@com_github_google_go_cmp//cmp", +# ], +# ) go_test( - name = "create_from_images_zip", - srcs = ["createfromimageszip_test.go"], + name = "snapshot", + srcs = ["snapshot_test.go"], data = [ ":images_zip", ":cvd_host_package", diff --git a/e2etests/orchestration/common.go b/e2etests/orchestration/common.go index 9dee8886c4..640b85a355 100644 --- a/e2etests/orchestration/common.go +++ b/e2etests/orchestration/common.go @@ -165,6 +165,69 @@ func (h *DockerHelper) RemoveContainer(id string) error { return nil } +func (h *DockerHelper) StartADBServer(id, adbBin string) error { + return h.exec(id, []string{adbBin, "start-server"}) +} + +func (h *DockerHelper) ConnectADB(id, adbBin, serial string) error { + return h.exec(id, []string{adbBin, "connect", serial}) +} + +func (h *DockerHelper) ExecADBShellCommand(id, adbBin, serial string, cmd []string) error { + return h.exec(id, append([]string{adbBin, "-s", serial, "shell"}, cmd...)) +} + +type DockerExecExitCodeError struct { + ExitCode int +} + +func (e DockerExecExitCodeError) Error() string { + return fmt.Sprintf("exit code: %d", e.ExitCode) +} + +func (h *DockerHelper) exec(id string, cmd []string) error { + if err := h.runExec(id, cmd); err != nil { + return fmt.Errorf("docker exec %v failed: %w", cmd, err) + } + return nil +} + +func (h *DockerHelper) runExec(id string, cmd []string) error { + ctx := context.TODO() + config := types.ExecConfig{ + User: "root", + Privileged: true, + Cmd: cmd, + } + cExec, err := h.client.ContainerExecCreate(ctx, id, config) + if err != nil { + return err + } + if err = h.client.ContainerExecStart(ctx, cExec.ID, types.ExecStartCheck{}); err != nil { + return err + } + // ContainerExecStart does not block, short poll process status for 60 seconds to + // check when it has been completed. return a time out error otherwise. + cExecStatus := types.ContainerExecInspect{} + for i := 0; i < 30; i++ { + time.Sleep(500 * time.Millisecond) + cExecStatus, err = h.client.ContainerExecInspect(ctx, cExec.ID) + if err != nil { + return err + } + if !cExecStatus.Running { + break + } + } + if cExecStatus.Running { + return fmt.Errorf("command %v timed out", cmd) + } + if cExecStatus.ExitCode != 0 { + return &DockerExecExitCodeError{ExitCode: cExecStatus.ExitCode} + } + return nil +} + func Cleanup(ctx *TestContext) { dockerHelper, err := NewDockerHelper() if err != nil { diff --git a/e2etests/orchestration/snapshot_test.go b/e2etests/orchestration/snapshot_test.go new file mode 100644 index 0000000000..c47638e97a --- /dev/null +++ b/e2etests/orchestration/snapshot_test.go @@ -0,0 +1,243 @@ +// Copyright (C) 2024 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package orchestration + +import ( + "encoding/json" + "errors" + "fmt" + "log" + "net/http" + "testing" + + "orchestration/e2etesting" + + hoapi "github.com/google/android-cuttlefish/frontend/src/liboperator/api/v1" + "github.com/google/cloud-android-orchestration/pkg/client" +) + +func TestSnapshot(t *testing.T) { + ctx, err := e2etesting.Setup(61003) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { + e2etesting.Cleanup(ctx) + }) + dh, err := e2etesting.NewDockerHelper() + if err != nil { + t.Fatal(err) + } + srv := client.NewHostOrchestratorService(ctx.ServiceURL) + uploadDir, err := srv.CreateUploadDir() + if err != nil { + t.Fatal(err) + } + if err := uploadArtifacts(srv, uploadDir); err != nil { + t.Fatal(err) + } + const groupName = "cvd" + cvd, err := createDevice(srv, groupName, uploadDir) + if err != nil { + t.Fatal(err) + } + log.Printf("cvd: %+v\n", cvd) + cID := ctx.DockerContainerID + adbBin := fmt.Sprintf("/var/lib/cuttlefish-common/user_artifacts/%s/bin/adb", uploadDir) + if err := dh.StartADBServer(cID, adbBin); err != nil { + t.Fatal(err) + } + if err := dh.ConnectADB(cID, adbBin, cvd.ADBSerial); err != nil { + t.Fatal(err) + } + const tmpFile = "/data/local/tmp/foo" + // Create temporary file + if err := dh.ExecADBShellCommand(cID, adbBin, cvd.ADBSerial, []string{"touch", tmpFile}); err != nil { + t.Fatal(err) + } + if err := dh.ExecADBShellCommand(cID, adbBin, cvd.ADBSerial, []string{"stat", tmpFile}); err != nil { + t.Fatal(err) + } + // Create a snapshot containing the temporary file. + createSnapshotRes, err := createSnapshot(ctx.ServiceURL, groupName, cvd.Name) + if err != nil { + t.Fatal(err) + } + // Remove temporary file + if err := dh.ExecADBShellCommand(cID, adbBin, cvd.ADBSerial, []string{"rm", tmpFile}); err != nil { + t.Fatal(err) + } + // Double check temporary file does not exist. + err = dh.ExecADBShellCommand(cID, adbBin, cvd.ADBSerial, []string{"stat", tmpFile}) + var exitCodeErr *e2etesting.DockerExecExitCodeError + if !errors.As(err, &exitCodeErr) { + t.Fatal(err) + } + // Stop the device. + if err := stopDevice(ctx.ServiceURL, groupName, cvd.Name); err != nil { + t.Fatal(err) + } + cvds, err := srv.ListCVDs() + if err != nil { + t.Fatal(err) + } + log.Printf("cvds: len: %d: %+v\n", len(cvds), cvds[0]) + // Restore the device from the snapshot. + startErr := startDevice(ctx.ServiceURL, groupName, cvd.Name, createSnapshotRes.SnapshotID) + // startErr := startDevice(ctx.ServiceURL, groupName, cvd.Name, "") + if err := e2etesting.DownloadHostBugReport(srv, groupName); err != nil { + t.Errorf("failed creating bugreport: %s", err) + } + if startErr != nil { + t.Fatal(err) + } + cvds, err = srv.ListCVDs() + if err != nil { + t.Fatal(err) + } + // log.Printf("cvds: len: %d: %+v\n", len(cvds), cvds[0]) + // // Verify the temporary file does exist. + // if err := dh.ConnectADB(cID, adbBin, cvd.ADBSerial); err != nil { + // t.Fatal(err) + // } + // if err := dh.ExecADBShellCommand(cID, adbBin, cvd.ADBSerial, []string{"stat", tmpFile}); err != nil { + // t.Fatal(err) + // } +} + +func uploadArtifacts(srv client.HostOrchestratorService, uploadDir string) error { + if err := e2etesting.UploadAndExtract(srv, uploadDir, "images.zip"); err != nil { + return err + } + if err := e2etesting.UploadAndExtract(srv, uploadDir, "cvd-host_package.tar.gz"); err != nil { + return err + } + return nil +} + +func createDevice(srv client.HostOrchestratorService, group_name, artifactsDir string) (*hoapi.CVD, error) { + config := ` + { + "common": { + "group_name": "` + group_name + `", + "host_package": "@user_artifacts/` + artifactsDir + `" + }, + "instances": [ + { + "vm": { + "memory_mb": 8192, + "setupwizard_mode": "OPTIONAL", + "cpus": 8, + "enable_virtiofs": "false" + }, + "graphics": { + "gpu_mode": "guest_swiftshader" + }, + "disk": { + "default_build": "@user_artifacts/` + artifactsDir + `" + }, + "streaming": { + "device_id": "cvd-1" + } + } + ] + } + ` + envConfig := make(map[string]interface{}) + if err := json.Unmarshal([]byte(config), &envConfig); err != nil { + return nil, err + } + createReq := &hoapi.CreateCVDRequest{EnvConfig: envConfig} + res, createErr := srv.CreateCVD(createReq /* buildAPICredentials */, "") + if createErr != nil { + if err := e2etesting.DownloadHostBugReport(srv, group_name); err != nil { + log.Printf("error downloading cvd bugreport: %v", err) + } + return nil, createErr + } + return res.CVDs[0], nil +} + +// TODO(b/370552105): Use HO API objects definitions from HEAD in e2e tests. +type CreateSnapshotResponse struct { + SnapshotID string `json:"snapshot_id"` +} + +// TODO(b/370552105): Use HO API objects definitions from HEAD in e2e tests. +type StartCVDRequest struct { + // Start from the relevant snaphost if not empty. + SnapshotID string `json:"snapshot_id,omitempty"` +} + +func createSnapshot(srvURL, group, name string) (*CreateSnapshotResponse, error) { + helper := client.HTTPHelper{ + Client: http.DefaultClient, + RootEndpoint: srvURL, + } + op := &hoapi.Operation{} + path := fmt.Sprintf("/cvds/%s/%s/snapshots", group, name) + rb := helper.NewPostRequest(path, nil) + if err := rb.JSONResDo(op); err != nil { + return nil, err + } + srv := client.NewHostOrchestratorService(srvURL) + res := &CreateSnapshotResponse{} + if err := srv.WaitForOperation(op.Name, res); err != nil { + return nil, err + } + return res, nil +} + +// TODO(b/370550070) Remove once this method is added to the client implementation. +func stopDevice(srvURL, group, name string) error { + helper := client.HTTPHelper{ + Client: http.DefaultClient, + RootEndpoint: srvURL, + } + op := &hoapi.Operation{} + path := fmt.Sprintf("/cvds/%s/%s/:stop", group, name) + rb := helper.NewPostRequest(path, nil) + if err := rb.JSONResDo(op); err != nil { + return err + } + srv := client.NewHostOrchestratorService(srvURL) + res := &hoapi.EmptyResponse{} + if err := srv.WaitForOperation(op.Name, &res); err != nil { + return err + } + return nil +} + +// TODO(b/370550070) Remove once this method is added to the client implementation. +func startDevice(srvURL, group, name, snapshotID string) error { + helper := client.HTTPHelper{ + Client: http.DefaultClient, + RootEndpoint: srvURL, + } + // body := &StartCVDRequest{SnapshotID: snapshotID} + body := &StartCVDRequest{} + op := &hoapi.Operation{} + path := fmt.Sprintf("/cvds/%s/%s/:start", group, name) + rb := helper.NewPostRequest(path, body) + if err := rb.JSONResDo(op); err != nil { + return err + } + srv := client.NewHostOrchestratorService(srvURL) + res := &hoapi.EmptyResponse{} + if err := srv.WaitForOperation(op.Name, &res); err != nil { + return err + } + return nil +}