Skip to content

Commit

Permalink
[Draft] Snapshot e2e test.
Browse files Browse the repository at this point in the history
  • Loading branch information
ser-io committed Oct 2, 2024
1 parent 4046044 commit b70f762
Show file tree
Hide file tree
Showing 4 changed files with 346 additions and 22 deletions.
2 changes: 2 additions & 0 deletions .github/workflows/presubmit.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -72,6 +72,8 @@ jobs:
e2e-tests-orchestration:
runs-on: ubuntu-22.04
steps:
- name: Free space
run: rm -rf /opt/hostedtoolcache
- name: Check kvm
run: |
ls /dev/kvm
Expand Down
60 changes: 38 additions & 22 deletions e2etests/orchestration/BUILD.bazel
Original file line number Diff line number Diff line change
Expand Up @@ -34,11 +34,11 @@ go_library(
],
)

create_single_instance_test(
name="create_fixed_build_id_and_target",
build_id="11510808",
build_target="aosp_cf_x86_64_phone-trunk_staging-userdebug",
)
# create_single_instance_test(
# name="create_fixed_build_id_and_target",
# build_id="11510808",
# build_target="aosp_cf_x86_64_phone-trunk_staging-userdebug",
# )

aosp_artifact(
name = "cvd_host_package",
Expand All @@ -56,25 +56,41 @@ aosp_artifact(
out_name = "images.zip",
)

go_test(
name = "create_local_image",
srcs = ["createlocalimage_test.go"],
data = [
":images_zip",
":cvd_host_package",
"@images//docker:orchestration_image_tar",
],
deps = [
":e2etesting",
"@com_github_google_android_cuttlefish_frontend_src_liboperator//api/v1:api",
"@com_github_google_cloud_android_orchestration//pkg/client",
"@com_github_google_go_cmp//cmp",
],
)
# go_test(
# name = "create_local_image",
# srcs = ["createlocalimage_test.go"],
# data = [
# ":images_zip",
# ":cvd_host_package",
# "@images//docker:orchestration_image_tar",
# ],
# deps = [
# ":e2etesting",
# "@com_github_google_android_cuttlefish_frontend_src_liboperator//api/v1:api",
# "@com_github_google_cloud_android_orchestration//pkg/client",
# "@com_github_google_go_cmp//cmp",
# ],
# )
#
# go_test(
# name = "create_from_images_zip",
# srcs = ["createfromimageszip_test.go"],
# data = [
# ":images_zip",
# ":cvd_host_package",
# "@images//docker:orchestration_image_tar",
# ],
# deps = [
# ":e2etesting",
# "@com_github_google_android_cuttlefish_frontend_src_liboperator//api/v1:api",
# "@com_github_google_cloud_android_orchestration//pkg/client",
# "@com_github_google_go_cmp//cmp",
# ],
# )

go_test(
name = "create_from_images_zip",
srcs = ["createfromimageszip_test.go"],
name = "snapshot",
srcs = ["snapshot_test.go"],
data = [
":images_zip",
":cvd_host_package",
Expand Down
63 changes: 63 additions & 0 deletions e2etests/orchestration/common.go
Original file line number Diff line number Diff line change
Expand Up @@ -165,6 +165,69 @@ func (h *DockerHelper) RemoveContainer(id string) error {
return nil
}

func (h *DockerHelper) StartADBServer(id, adbBin string) error {
return h.exec(id, []string{adbBin, "start-server"})
}

func (h *DockerHelper) ConnectADB(id, adbBin, serial string) error {
return h.exec(id, []string{adbBin, "connect", serial})
}

func (h *DockerHelper) ExecADBShellCommand(id, adbBin, serial string, cmd []string) error {
return h.exec(id, append([]string{adbBin, "-s", serial, "shell"}, cmd...))
}

type DockerExecExitCodeError struct {
ExitCode int
}

func (e DockerExecExitCodeError) Error() string {
return fmt.Sprintf("exit code: %d", e.ExitCode)
}

func (h *DockerHelper) exec(id string, cmd []string) error {
if err := h.runExec(id, cmd); err != nil {
return fmt.Errorf("docker exec %v failed: %w", cmd, err)
}
return nil
}

func (h *DockerHelper) runExec(id string, cmd []string) error {
ctx := context.TODO()
config := types.ExecConfig{
User: "root",
Privileged: true,
Cmd: cmd,
}
cExec, err := h.client.ContainerExecCreate(ctx, id, config)
if err != nil {
return err
}
if err = h.client.ContainerExecStart(ctx, cExec.ID, types.ExecStartCheck{}); err != nil {
return err
}
// ContainerExecStart does not block, short poll process status for 60 seconds to
// check when it has been completed. return a time out error otherwise.
cExecStatus := types.ContainerExecInspect{}
for i := 0; i < 30; i++ {
time.Sleep(500 * time.Millisecond)
cExecStatus, err = h.client.ContainerExecInspect(ctx, cExec.ID)
if err != nil {
return err
}
if !cExecStatus.Running {
break
}
}
if cExecStatus.Running {
return fmt.Errorf("command %v timed out", cmd)
}
if cExecStatus.ExitCode != 0 {
return &DockerExecExitCodeError{ExitCode: cExecStatus.ExitCode}
}
return nil
}

func Cleanup(ctx *TestContext) {
dockerHelper, err := NewDockerHelper()
if err != nil {
Expand Down
243 changes: 243 additions & 0 deletions e2etests/orchestration/snapshot_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,243 @@
// Copyright (C) 2024 The Android Open Source Project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package orchestration

import (
"encoding/json"
// "errors"
"fmt"
"log"
"net/http"
"testing"

"orchestration/e2etesting"

hoapi "github.com/google/android-cuttlefish/frontend/src/liboperator/api/v1"
"github.com/google/cloud-android-orchestration/pkg/client"
)

func TestSnapshot(t *testing.T) {
ctx, err := e2etesting.Setup(61003)
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() {
e2etesting.Cleanup(ctx)
})
// dh, err := e2etesting.NewDockerHelper()
// if err != nil {
// t.Fatal(err)
// }
srv := client.NewHostOrchestratorService(ctx.ServiceURL)
uploadDir, err := srv.CreateUploadDir()
if err != nil {
t.Fatal(err)
}
if err := uploadArtifacts(srv, uploadDir); err != nil {
t.Fatal(err)
}
const groupName = "cvd"
cvd, err := createDevice(srv, groupName, uploadDir)
if err != nil {
t.Fatal(err)
}
log.Printf("cvd: %+v\n", cvd)
// cID := ctx.DockerContainerID
// adbBin := fmt.Sprintf("/var/lib/cuttlefish-common/user_artifacts/%s/bin/adb", uploadDir)
// if err := dh.StartADBServer(cID, adbBin); err != nil {
// t.Fatal(err)
// }
// if err := dh.ConnectADB(cID, adbBin, cvd.ADBSerial); err != nil {
// t.Fatal(err)
// }
// const tmpFile = "/data/local/tmp/foo"
// // Create temporary file
// if err := dh.ExecADBShellCommand(cID, adbBin, cvd.ADBSerial, []string{"touch", tmpFile}); err != nil {
// t.Fatal(err)
// }
// if err := dh.ExecADBShellCommand(cID, adbBin, cvd.ADBSerial, []string{"stat", tmpFile}); err != nil {
// t.Fatal(err)
// }
// // Create a snapshot containing the temporary file.
// createSnapshotRes, err := createSnapshot(ctx.ServiceURL, groupName, cvd.Name)
// if err != nil {
// t.Fatal(err)
// }
// // Remove temporary file
// if err := dh.ExecADBShellCommand(cID, adbBin, cvd.ADBSerial, []string{"rm", tmpFile}); err != nil {
// t.Fatal(err)
// }
// // Double check temporary file does not exist.
// err = dh.ExecADBShellCommand(cID, adbBin, cvd.ADBSerial, []string{"stat", tmpFile})
// var exitCodeErr *e2etesting.DockerExecExitCodeError
// if !errors.As(err, &exitCodeErr) {
// t.Fatal(err)
// }
// Stop the device.
if err := stopDevice(ctx.ServiceURL, groupName, cvd.Name); err != nil {
t.Fatal(err)
}
cvds, err := srv.ListCVDs()
if err != nil {
t.Fatal(err)
}
log.Printf("cvds: len: %d: %+v\n", len(cvds), cvds[0])
// Restore the device from the snapshot.
// startErr := startDevice(ctx.ServiceURL, groupName, cvd.Name, createSnapshotRes.SnapshotID)
startErr := startDevice(ctx.ServiceURL, groupName, cvd.Name, "")
// if err := e2etesting.DownloadHostBugReport(srv, groupName); err != nil {
// t.Errorf("failed creating bugreport: %s", err)
// }
if startErr != nil {
t.Fatal(err)
}
cvds, err = srv.ListCVDs()
if err != nil {
t.Fatal(err)
}
log.Printf("cvds: len: %d: %+v\n", len(cvds), cvds[0])
// Verify the temporary file does exist.
// if err := dh.ConnectADB(cID, adbBin, cvd.ADBSerial); err != nil {
// t.Fatal(err)
// }
// if err := dh.ExecADBShellCommand(cID, adbBin, cvd.ADBSerial, []string{"stat", tmpFile}); err != nil {
// t.Fatal(err)
// }
}

func uploadArtifacts(srv client.HostOrchestratorService, uploadDir string) error {
if err := e2etesting.UploadAndExtract(srv, uploadDir, "images.zip"); err != nil {
return err
}
if err := e2etesting.UploadAndExtract(srv, uploadDir, "cvd-host_package.tar.gz"); err != nil {
return err
}
return nil
}

func createDevice(srv client.HostOrchestratorService, group_name, artifactsDir string) (*hoapi.CVD, error) {
config := `
{
"common": {
"group_name": "` + group_name + `",
"host_package": "@user_artifacts/` + artifactsDir + `"
},
"instances": [
{
"vm": {
"memory_mb": 8192,
"setupwizard_mode": "OPTIONAL",
"cpus": 8,
"enable_virtiofs": "false"
},
"graphics": {
"gpu_mode": "guest_swiftshader"
},
"disk": {
"default_build": "@user_artifacts/` + artifactsDir + `"
},
"streaming": {
"device_id": "cvd-1"
}
}
]
}
`
envConfig := make(map[string]interface{})
if err := json.Unmarshal([]byte(config), &envConfig); err != nil {
return nil, err
}
createReq := &hoapi.CreateCVDRequest{EnvConfig: envConfig}
res, createErr := srv.CreateCVD(createReq /* buildAPICredentials */, "")
if createErr != nil {
if err := e2etesting.DownloadHostBugReport(srv, group_name); err != nil {
log.Printf("error downloading cvd bugreport: %v", err)
}
return nil, createErr
}
return res.CVDs[0], nil
}

// TODO(b/370552105): Use HO API objects definitions from HEAD in e2e tests.
type CreateSnapshotResponse struct {
SnapshotID string `json:"snapshot_id"`
}

// TODO(b/370552105): Use HO API objects definitions from HEAD in e2e tests.
type StartCVDRequest struct {
// Start from the relevant snaphost if not empty.
SnapshotID string `json:"snapshot_id,omitempty"`
}

func createSnapshot(srvURL, group, name string) (*CreateSnapshotResponse, error) {
helper := client.HTTPHelper{
Client: http.DefaultClient,
RootEndpoint: srvURL,
}
op := &hoapi.Operation{}
path := fmt.Sprintf("/cvds/%s/%s/snapshots", group, name)
rb := helper.NewPostRequest(path, nil)
if err := rb.JSONResDo(op); err != nil {
return nil, err
}
srv := client.NewHostOrchestratorService(srvURL)
res := &CreateSnapshotResponse{}
if err := srv.WaitForOperation(op.Name, res); err != nil {
return nil, err
}
return res, nil
}

// TODO(b/370550070) Remove once this method is added to the client implementation.
func stopDevice(srvURL, group, name string) error {
helper := client.HTTPHelper{
Client: http.DefaultClient,
RootEndpoint: srvURL,
}
op := &hoapi.Operation{}
path := fmt.Sprintf("/cvds/%s/%s/:stop", group, name)
rb := helper.NewPostRequest(path, nil)
if err := rb.JSONResDo(op); err != nil {
return err
}
srv := client.NewHostOrchestratorService(srvURL)
res := &hoapi.EmptyResponse{}
if err := srv.WaitForOperation(op.Name, &res); err != nil {
return err
}
return nil
}

// TODO(b/370550070) Remove once this method is added to the client implementation.
func startDevice(srvURL, group, name, snapshotID string) error {
helper := client.HTTPHelper{
Client: http.DefaultClient,
RootEndpoint: srvURL,
}
// body := &StartCVDRequest{SnapshotID: snapshotID}
body := &StartCVDRequest{}
op := &hoapi.Operation{}
path := fmt.Sprintf("/cvds/%s/%s/:start", group, name)
rb := helper.NewPostRequest(path, body)
if err := rb.JSONResDo(op); err != nil {
return err
}
srv := client.NewHostOrchestratorService(srvURL)
res := &hoapi.EmptyResponse{}
if err := srv.WaitForOperation(op.Name, &res); err != nil {
return err
}
return nil
}

0 comments on commit b70f762

Please sign in to comment.