From ddd1998bfcfadd3d33378e8185c3238417c93c66 Mon Sep 17 00:00:00 2001 From: zhoumeiting Date: Thu, 26 May 2016 14:35:39 +0800 Subject: [PATCH 1/8] confict --- galaxy.flag | 8 ++++---- src/client/galaxy_job_action.cc | 34 ++++++++++++++++++++++++++++++++- src/client/galaxy_res_action.cc | 10 +++++----- src/protocol/galaxy.proto | 2 +- src/sdk/galaxy_sdk_resman.cc | 2 +- 5 files changed, 44 insertions(+), 12 deletions(-) diff --git a/galaxy.flag b/galaxy.flag index 170af61d..d469a146 100644 --- a/galaxy.flag +++ b/galaxy.flag @@ -1,9 +1,9 @@ ---nexus_servers= +--nexus_servers=wanghaitao01-pc:8868,wanghaitao01-pc:8869,wanghaitao01-pc:8870,wanghaitao01-pc:8871,wanghaitao01-pc:8872 --appworker_endpoint=127.0.0.1:8221 --appworker_container_id=test-1 --logbufsecs=0 ---username=default ---token=default ---nexus_addr= +--username=zhoumeiting +--token=1234 +--nexus_addr=cq01-ps-dev377.cq01.baidu.com:8868 --nexus_root=/galaxy3 --resman_path=/resman diff --git a/src/client/galaxy_job_action.cc b/src/client/galaxy_job_action.cc index b7c37116..92df6bd4 100644 --- a/src/client/galaxy_job_action.cc +++ b/src/client/galaxy_job_action.cc @@ -179,11 +179,23 @@ bool JobAction::ListJobs() { return false; } + struct timeval t_start; + struct timeval t_finish; + double duration; + ::baidu::galaxy::sdk::ListContainerGroupsRequest resman_request; ::baidu::galaxy::sdk::ListContainerGroupsResponse resman_response; resman_request.user = user_; std::map containers; + + gettimeofday(&t_start, NULL); + bool ret = resman_->ListContainerGroups(resman_request, &resman_response); + + gettimeofday(&t_finish, NULL); + duration = (t_finish.tv_sec - t_start.tv_sec)*1000000.0 + (t_finish.tv_usec - t_start.tv_usec); + fprintf(stderr, "list container time is %d\n", duration); + if (ret) { for (uint32_t i = 0; i < resman_response.containers.size(); ++i) { containers[resman_response.containers[i].id] = resman_response.containers[i]; @@ -198,9 +210,15 @@ bool JobAction::ListJobs() { ::baidu::galaxy::sdk::ListJobsRequest request; ::baidu::galaxy::sdk::ListJobsResponse response; request.user = user_; - + + gettimeofday(&t_start, NULL); ret = app_master_->ListJobs(request, &response); + gettimeofday(&t_finish, NULL); + duration = (t_finish.tv_sec - t_start.tv_sec)*1000000.0 + (t_finish.tv_usec - t_start.tv_usec); + fprintf(stderr, "list job time is %d\n", duration); if (ret) { + struct timeval tt_start; + gettimeofday(&tt_start, NULL); baidu::common::TPrinter jobs(12); jobs.AddRow(12, "", "id", "name", "type","status", "stat(r/p/dep/dea/f)", "replica", "cpu(a/u)", "memory(a/u)", "volums(med/a/u)", "create", "update"); @@ -213,8 +231,12 @@ bool JobAction::ListJobs() { std::string scpu; std::string smem; std::string svolums; + gettimeofday(&t_start, NULL); std::map::iterator it = containers.find(response.jobs[i].jobid); + gettimeofday(&t_finish, NULL); + duration = (t_finish.tv_sec - t_start.tv_sec)*1000000.0 + (t_finish.tv_usec - t_start.tv_usec); + fprintf(stderr, "find time is %d\n", duration); if (it != containers.end()) { scpu = ::baidu::common::NumToString(it->second.cpu.assigned / 1000.0) + "/" + ::baidu::common::NumToString(it->second.cpu.used / 1000.0); @@ -222,9 +244,13 @@ bool JobAction::ListJobs() { + ::baidu::common::HumanReadableString(it->second.memory.used); for (size_t j = 0; j < it->second.volums.size(); ++j) { std::string svolums; + gettimeofday(&t_start, NULL); svolums = StringVolumMedium(it->second.volums[j].medium) + "/" + ::baidu::common::HumanReadableString(it->second.volums[j].volum.assigned) + "/" + ::baidu::common::HumanReadableString(it->second.volums[j].volum.used); + gettimeofday(&t_finish, NULL); + duration = (t_finish.tv_sec - t_start.tv_sec)*1000000.0 + (t_finish.tv_usec - t_start.tv_usec); + fprintf(stderr, "volum [%u] time is %d\n", j, duration); if (j == 0) { jobs.AddRow(12, ::baidu::common::NumToString(i).c_str(), response.jobs[i].jobid.c_str(), @@ -290,7 +316,13 @@ bool JobAction::ListJobs() { } } + gettimeofday(&t_start, NULL); printf("%s\n", jobs.ToString().c_str()); + gettimeofday(&t_finish, NULL); + duration = (t_finish.tv_sec - t_start.tv_sec)*1000000.0 + (t_finish.tv_usec - t_start.tv_usec); + fprintf(stderr, "print time is %d\n", duration); + duration = (t_finish.tv_sec - tt_start.tv_sec)*1000000.0 + (t_finish.tv_usec - tt_start.tv_usec); + fprintf(stderr, "all time is %d\n", duration); } else { printf("List job failed for reason %s:%s\n", StringStatus(response.error_code.status).c_str(), response.error_code.reason.c_str()); diff --git a/src/client/galaxy_res_action.cc b/src/client/galaxy_res_action.cc index 6070b59c..d548a0b3 100755 --- a/src/client/galaxy_res_action.cc +++ b/src/client/galaxy_res_action.cc @@ -370,7 +370,8 @@ bool ResAction::ShowContainerGroup(const std::string& id) { } base.AddRow(7, response.desc.run_user.c_str(), response.desc.version.c_str(), - ::baidu::common::NumToString(response.desc.priority).c_str(), + //::baidu::common::NumToString(response.desc.priority).c_str(), + StringJobType((::baidu::galaxy::sdk::JobType)response.desc.priority).c_str(), response.desc.cmd_line.c_str(), ::baidu::common::NumToString(response.desc.max_per_host).c_str(), response.desc.tag.c_str(), @@ -380,12 +381,11 @@ bool ResAction::ShowContainerGroup(const std::string& id) { printf("%s\n", base.ToString().c_str()); printf("workspace volum infomation\n"); - ::baidu::common::TPrinter workspace_volum(8); - workspace_volum.AddRow(8, "size", "type", "medium", "source_path", "dest_path", "readonly", "exclusive", "use_symlink"); - workspace_volum.AddRow(8, ::baidu::common::HumanReadableString(response.desc.workspace_volum.size).c_str(), + ::baidu::common::TPrinter workspace_volum(7); + workspace_volum.AddRow(7, "size", "type", "medium", "dest_path", "readonly", "exclusive", "use_symlink"); + workspace_volum.AddRow(7, ::baidu::common::HumanReadableString(response.desc.workspace_volum.size).c_str(), StringVolumType(response.desc.workspace_volum.type).c_str(), StringVolumMedium(response.desc.workspace_volum.medium).c_str(), - response.desc.workspace_volum.source_path.c_str(), response.desc.workspace_volum.dest_path.c_str(), StringBool(response.desc.workspace_volum.readonly).c_str(), StringBool(response.desc.workspace_volum.exclusive).c_str(), diff --git a/src/protocol/galaxy.proto b/src/protocol/galaxy.proto index 533d32b2..a7074e7e 100755 --- a/src/protocol/galaxy.proto +++ b/src/protocol/galaxy.proto @@ -305,7 +305,7 @@ message Cgroup { // resource manager -> agent message ContainerDescription { - optional uint32 priority = 1; + optional int32 priority = 1; optional string run_user = 2; optional string version = 3; optional VolumRequired workspace_volum = 4; diff --git a/src/sdk/galaxy_sdk_resman.cc b/src/sdk/galaxy_sdk_resman.cc index e5526b4a..bb8e1c49 100644 --- a/src/sdk/galaxy_sdk_resman.cc +++ b/src/sdk/galaxy_sdk_resman.cc @@ -301,7 +301,7 @@ bool ResourceManager::ShowContainerGroup(const ShowContainerGroupRequest& reques return false; } - response->desc.priority = pb_response.desc().priority(); + response->desc.priority = (JobType)pb_response.desc().priority(); response->desc.run_user = pb_response.desc().run_user(); response->desc.version = pb_response.desc().version(); response->desc.cmd_line = pb_response.desc().cmd_line(); From c995aeec0e8086d834569715fb6b5613274b8bd2 Mon Sep 17 00:00:00 2001 From: zhoumeiting Date: Thu, 26 May 2016 15:36:38 +0800 Subject: [PATCH 2/8] update show_ user --- src/client/galaxy_job_action.cc | 2 +- src/client/galaxy_res_action.cc | 55 +++++++++++++++++++-------------- src/client/galaxy_res_action.h | 2 +- src/client/galaxy_res_client.cc | 8 ++--- src/sdk/galaxy_sdk.h | 3 +- src/sdk/galaxy_sdk_resman.cc | 10 ++++++ 6 files changed, 49 insertions(+), 31 deletions(-) diff --git a/src/client/galaxy_job_action.cc b/src/client/galaxy_job_action.cc index 92df6bd4..e1012e6a 100644 --- a/src/client/galaxy_job_action.cc +++ b/src/client/galaxy_job_action.cc @@ -475,13 +475,13 @@ bool JobAction::ShowJob(const std::string& jobid) { std::string podid(response.job.pods[i].podid, pos + 1, response.job.pods[i].podid.size()- (pos + 1)); pods.AddRow(7, ::baidu::common::NumToString(i).c_str(), response.job.pods[i].podid.c_str(), - //response.job.pods[i].jobid.c_str(), response.job.pods[i].endpoint.c_str(), StringPodStatus(response.job.pods[i].status).c_str(), response.job.pods[i].version.c_str(), FormatDate(response.job.pods[i].start_time).c_str(), ::baidu::common::NumToString(response.job.pods[i].fail_count).c_str() ); + fprintf(stderr, "status is %d\n", response.job.pods[i].status); } printf("%s\n", pods.ToString().c_str()); } diff --git a/src/client/galaxy_res_action.cc b/src/client/galaxy_res_action.cc index d548a0b3..1a032432 100755 --- a/src/client/galaxy_res_action.cc +++ b/src/client/galaxy_res_action.cc @@ -1148,8 +1148,8 @@ bool ResAction::ListUsers() { return ret; } -bool ResAction::ShowUser(const std::string& user, const std::string& token) { - if (user.empty() || token.empty()) { +bool ResAction::ShowUser(const std::string& user) { + if (user.empty()) { return false; } @@ -1161,35 +1161,44 @@ bool ResAction::ShowUser(const std::string& user, const std::string& token) { ::baidu::galaxy::sdk::ShowUserResponse response; request.admin = user_; request.user.user = user; - request.user.token = token; + //request.user.token = token; bool ret = resman_->ShowUser(request, &response); if (ret) { - printf("pools infomation\n"); - ::baidu::common::TPrinter pools(2); - pools.AddRow(2, "", "pool"); - for (uint32_t i = 0; i < response.pools.size(); ++i) { - pools.AddRow(2, ::baidu::common::NumToString(i).c_str(), - response.pools[i].c_str() - ); - } - printf("%s\n", pools.ToString().c_str()); - - printf("authority infomation\n"); - ::baidu::common::TPrinter authority(2); - pools.AddRow(2, "", "authority"); - for (uint32_t i = 0; i < response.authority.size(); ++i) { - pools.AddRow(2, ::baidu::common::NumToString(i).c_str(), - response.authority[i] - ); + printf("grants infomation\n"); + ::baidu::common::TPrinter grants(4); + grants.AddRow(4, "", "pool", "action", "authority"); + for (uint32_t i = 0; i < response.grants.size(); ++i) { + for (uint32_t j = 0; j < response.grants[i].authority.size(); ++j) { + if (j == 0) { + grants.AddRow(4, ::baidu::common::NumToString(i).c_str(), + response.grants[i].pool.c_str(), + StringAuthorityAction(response.grants[i].action).c_str(), + StringAuthority(response.grants[i].authority[j]).c_str() + ); + } else { + grants.AddRow(4, "", + "", + "", + StringAuthority(response.grants[i].authority[j]).c_str() + ); + } + } + if (response.grants[i].authority.size() == 0) { + grants.AddRow(4, ::baidu::common::NumToString(i).c_str(), + response.grants[i].pool.c_str(), + StringAuthorityAction(response.grants[i].action).c_str(), + "" + ); + } } - printf("%s\n", authority.ToString().c_str()); + printf("%s\n", grants.ToString().c_str()); printf("quota infomation\n"); ::baidu::common::TPrinter quota(5); quota.AddRow(5, "millicore", "memory", "disk", "ssd", "replica"); - quota.AddRow(5, ::baidu::common::NumToString(response.quota.millicore).c_str(), + quota.AddRow(5, ::baidu::common::NumToString(response.quota.millicore / 1000.0).c_str(), ::baidu::common::HumanReadableString(response.quota.memory).c_str(), ::baidu::common::HumanReadableString(response.quota.disk).c_str(), ::baidu::common::HumanReadableString(response.quota.ssd).c_str(), @@ -1200,7 +1209,7 @@ bool ResAction::ShowUser(const std::string& user, const std::string& token) { printf("assignd infomation\n"); ::baidu::common::TPrinter assign(5); assign.AddRow(5, "millicore", "memory", "disk", "ssd", "replica"); - assign.AddRow(5, ::baidu::common::NumToString(response.assigned.millicore).c_str(), + assign.AddRow(5, ::baidu::common::NumToString(response.assigned.millicore / 1000.0).c_str(), ::baidu::common::HumanReadableString(response.assigned.memory).c_str(), ::baidu::common::HumanReadableString(response.assigned.disk).c_str(), ::baidu::common::HumanReadableString(response.assigned.ssd).c_str(), diff --git a/src/client/galaxy_res_action.h b/src/client/galaxy_res_action.h index 41f15857..adbcf8a8 100644 --- a/src/client/galaxy_res_action.h +++ b/src/client/galaxy_res_action.h @@ -40,7 +40,7 @@ class ResAction { bool AddUser(const std::string& user, const std::string& token); bool RemoveUser(const std::string& user, const std::string& token); bool ListUsers(); - bool ShowUser(const std::string& user, const std::string& token); + bool ShowUser(const std::string& user); bool GrantUser(const std::string& user, const std::string& token, const std::string& pool, const std::string& opration, const std::string& authority); diff --git a/src/client/galaxy_res_client.cc b/src/client/galaxy_res_client.cc index 8d8f710d..aa65a669 100644 --- a/src/client/galaxy_res_client.cc +++ b/src/client/galaxy_res_client.cc @@ -42,7 +42,7 @@ const std::string kGalaxyUsage = "galaxy_res_client.\n" " galaxy_res_client add_user -u user -t token\n" " galaxy_res_client remove_user -u user -t token\n" " galaxy_res_client list_users\n" - " galaxy_res_client show_user -u user -t token\n" + " galaxy_res_client show_user -u user\n" " galaxy_res_client grant_user -u user -t token -p pool -o [add/remove/set/clear]\n" " -a [create_container,remove_container,update_container,\n" " list_containers,submit_job,remove_job,update_job,list_jobs] \n" @@ -180,11 +180,11 @@ int main(int argc, char** argv) { } else if (strcmp(argv[1], "list_users") == 0) { return resAction->ListUsers(); } else if (strcmp(argv[1], "show_user") == 0) { - if (FLAGS_u.empty() || FLAGS_t.empty()) { - fprintf(stderr, "-u and -t are needed\n"); + if (FLAGS_u.empty()) { + fprintf(stderr, "-u\n"); return -1; } - return resAction->ShowUser(FLAGS_u, FLAGS_t); + return resAction->ShowUser(FLAGS_u); } else if (strcmp(argv[1], "grant_user") == 0) { if (FLAGS_u.empty() || FLAGS_t.empty() || FLAGS_p.empty()) { fprintf(stderr, "-u, -t and -p are needed\n"); diff --git a/src/sdk/galaxy_sdk.h b/src/sdk/galaxy_sdk.h index e3efa7d7..4889ab01 100755 --- a/src/sdk/galaxy_sdk.h +++ b/src/sdk/galaxy_sdk.h @@ -452,8 +452,7 @@ struct ShowUserRequest { }; struct ShowUserResponse { ErrorCode error_code; - std::vector pools; - std::vector authority; + std::vector grants; Quota quota; Quota assigned; }; diff --git a/src/sdk/galaxy_sdk_resman.cc b/src/sdk/galaxy_sdk_resman.cc index f09b5272..2c469d05 100644 --- a/src/sdk/galaxy_sdk_resman.cc +++ b/src/sdk/galaxy_sdk_resman.cc @@ -944,6 +944,16 @@ bool ResourceManager::ShowUser(const ShowUserRequest& request, ShowUserResponse* response->assigned.ssd = pb_response.assigned().ssd(); response->assigned.replica = pb_response.assigned().replica(); + for (int i = 0; i < pb_response.grants().size(); ++i) { + Grant grant; + grant.pool = pb_response.grants(i).pool(); + grant.action = (AuthorityAction)pb_response.grants(i).action(); + for (int j = 0; j < pb_response.grants(i).authority().size(); ++j) { + grant.authority.push_back((Authority)pb_response.grants(i).authority(j)); + } + response->grants.push_back(grant); + } + return true; } From bd84fff24ac9bbfc3c5c19d9d7f5a55eb3bbc159 Mon Sep 17 00:00:00 2001 From: zhoumeiting Date: Thu, 26 May 2016 21:31:07 +0800 Subject: [PATCH 3/8] add GenerateJson to galaxy_client --- src/client/galaxy_job_action.cc | 137 ++++++++++++++++++---------- src/client/galaxy_job_client.cc | 26 ++++-- src/client/galaxy_res_action.cc | 15 ++-- src/client/galaxy_res_client.cc | 56 ++++++------ src/client/galaxy_util.cc | 153 ++++++++++++++++++++++++++++++++ src/client/galaxy_util.h | 3 + src/sdk/galaxy_sdk.h | 3 + src/sdk/galaxy_sdk_util.cc | 1 - 8 files changed, 302 insertions(+), 92 deletions(-) diff --git a/src/client/galaxy_job_action.cc b/src/client/galaxy_job_action.cc index e1012e6a..d138cdf9 100644 --- a/src/client/galaxy_job_action.cc +++ b/src/client/galaxy_job_action.cc @@ -175,26 +175,32 @@ bool JobAction::RemoveJob(const std::string& jobid) { bool JobAction::ListJobs() { + /*struct timeval t_start; + struct timeval t_finish; + double duration; + + struct timeval tt_start; + gettimeofday(&tt_start, NULL); + */ if(!this->Init()) { return false; } - struct timeval t_start; - struct timeval t_finish; - double duration; - + ::baidu::galaxy::sdk::ListContainerGroupsRequest resman_request; ::baidu::galaxy::sdk::ListContainerGroupsResponse resman_response; resman_request.user = user_; std::map containers; - gettimeofday(&t_start, NULL); + //gettimeofday(&t_start, NULL); bool ret = resman_->ListContainerGroups(resman_request, &resman_response); - gettimeofday(&t_finish, NULL); + /*gettimeofday(&t_finish, NULL); duration = (t_finish.tv_sec - t_start.tv_sec)*1000000.0 + (t_finish.tv_usec - t_start.tv_usec); - fprintf(stderr, "list container time is %d\n", duration); + fprintf(stderr, "list container start time is %lf\n", t_start.tv_sec*1000000.0 + t_start.tv_usec); + fprintf(stderr, "list container finish time is %lf\n", t_finish.tv_sec*1000000.0 + t_finish.tv_usec); + fprintf(stderr, "list container duration is %lf\n", duration);*/ if (ret) { for (uint32_t i = 0; i < resman_response.containers.size(); ++i) { @@ -211,14 +217,14 @@ bool JobAction::ListJobs() { ::baidu::galaxy::sdk::ListJobsResponse response; request.user = user_; - gettimeofday(&t_start, NULL); + //gettimeofday(&t_start, NULL); ret = app_master_->ListJobs(request, &response); - gettimeofday(&t_finish, NULL); + /*gettimeofday(&t_finish, NULL); duration = (t_finish.tv_sec - t_start.tv_sec)*1000000.0 + (t_finish.tv_usec - t_start.tv_usec); - fprintf(stderr, "list job time is %d\n", duration); + fprintf(stderr, "list job start time is %lf\n", t_start.tv_sec*1000000.0 + t_start.tv_usec); + fprintf(stderr, "list job finish time is %lf\n", t_finish.tv_sec*1000000.0 + t_finish.tv_usec); + fprintf(stderr, "list duration is %lf\n", duration);*/ if (ret) { - struct timeval tt_start; - gettimeofday(&tt_start, NULL); baidu::common::TPrinter jobs(12); jobs.AddRow(12, "", "id", "name", "type","status", "stat(r/p/dep/dea/f)", "replica", "cpu(a/u)", "memory(a/u)", "volums(med/a/u)", "create", "update"); @@ -231,12 +237,9 @@ bool JobAction::ListJobs() { std::string scpu; std::string smem; std::string svolums; - gettimeofday(&t_start, NULL); + std::map::iterator it = containers.find(response.jobs[i].jobid); - gettimeofday(&t_finish, NULL); - duration = (t_finish.tv_sec - t_start.tv_sec)*1000000.0 + (t_finish.tv_usec - t_start.tv_usec); - fprintf(stderr, "find time is %d\n", duration); if (it != containers.end()) { scpu = ::baidu::common::NumToString(it->second.cpu.assigned / 1000.0) + "/" + ::baidu::common::NumToString(it->second.cpu.used / 1000.0); @@ -244,13 +247,9 @@ bool JobAction::ListJobs() { + ::baidu::common::HumanReadableString(it->second.memory.used); for (size_t j = 0; j < it->second.volums.size(); ++j) { std::string svolums; - gettimeofday(&t_start, NULL); svolums = StringVolumMedium(it->second.volums[j].medium) + "/" + ::baidu::common::HumanReadableString(it->second.volums[j].volum.assigned) + "/" + ::baidu::common::HumanReadableString(it->second.volums[j].volum.used); - gettimeofday(&t_finish, NULL); - duration = (t_finish.tv_sec - t_start.tv_sec)*1000000.0 + (t_finish.tv_usec - t_start.tv_usec); - fprintf(stderr, "volum [%u] time is %d\n", j, duration); if (j == 0) { jobs.AddRow(12, ::baidu::common::NumToString(i).c_str(), response.jobs[i].jobid.c_str(), @@ -316,17 +315,23 @@ bool JobAction::ListJobs() { } } - gettimeofday(&t_start, NULL); + //gettimeofday(&t_start, NULL); printf("%s\n", jobs.ToString().c_str()); - gettimeofday(&t_finish, NULL); + /*gettimeofday(&t_finish, NULL); duration = (t_finish.tv_sec - t_start.tv_sec)*1000000.0 + (t_finish.tv_usec - t_start.tv_usec); - fprintf(stderr, "print time is %d\n", duration); - duration = (t_finish.tv_sec - tt_start.tv_sec)*1000000.0 + (t_finish.tv_usec - tt_start.tv_usec); - fprintf(stderr, "all time is %d\n", duration); + fprintf(stderr, "print start time is %lf\n", t_start.tv_sec*1000000.0 + t_start.tv_usec); + fprintf(stderr, "print finish time is %lf\n", t_finish.tv_sec*1000000.0 + t_finish.tv_usec); + fprintf(stderr, "print time is %lf\n", duration);*/ } else { printf("List job failed for reason %s:%s\n", StringStatus(response.error_code.status).c_str(), response.error_code.reason.c_str()); } + + /*gettimeofday(&t_finish, NULL); + duration = (t_finish.tv_sec - tt_start.tv_sec)*1000000.0 + (t_finish.tv_usec - tt_start.tv_usec); + fprintf(stderr, "all start time is %lf\n", tt_start.tv_sec*1000000.0 + tt_start.tv_usec); + fprintf(stderr, "all finish time is %lf\n", t_finish.tv_sec*1000000.0 + t_finish.tv_usec); + fprintf(stderr, "all time is %lf\n", duration);*/ return ret; } @@ -414,10 +419,11 @@ bool JobAction::ShowJob(const std::string& jobid) { printf("%s\n", desc_data_volums.ToString().c_str()); printf("job description pod task infomation\n"); - ::baidu::common::TPrinter desc_tasks(7); - //desc_tasks.AddRow(10, "", "id", "cpu(cores/excess)", "memory(cores)", "tcp_throt", "blkio", "ports(name/port)", "exe_package", "data_package", "services"); - desc_tasks.AddRow(7, "", "id", "cpu(cores/excess)", "memory(size/excess)", "tcp_throt(r/re/s/se)", "blkio", "ports(name/port)"); for (uint32_t i = 0; i < response.job.desc.pod.tasks.size(); ++i) { + printf("=========================================================\n"); + printf("job description pod task [%u] base infomation\n", i); + ::baidu::common::TPrinter desc_task(7); + desc_task.AddRow(7, "", "id", "cpu(cores/excess)", "memory(size/excess)", "tcp_throt(r/re/s/se)", "blkio", "ports(name/port)"); std::string scpu = ::baidu::common::NumToString(response.job.desc.pod.tasks[i].cpu.milli_core / 1000.0) + "/" + StringBool(response.job.desc.pod.tasks[i].cpu.excess); std::string smem = ::baidu::common::HumanReadableString(response.job.desc.pod.tasks[i].memory.size) + "/" @@ -436,7 +442,7 @@ bool JobAction::ShowJob(const std::string& jobid) { + response.job.desc.pod.tasks[i].ports[j].port; //+ response.job.desc.pod.tasks[i].ports[j].real_port; if (j == 0) { - desc_tasks.AddRow(7, ::baidu::common::NumToString(i).c_str(), + desc_task.AddRow(7, ::baidu::common::NumToString(i).c_str(), response.job.desc.pod.tasks[i].id.c_str(), scpu.c_str(), smem.c_str(), @@ -445,7 +451,7 @@ bool JobAction::ShowJob(const std::string& jobid) { sports.c_str() ); } else { - desc_tasks.AddRow(7, "", + desc_task.AddRow(7, "", "", "", "", @@ -454,9 +460,11 @@ bool JobAction::ShowJob(const std::string& jobid) { sports.c_str() ); } + } + if (response.job.desc.pod.tasks[i].ports.size() == 0) { - desc_tasks.AddRow(7, ::baidu::common::NumToString(i).c_str(), + desc_task.AddRow(7, ::baidu::common::NumToString(i).c_str(), response.job.desc.pod.tasks[i].id.c_str(), scpu.c_str(), smem.c_str(), @@ -465,26 +473,59 @@ bool JobAction::ShowJob(const std::string& jobid) { "" ); } - printf("%s\n", desc_tasks.ToString().c_str()); - - printf("podinfo infomation\n"); - ::baidu::common::TPrinter pods(7); - pods.AddRow(7, "", "podid", "endpoint", "status", "version", "start_time", "fail_count"); - for (uint32_t i = 0; i < response.job.pods.size(); ++i) { - size_t pos = response.job.pods[i].podid.rfind("."); - std::string podid(response.job.pods[i].podid, pos + 1, response.job.pods[i].podid.size()- (pos + 1)); - pods.AddRow(7, ::baidu::common::NumToString(i).c_str(), - response.job.pods[i].podid.c_str(), - response.job.pods[i].endpoint.c_str(), - StringPodStatus(response.job.pods[i].status).c_str(), - response.job.pods[i].version.c_str(), - FormatDate(response.job.pods[i].start_time).c_str(), - ::baidu::common::NumToString(response.job.pods[i].fail_count).c_str() + printf("%s\n", desc_task.ToString().c_str()); + + printf("job description pod task [%u] exe_package infomation\n", i); + printf("-----------------------------------------------\n"); + printf("start_cmd: %s\n", response.job.desc.pod.tasks[i].exe_package.start_cmd.c_str()); + printf("stop_cmd: %s\n", response.job.desc.pod.tasks[i].exe_package.stop_cmd.c_str()); + printf("dest_path: %s\n", response.job.desc.pod.tasks[i].exe_package.package.dest_path.c_str()); + printf("version: %s\n", response.job.desc.pod.tasks[i].exe_package.package.version.c_str()); + + printf("\njob description pod task [%u] data_package infomation\n", i); + printf("-----------------------------------------------\n"); + printf("reload_cmd: %s\n", response.job.desc.pod.tasks[i].data_package.reload_cmd.c_str()); + ::baidu::common::TPrinter packages(3); + packages.AddRow(3, "", "version", "dest_path"); + for (uint32_t j = 0; j < response.job.desc.pod.tasks[i].data_package.packages.size(); ++j) { + packages.AddRow(3, ::baidu::common::NumToString(j).c_str(), + response.job.desc.pod.tasks[i].data_package.packages[j].version.c_str(), + response.job.desc.pod.tasks[i].data_package.packages[j].dest_path.c_str() + ); + } + printf("%s\n", packages.ToString().c_str()); + + printf("job description pod task [%u] services infomation\n", i); + ::baidu::common::TPrinter services(4); + services.AddRow(4, "", "name", "port_name", "use_bns"); + for (uint32_t j = 0; j < response.job.desc.pod.tasks[i].services.size(); ++j) { + services.AddRow(4, ::baidu::common::NumToString(j).c_str(), + response.job.desc.pod.tasks[i].services[j].service_name.c_str(), + response.job.desc.pod.tasks[i].services[j].port_name.c_str(), + StringBool(response.job.desc.pod.tasks[i].services[j].use_bns).c_str() + ); + } + printf("%s\n", services.ToString().c_str()); + + } + + printf("podinfo infomation\n"); + ::baidu::common::TPrinter pods(7); + pods.AddRow(7, "", "podid", "endpoint", "status", "version", "start_time", "fail_count"); + for (uint32_t i = 0; i < response.job.pods.size(); ++i) { + size_t pos = response.job.pods[i].podid.rfind("."); + std::string podid(response.job.pods[i].podid, pos + 1, response.job.pods[i].podid.size()- (pos + 1)); + pods.AddRow(7, ::baidu::common::NumToString(i).c_str(), + response.job.pods[i].podid.c_str(), + response.job.pods[i].endpoint.c_str(), + StringPodStatus(response.job.pods[i].status).c_str(), + response.job.pods[i].version.c_str(), + FormatDate(response.job.pods[i].start_time).c_str(), + ::baidu::common::NumToString(response.job.pods[i].fail_count).c_str() ); - fprintf(stderr, "status is %d\n", response.job.pods[i].status); } printf("%s\n", pods.ToString().c_str()); - } + } else { printf("Show job failed for reason %s:%s\n", StringStatus(response.error_code.status).c_str(), diff --git a/src/client/galaxy_job_client.cc b/src/client/galaxy_job_client.cc index f0a14a35..5d387d45 100644 --- a/src/client/galaxy_job_client.cc +++ b/src/client/galaxy_job_client.cc @@ -19,12 +19,14 @@ const std::string kGalaxyUsage = "galaxy.\n" " galaxy list\n" " galaxy show -i id\n" " galaxy exec -i id -c cmd\n" + " galaxy json" "Optionss: \n" " -f specify config file, job config file or label config file.\n" " -c specify cmd.\n" " -i specify job id.\n"; int main(int argc, char** argv) { + bool ok = true; FLAGS_flagfile = "./galaxy.flag"; ::google::SetUsageMessage(kGalaxyUsage); ::google::ParseCommandLineFlags(&argc, &argv, true); @@ -41,7 +43,7 @@ int main(int argc, char** argv) { fprintf(stderr, "-f is needed\n"); return -1; } - return jobAction->SubmitJob(FLAGS_f); + ok = jobAction->SubmitJob(FLAGS_f); } else if (strcmp(argv[1], "update") == 0) { if (FLAGS_f.empty()) { fprintf(stderr, "-f is needed\n"); @@ -53,29 +55,29 @@ int main(int argc, char** argv) { return -1; } - return jobAction->UpdateJob(FLAGS_f, FLAGS_i); + ok = jobAction->UpdateJob(FLAGS_f, FLAGS_i); } else if (strcmp(argv[1], "remove") == 0) { if (FLAGS_i.empty()) { fprintf(stderr, "-i is needed\n"); return -1; } - return jobAction->RemoveJob(FLAGS_i); + ok = jobAction->RemoveJob(FLAGS_i); } else if (strcmp(argv[1], "list") == 0) { - return jobAction->ListJobs(); + ok = jobAction->ListJobs(); } else if (strcmp(argv[1], "stop") == 0) { if (FLAGS_i.empty()) { fprintf(stderr, "-i is needed\n"); return -1; } - return jobAction->StopJob(FLAGS_i); + ok = jobAction->StopJob(FLAGS_i); } else if (strcmp(argv[1], "show") == 0) { if (FLAGS_i.empty()) { fprintf(stderr, "-i is needed\n"); return -1; } - return jobAction->ShowJob(FLAGS_i); + ok = jobAction->ShowJob(FLAGS_i); } else if (strcmp(argv[1], "exec") == 0) { if (FLAGS_i.empty()) { @@ -87,12 +89,18 @@ int main(int argc, char** argv) { fprintf(stderr, "-c is needed\n"); return -1; } - return jobAction->ExecuteCmd(FLAGS_i, FLAGS_c); - } else { + ok = jobAction->ExecuteCmd(FLAGS_i, FLAGS_c); + } else if (strcmp(argv[1], "json") == 0) { + ok = ::baidu::galaxy::client::GenerateJson(2, 2, 2, 2, 2); + }else { fprintf(stderr, "%s", kGalaxyUsage.c_str()); return -1; } - return 0; + if (ok) { + return 0; + } else { + return -1; + } } /* vim: set ts=4 sw=4 sts=4 tw=100 */ diff --git a/src/client/galaxy_res_action.cc b/src/client/galaxy_res_action.cc index 1a032432..b4e6a959 100755 --- a/src/client/galaxy_res_action.cc +++ b/src/client/galaxy_res_action.cc @@ -1166,29 +1166,26 @@ bool ResAction::ShowUser(const std::string& user) { bool ret = resman_->ShowUser(request, &response); if (ret) { - printf("grants infomation\n"); - ::baidu::common::TPrinter grants(4); - grants.AddRow(4, "", "pool", "action", "authority"); + printf("authority infomation\n"); + ::baidu::common::TPrinter grants(3); + grants.AddRow(3, "", "pool", "authority"); for (uint32_t i = 0; i < response.grants.size(); ++i) { for (uint32_t j = 0; j < response.grants[i].authority.size(); ++j) { if (j == 0) { - grants.AddRow(4, ::baidu::common::NumToString(i).c_str(), + grants.AddRow(3, ::baidu::common::NumToString(i).c_str(), response.grants[i].pool.c_str(), - StringAuthorityAction(response.grants[i].action).c_str(), StringAuthority(response.grants[i].authority[j]).c_str() ); } else { - grants.AddRow(4, "", - "", + grants.AddRow(3, "", "", StringAuthority(response.grants[i].authority[j]).c_str() ); } } if (response.grants[i].authority.size() == 0) { - grants.AddRow(4, ::baidu::common::NumToString(i).c_str(), + grants.AddRow(3, ::baidu::common::NumToString(i).c_str(), response.grants[i].pool.c_str(), - StringAuthorityAction(response.grants[i].action).c_str(), "" ); } diff --git a/src/client/galaxy_res_client.cc b/src/client/galaxy_res_client.cc index aa65a669..3caacebe 100644 --- a/src/client/galaxy_res_client.cc +++ b/src/client/galaxy_res_client.cc @@ -58,6 +58,7 @@ const std::string kGalaxyUsage = "galaxy_res_client.\n" int main(int argc, char** argv) { + bool ok = true; FLAGS_flagfile = "./galaxy.flag"; ::google::SetUsageMessage(kGalaxyUsage); ::google::ParseCommandLineFlags(&argc, &argv, true); @@ -73,7 +74,7 @@ int main(int argc, char** argv) { fprintf(stderr, "-f is needed\n"); return -1; } - return resAction->CreateContainerGroup(FLAGS_f); + ok = resAction->CreateContainerGroup(FLAGS_f); } else if (strcmp(argv[1], "update_container") == 0) { if (FLAGS_f.empty()) { fprintf(stderr, "-f is needed\n"); @@ -85,22 +86,22 @@ int main(int argc, char** argv) { return -1; } - return resAction->UpdateContainerGroup(FLAGS_f, FLAGS_i); + ok = resAction->UpdateContainerGroup(FLAGS_f, FLAGS_i); } else if (strcmp(argv[1], "remove_container") == 0) { if (FLAGS_i.empty()) { fprintf(stderr, "-i is needed\n"); return -1; } - return resAction->RemoveContainerGroup(FLAGS_i); + ok = resAction->RemoveContainerGroup(FLAGS_i); } else if (strcmp(argv[1], "list_containers") == 0) { - return resAction->ListContainerGroups(); + ok = resAction->ListContainerGroups(); } else if (strcmp(argv[1], "show_container") == 0) { if (FLAGS_i.empty()) { fprintf(stderr, "-i is needed\n"); return -1; } - return resAction->ShowContainerGroup(FLAGS_i); + ok = resAction->ShowContainerGroup(FLAGS_i); } else if (strcmp(argv[1], "add_agent") == 0) { if (FLAGS_p.empty()) { @@ -111,80 +112,80 @@ int main(int argc, char** argv) { fprintf(stderr, "-e is needed\n"); return -1; } - return resAction->AddAgent(FLAGS_p, FLAGS_e); + ok = resAction->AddAgent(FLAGS_p, FLAGS_e); } else if (strcmp(argv[1], "show_agent") == 0) { if (FLAGS_e.empty()) { fprintf(stderr, "-e is needed\n"); return -1; } - return resAction->ShowAgent(FLAGS_e); + ok = resAction->ShowAgent(FLAGS_e); } else if (strcmp(argv[1], "remove_agent") == 0) { if (FLAGS_e.empty()) { fprintf(stderr, "-e is needed\n"); return -1; } - return resAction->RemoveAgent(FLAGS_e); + ok = resAction->RemoveAgent(FLAGS_e); } else if (strcmp(argv[1], "list_agents") == 0) { if (!FLAGS_p.empty()) { - return resAction->ListAgentsByPool(FLAGS_p); + ok = resAction->ListAgentsByPool(FLAGS_p); } else if (!FLAGS_t.empty()) { - return resAction->ListAgentsByTag(FLAGS_t); + ok = resAction->ListAgentsByTag(FLAGS_t); } else { - return resAction->ListAgents(); + ok = resAction->ListAgents(); } } else if (strcmp(argv[1], "enter_safemode") == 0) { - return resAction->EnterSafeMode(); + ok = resAction->EnterSafeMode(); } else if (strcmp(argv[1], "leave_safemode") == 0) { - return resAction->LeaveSafeMode(); + ok = resAction->LeaveSafeMode(); } else if (strcmp(argv[1], "online_agent") == 0) { if (FLAGS_e.empty()) { fprintf(stderr, "-e is needed\n"); return -1; } - return resAction->OnlineAgent(FLAGS_e); + ok = resAction->OnlineAgent(FLAGS_e); } else if (strcmp(argv[1], "offline_agent") == 0) { if (FLAGS_e.empty()) { fprintf(stderr, "-e is needed\n"); return -1; } - return resAction->OfflineAgent(FLAGS_e); + ok = resAction->OfflineAgent(FLAGS_e); } else if (strcmp(argv[1], "status") == 0) { - return resAction->Status(); + ok = resAction->Status(); } else if (strcmp(argv[1], "create_tag") == 0) { if (FLAGS_t.empty() || FLAGS_f.empty()) { fprintf(stderr, "-t and -f needed\n"); return -1; } - return resAction->CreateTag(FLAGS_t, FLAGS_f); + ok = resAction->CreateTag(FLAGS_t, FLAGS_f); } else if (strcmp(argv[1], "list_tags") == 0) { - return resAction->ListTags(); + ok = resAction->ListTags(); } else if (strcmp(argv[1], "list_tags") == 0) { if (FLAGS_e.empty()) { fprintf(stderr, "-e is needed\n"); return -1; } - return resAction->GetPoolByAgent(FLAGS_e); + ok = resAction->GetPoolByAgent(FLAGS_e); } else if (strcmp(argv[1], "add_user") == 0) { if (FLAGS_u.empty() || FLAGS_t.empty()) { fprintf(stderr, "-u and -t are needed\n"); return -1; } - return resAction->AddUser(FLAGS_u, FLAGS_t); + ok = resAction->AddUser(FLAGS_u, FLAGS_t); } else if (strcmp(argv[1], "remove_user") == 0) { if (FLAGS_u.empty() || FLAGS_t.empty()) { fprintf(stderr, "-u and -t are needed\n"); return -1; } - return resAction->RemoveUser(FLAGS_u, FLAGS_t); + ok = resAction->RemoveUser(FLAGS_u, FLAGS_t); } else if (strcmp(argv[1], "list_users") == 0) { - return resAction->ListUsers(); + ok = resAction->ListUsers(); } else if (strcmp(argv[1], "show_user") == 0) { if (FLAGS_u.empty()) { fprintf(stderr, "-u\n"); return -1; } - return resAction->ShowUser(FLAGS_u); + ok = resAction->ShowUser(FLAGS_u); } else if (strcmp(argv[1], "grant_user") == 0) { if (FLAGS_u.empty() || FLAGS_t.empty() || FLAGS_p.empty()) { fprintf(stderr, "-u, -t and -p are needed\n"); @@ -196,13 +197,18 @@ int main(int argc, char** argv) { return -1; } - return resAction->GrantUser(FLAGS_u, FLAGS_t, FLAGS_p, FLAGS_o, FLAGS_a); + ok = resAction->GrantUser(FLAGS_u, FLAGS_t, FLAGS_p, FLAGS_o, FLAGS_a); } else { fprintf(stderr, "%s", kGalaxyUsage.c_str()); return -1; } - return 0; + + if (ok) { + return 0; + } else { + return -1; + } } /* vim: set ts=4 sw=4 sts=4 tw=100 */ diff --git a/src/client/galaxy_util.cc b/src/client/galaxy_util.cc index e7a98343..5f6a24b7 100644 --- a/src/client/galaxy_util.cc +++ b/src/client/galaxy_util.cc @@ -9,6 +9,9 @@ #include #include #include +#include "rapidjson/prettywriter.h" +#include "rapidjson/document.h" +#include "rapidjson/stringbuffer.h" #include "sdk/galaxy_sdk.h" namespace baidu { @@ -174,6 +177,15 @@ std::string StringPodStatus(const ::baidu::galaxy::sdk::PodStatus& status) { case ::baidu::galaxy::sdk::kPodFinished: result = "Finished"; break; + case ::baidu::galaxy::sdk::kPodRunning: + result = "Running"; + break; + case ::baidu::galaxy::sdk::kPodStopping: + result = "Stoping"; + break; + case ::baidu::galaxy::sdk::kPodTerminated: + result = "Terminated"; + break; default: result = ""; } @@ -473,6 +485,147 @@ bool LoadAgentEndpointsFromFile(const std::string& file_name, std::vector writer(buffer); + root.Accept(writer); + std::string reststring = buffer.GetString(); + fprintf(stderr, "%s\n", reststring.c_str()); + return true; +} + } // end namespace client } // end namespace galaxy } // end namespace baidu diff --git a/src/client/galaxy_util.h b/src/client/galaxy_util.h index 2a2bd4be..0670a2eb 100644 --- a/src/client/galaxy_util.h +++ b/src/client/galaxy_util.h @@ -45,6 +45,9 @@ bool GetHostname(std::string* hostname); //读取endpoint bool LoadAgentEndpointsFromFile(const std::string& file_name, std::vector* agents); +//自动生成json文件 +bool GenerateJson(int num_task, int num_data_volums, int num_port, int num_data_packages, int num_services); + } //end namespace client } //end namespace galaxy } //end namespace baidu diff --git a/src/sdk/galaxy_sdk.h b/src/sdk/galaxy_sdk.h index 4889ab01..73935fb1 100755 --- a/src/sdk/galaxy_sdk.h +++ b/src/sdk/galaxy_sdk.h @@ -143,6 +143,9 @@ enum PodStatus { kPodServing=5, kPodFailed=6, kPodFinished=7, + kPodRunning = 8, + kPodStopping = 9, + kPodTerminated = 10, }; enum TaskStatus { kTaskPending=1, diff --git a/src/sdk/galaxy_sdk_util.cc b/src/sdk/galaxy_sdk_util.cc index 0d7d3773..ed4f3a0c 100755 --- a/src/sdk/galaxy_sdk_util.cc +++ b/src/sdk/galaxy_sdk_util.cc @@ -174,7 +174,6 @@ void FillDeploy(const Deploy& sdk_deploy, ::baidu::galaxy::proto::Deploy* deploy deploy->set_max_per_host(sdk_deploy.max_per_host); deploy->set_tag(sdk_deploy.tag); for (size_t i = 0; i < sdk_deploy.pools.size(); ++i) { - fprintf(stderr, "port is %s\n", sdk_deploy.pools[i].c_str()); deploy->add_pools(sdk_deploy.pools[i]); } } From 98b51f3385ba9321340aa1c580eb742c52d4f4a8 Mon Sep 17 00:00:00 2001 From: zhoumeiting Date: Fri, 27 May 2016 11:29:43 +0800 Subject: [PATCH 4/8] add generate json --- src/client/galaxy_util.cc | 61 +++++++++++++++++++++++++++++--------- src/sdk/galaxy_sdk_util.cc | 5 ++++ 2 files changed, 52 insertions(+), 14 deletions(-) diff --git a/src/client/galaxy_util.cc b/src/client/galaxy_util.cc index 5f6a24b7..9bf5b365 100644 --- a/src/client/galaxy_util.cc +++ b/src/client/galaxy_util.cc @@ -9,6 +9,7 @@ #include #include #include +#include "string_util.h" #include "rapidjson/prettywriter.h" #include "rapidjson/document.h" #include "rapidjson/stringbuffer.h" @@ -489,6 +490,11 @@ bool GenerateJson(int num_tasks, int num_data_volums, int num_ports, int num_dat rapidjson::Document document; rapidjson::Document::AllocatorType& allocator = document.GetAllocator(); + + //设置临时字符串使用 + rapidjson::Value obj_str(rapidjson::kStringType); + std::string str; + //根节点 rapidjson::Value root(rapidjson::kObjectType); @@ -524,14 +530,14 @@ bool GenerateJson(int num_tasks, int num_data_volums, int num_ports, int num_dat rapidjson::Value data_volums(rapidjson::kArrayType); for (int i = 0; i < num_data_volums; ++i) { - //char dest_path[20]; - //sprintf(dest_path, "/home/work%d", i); - //std::string dest = dest_path; + str = "/home/data/" + ::baidu::common::NumToString(i); + obj_str.SetString(str.c_str(), allocator); + rapidjson::Value data_volum(rapidjson::kObjectType); data_volum.AddMember("size", "10M", allocator); data_volum.AddMember("type", "kEmptyDir", allocator); data_volum.AddMember("medium", "kDisk", allocator); - data_volum.AddMember("dest_path", "/home/work", allocator); + data_volum.AddMember("dest_path", obj_str, allocator); data_volum.AddMember("readonly", false, allocator); data_volum.AddMember("exclusive", false, allocator); data_volum.AddMember("use_symlink", true, allocator); @@ -553,9 +559,9 @@ bool GenerateJson(int num_tasks, int num_data_volums, int num_ports, int num_dat rapidjson::Value tcp(rapidjson::kObjectType); tcp.AddMember("recv_bps_quota", "15M", allocator); - tcp.AddMember("recv_bps_excess", "15M", allocator); + tcp.AddMember("recv_bps_excess", false, allocator); tcp.AddMember("send_bps_quota", "15M", allocator); - tcp.AddMember("send_bps_excess", "15M", allocator); + tcp.AddMember("send_bps_excess", false, allocator); rapidjson::Value blkio(rapidjson::kObjectType); blkio.AddMember("weight", 500, allocator); @@ -563,14 +569,27 @@ bool GenerateJson(int num_tasks, int num_data_volums, int num_ports, int num_dat rapidjson::Value ports(rapidjson::kArrayType); for (int j = 0; j < num_ports; ++j) { rapidjson::Value port(rapidjson::kObjectType); - port.AddMember("name", "port0", allocator); - port.AddMember("port", "1234", allocator); + str = "port" + ::baidu::common::NumToString(i) + ::baidu::common::NumToString(j); + obj_str.SetString(str.c_str(), allocator); + port.AddMember("name", obj_str, allocator); + + str = "123" + ::baidu::common::NumToString(i) + ::baidu::common::NumToString(j); + obj_str.SetString(str.c_str(), allocator); + port.AddMember("port", obj_str, allocator); + ports.PushBack(port, allocator); } + rapidjson::Value package(rapidjson::kObjectType); - package.AddMember("source_path", "ftp://***.baidu.com/home/users/***/exec", allocator); - package.AddMember("dest_path", "/home/spider", allocator); + + str = "ftp://***.baidu.com/home/users/***/exec/" + ::baidu::common::NumToString(i) + "/linkbase.tar.gz";; + obj_str.SetString(str.c_str(), allocator); + package.AddMember("source_path", obj_str, allocator); + + str = "/home/spider/" + ::baidu::common::NumToString(i); + obj_str.SetString(str.c_str(), allocator); + package.AddMember("dest_path", obj_str, allocator); package.AddMember("version", "1.0.0", allocator); rapidjson::Value exec_package(rapidjson::kObjectType); @@ -580,9 +599,17 @@ bool GenerateJson(int num_tasks, int num_data_volums, int num_ports, int num_dat rapidjson::Value data_packages(rapidjson::kArrayType); for (int j = 0; j < num_data_packages; ++j) { + + str = "ftp://***.baidu.com/home/users/***/data/" + ::baidu::common::NumToString(i) + + ::baidu::common::NumToString(j) + "/linkbase.dict.tar.gz"; + obj_str.SetString(str.c_str(), allocator); + rapidjson::Value package(rapidjson::kObjectType); - package.AddMember("source_path", "ftp://***.baidu.com/home/users/***/data", allocator); - package.AddMember("dest_path", "/home/spider", allocator); + package.AddMember("source_path", obj_str, allocator); + + str = "/home/spider/" + ::baidu::common::NumToString(i) + ::baidu::common::NumToString(j) + "/dict"; + obj_str.SetString(str.c_str(), allocator); + package.AddMember("dest_path", obj_str, allocator); package.AddMember("version", "1.0.0", allocator); data_packages.PushBack(package, allocator); @@ -596,9 +623,15 @@ bool GenerateJson(int num_tasks, int num_data_volums, int num_ports, int num_dat rapidjson::Value services(rapidjson::kArrayType); for (int j = 0; j < num_services; ++j) { rapidjson::Value service(rapidjson::kObjectType); - service.AddMember("service_name", "service0", allocator); - service.AddMember("port_name", "port0", allocator); + str = "service" + ::baidu::common::NumToString(i) + ::baidu::common::NumToString(j); + obj_str.SetString(str.c_str(), allocator); + service.AddMember("service_name", obj_str, allocator); + + str = "port" + ::baidu::common::NumToString(i) + ::baidu::common::NumToString(j); + obj_str.SetString(str.c_str(), allocator); + service.AddMember("port_name", obj_str, allocator); service.AddMember("user_bns", false, allocator); + services.PushBack(service, allocator); } rapidjson::Value task(rapidjson::kObjectType); diff --git a/src/sdk/galaxy_sdk_util.cc b/src/sdk/galaxy_sdk_util.cc index ed4f3a0c..997559f2 100755 --- a/src/sdk/galaxy_sdk_util.cc +++ b/src/sdk/galaxy_sdk_util.cc @@ -291,6 +291,11 @@ void PbJobDescription2SdkJobDescription(const ::baidu::galaxy::proto::JobDescrip task.cpu.excess = pb_job.pod().tasks(i).cpu().excess(); task.memory.size = pb_job.pod().tasks(i).memory().size(); task.memory.excess = pb_job.pod().tasks(i).memory().excess(); + task.tcp_throt.recv_bps_quota = pb_job.pod().tasks(i).tcp_throt().recv_bps_quota(); + task.tcp_throt.recv_bps_excess = pb_job.pod().tasks(i).tcp_throt().recv_bps_excess(); + task.tcp_throt.send_bps_quota = pb_job.pod().tasks(i).tcp_throt().send_bps_quota(); + task.tcp_throt.send_bps_excess = pb_job.pod().tasks(i).tcp_throt().send_bps_excess(); + task.blkio.weight = pb_job.pod().tasks(i).blkio().weight(); for (int j = 0; j < pb_job.pod().tasks(i).ports().size(); ++j) { PortRequired port; port.port_name = pb_job.pod().tasks(i).ports(j).port_name(); From 2dd2f81df6036f38353f6acc7cdf13b87f677a58 Mon Sep 17 00:00:00 2001 From: zhoumeiting Date: Fri, 27 May 2016 15:39:51 +0800 Subject: [PATCH 5/8] format json output --- src/client/galaxy_job_action.cc | 44 +++++---------------------------- src/client/galaxy_job_client.cc | 43 +++++++++++++++++++++++++++++--- src/client/galaxy_parse.cc | 3 ++- src/client/galaxy_util.cc | 10 +++++--- src/sdk/galaxy_sdk.h | 1 + src/sdk/galaxy_sdk_appmaster.cc | 1 + src/sdk/galaxy_sdk_util.cc | 2 ++ 7 files changed, 58 insertions(+), 46 deletions(-) diff --git a/src/client/galaxy_job_action.cc b/src/client/galaxy_job_action.cc index d138cdf9..8b414e67 100644 --- a/src/client/galaxy_job_action.cc +++ b/src/client/galaxy_job_action.cc @@ -175,13 +175,6 @@ bool JobAction::RemoveJob(const std::string& jobid) { bool JobAction::ListJobs() { - /*struct timeval t_start; - struct timeval t_finish; - double duration; - - struct timeval tt_start; - gettimeofday(&tt_start, NULL); - */ if(!this->Init()) { return false; } @@ -192,16 +185,8 @@ bool JobAction::ListJobs() { resman_request.user = user_; std::map containers; - //gettimeofday(&t_start, NULL); - bool ret = resman_->ListContainerGroups(resman_request, &resman_response); - /*gettimeofday(&t_finish, NULL); - duration = (t_finish.tv_sec - t_start.tv_sec)*1000000.0 + (t_finish.tv_usec - t_start.tv_usec); - fprintf(stderr, "list container start time is %lf\n", t_start.tv_sec*1000000.0 + t_start.tv_usec); - fprintf(stderr, "list container finish time is %lf\n", t_finish.tv_sec*1000000.0 + t_finish.tv_usec); - fprintf(stderr, "list container duration is %lf\n", duration);*/ - if (ret) { for (uint32_t i = 0; i < resman_response.containers.size(); ++i) { containers[resman_response.containers[i].id] = resman_response.containers[i]; @@ -216,14 +201,7 @@ bool JobAction::ListJobs() { ::baidu::galaxy::sdk::ListJobsRequest request; ::baidu::galaxy::sdk::ListJobsResponse response; request.user = user_; - - //gettimeofday(&t_start, NULL); ret = app_master_->ListJobs(request, &response); - /*gettimeofday(&t_finish, NULL); - duration = (t_finish.tv_sec - t_start.tv_sec)*1000000.0 + (t_finish.tv_usec - t_start.tv_usec); - fprintf(stderr, "list job start time is %lf\n", t_start.tv_sec*1000000.0 + t_start.tv_usec); - fprintf(stderr, "list job finish time is %lf\n", t_finish.tv_sec*1000000.0 + t_finish.tv_usec); - fprintf(stderr, "list duration is %lf\n", duration);*/ if (ret) { baidu::common::TPrinter jobs(12); jobs.AddRow(12, "", "id", "name", "type","status", "stat(r/p/dep/dea/f)", "replica", @@ -315,23 +293,12 @@ bool JobAction::ListJobs() { } } - //gettimeofday(&t_start, NULL); printf("%s\n", jobs.ToString().c_str()); - /*gettimeofday(&t_finish, NULL); - duration = (t_finish.tv_sec - t_start.tv_sec)*1000000.0 + (t_finish.tv_usec - t_start.tv_usec); - fprintf(stderr, "print start time is %lf\n", t_start.tv_sec*1000000.0 + t_start.tv_usec); - fprintf(stderr, "print finish time is %lf\n", t_finish.tv_sec*1000000.0 + t_finish.tv_usec); - fprintf(stderr, "print time is %lf\n", duration);*/ } else { printf("List job failed for reason %s:%s\n", StringStatus(response.error_code.status).c_str(), response.error_code.reason.c_str()); } - /*gettimeofday(&t_finish, NULL); - duration = (t_finish.tv_sec - tt_start.tv_sec)*1000000.0 + (t_finish.tv_usec - tt_start.tv_usec); - fprintf(stderr, "all start time is %lf\n", tt_start.tv_sec*1000000.0 + tt_start.tv_usec); - fprintf(stderr, "all finish time is %lf\n", t_finish.tv_sec*1000000.0 + t_finish.tv_usec); - fprintf(stderr, "all time is %lf\n", duration);*/ return ret; } @@ -510,18 +477,19 @@ bool JobAction::ShowJob(const std::string& jobid) { } printf("podinfo infomation\n"); - ::baidu::common::TPrinter pods(7); - pods.AddRow(7, "", "podid", "endpoint", "status", "version", "start_time", "fail_count"); + ::baidu::common::TPrinter pods(8); + pods.AddRow(8, "", "podid", "endpoint", "version", "status", "fail_count", "start_time", "update_time"); for (uint32_t i = 0; i < response.job.pods.size(); ++i) { size_t pos = response.job.pods[i].podid.rfind("."); std::string podid(response.job.pods[i].podid, pos + 1, response.job.pods[i].podid.size()- (pos + 1)); - pods.AddRow(7, ::baidu::common::NumToString(i).c_str(), + pods.AddRow(8, ::baidu::common::NumToString(i).c_str(), response.job.pods[i].podid.c_str(), response.job.pods[i].endpoint.c_str(), - StringPodStatus(response.job.pods[i].status).c_str(), response.job.pods[i].version.c_str(), + StringPodStatus(response.job.pods[i].status).c_str(), + ::baidu::common::NumToString(response.job.pods[i].fail_count).c_str(), FormatDate(response.job.pods[i].start_time).c_str(), - ::baidu::common::NumToString(response.job.pods[i].fail_count).c_str() + FormatDate(response.job.pods[i].update_time).c_str() ); } printf("%s\n", pods.ToString().c_str()); diff --git a/src/client/galaxy_job_client.cc b/src/client/galaxy_job_client.cc index 5d387d45..98c189e9 100644 --- a/src/client/galaxy_job_client.cc +++ b/src/client/galaxy_job_client.cc @@ -7,6 +7,11 @@ DEFINE_string(f, "", "specify config file"); DEFINE_string(i, "", "specify job id"); DEFINE_string(c, "", "specify cmd"); +DEFINE_string(t, "", "specify task num"); +DEFINE_string(d, "", "specify data_volums num"); +DEFINE_string(p, "", "specify port num"); +DEFINE_string(a, "", "specify packages num in data_package"); +DEFINE_string(s, "", "specify service num"); DECLARE_string(flagfile); @@ -19,11 +24,16 @@ const std::string kGalaxyUsage = "galaxy.\n" " galaxy list\n" " galaxy show -i id\n" " galaxy exec -i id -c cmd\n" - " galaxy json" + " galaxy json [-t num_task -d num_data_volums -p num_port -a num_packages in data_package -s num_service]\n" "Optionss: \n" " -f specify config file, job config file or label config file.\n" " -c specify cmd.\n" - " -i specify job id.\n"; + " -i specify job id.\n" + " -t specify specify task num, default 1.\n" + " -d spicify data_volums num, default 1\n" + " -p specify port num, default 1\n" + " -a specify specify packages num in data_package, default 1\n" + " -s specify specify service num, default 1\n"; int main(int argc, char** argv) { bool ok = true; @@ -91,7 +101,34 @@ int main(int argc, char** argv) { } ok = jobAction->ExecuteCmd(FLAGS_i, FLAGS_c); } else if (strcmp(argv[1], "json") == 0) { - ok = ::baidu::galaxy::client::GenerateJson(2, 2, 2, 2, 2); + int num_tasks = atoi(FLAGS_t.c_str()); + if (FLAGS_t.empty()) { + num_tasks = 1; + } + + int num_data_volums = atoi(FLAGS_d.c_str()); + if (FLAGS_d.empty()) { + num_data_volums =1; + } + + int num_ports = atoi(FLAGS_p.c_str()); + if (FLAGS_p.empty()) { + num_ports = 1; + } + int num_packages = atoi(FLAGS_a.c_str()); + if (FLAGS_a.empty()) { + num_packages = 1; + } + int num_services = atoi(FLAGS_s.c_str()); + if (FLAGS_s.empty()) { + num_services = 1; + } + ok = ::baidu::galaxy::client::GenerateJson(num_tasks, + num_data_volums, + num_ports, + num_packages, + num_services + ); }else { fprintf(stderr, "%s", kGalaxyUsage.c_str()); return -1; diff --git a/src/client/galaxy_parse.cc b/src/client/galaxy_parse.cc index 07abc50a..5a70fbb3 100644 --- a/src/client/galaxy_parse.cc +++ b/src/client/galaxy_parse.cc @@ -365,7 +365,7 @@ int ParseTask(const rapidjson::Value& task_json, ::baidu::galaxy::sdk::TaskDescr time_t timestamp; time(×tamp); - task->id = baidu::common::NumToString(timestamp); + //task->id = baidu::common::NumToString(timestamp); if (!task_json.HasMember("cpu")) { fprintf(stderr, "cpu is required in task\n"); @@ -567,6 +567,7 @@ int ParsePod(const rapidjson::Value& pod_json, ::baidu::galaxy::sdk::PodDescript if (ok != 0) { break; } + task.id = ::baidu::common::NumToString((uint32_t)i); tasks.push_back(task); } diff --git a/src/client/galaxy_util.cc b/src/client/galaxy_util.cc index 9bf5b365..1367f894 100644 --- a/src/client/galaxy_util.cc +++ b/src/client/galaxy_util.cc @@ -13,6 +13,8 @@ #include "rapidjson/prettywriter.h" #include "rapidjson/document.h" #include "rapidjson/stringbuffer.h" +#include "rapidjson/filewritestream.h" +//#include "rapidjson/filestream.h" #include "sdk/galaxy_sdk.h" namespace baidu { @@ -510,7 +512,7 @@ bool GenerateJson(int num_tasks, int num_data_volums, int num_ports, int num_dat deploy.AddMember("interval", 1, allocator); deploy.AddMember("max_per_host", 1, allocator); deploy.AddMember("tag", "example", allocator); - deploy.AddMember("pools", "example", allocator); + deploy.AddMember("pools", "example1,example2", allocator); root.AddMember("deploy", deploy, allocator); @@ -652,10 +654,10 @@ bool GenerateJson(int num_tasks, int num_data_volums, int num_ports, int num_dat root.AddMember("pod", pod, allocator); rapidjson::StringBuffer buffer; - rapidjson::Writer writer(buffer); + rapidjson::PrettyWriter writer(buffer); root.Accept(writer); - std::string reststring = buffer.GetString(); - fprintf(stderr, "%s\n", reststring.c_str()); + std::string str_json = buffer.GetString(); + fprintf(stdout, "%s\n", str_json.c_str()); return true; } diff --git a/src/sdk/galaxy_sdk.h b/src/sdk/galaxy_sdk.h index 73935fb1..4973e5bd 100755 --- a/src/sdk/galaxy_sdk.h +++ b/src/sdk/galaxy_sdk.h @@ -609,6 +609,7 @@ struct PodInfo { PodStatus status; std::string version; int64_t start_time; + int64_t update_time; int32_t fail_count; }; struct JobInfo { diff --git a/src/sdk/galaxy_sdk_appmaster.cc b/src/sdk/galaxy_sdk_appmaster.cc index 3e5e02b4..a5cc3260 100644 --- a/src/sdk/galaxy_sdk_appmaster.cc +++ b/src/sdk/galaxy_sdk_appmaster.cc @@ -212,6 +212,7 @@ bool AppMaster::ShowJob(const ShowJobRequest& request, ShowJobResponse* response pod.status = (PodStatus)pb_response.job().pods(i).status(); pod.version = pb_response.job().pods(i).version(); pod.start_time = pb_response.job().pods(i).start_time(); + pod.update_time = pb_response.job().pods(i).update_time(); pod.fail_count = pb_response.job().pods(i).fail_count(); response->job.pods.push_back(pod); } diff --git a/src/sdk/galaxy_sdk_util.cc b/src/sdk/galaxy_sdk_util.cc index 997559f2..6407a2e2 100755 --- a/src/sdk/galaxy_sdk_util.cc +++ b/src/sdk/galaxy_sdk_util.cc @@ -146,6 +146,8 @@ void FillTaskDescription(const TaskDescription& sdk_task, ::baidu::galaxy::proto::PortRequired* port = task->add_ports(); FillPortRequired(sdk_task.ports[i], port); } + FillTcpthrotRequired(sdk_task.tcp_throt, task->mutable_tcp_throt()); + FillBlkioRequired(sdk_task.blkio, task->mutable_blkio()); FillImagePackage(sdk_task.exe_package, task->mutable_exe_package()); FilldataPackage(sdk_task.data_package, task->mutable_data_package()); for (size_t i = 0; i < sdk_task.services.size(); ++i) { From 4f798e850c53c708529bd8662c07379e07197d07 Mon Sep 17 00:00:00 2001 From: zhoumeiting Date: Fri, 27 May 2016 16:53:23 +0800 Subject: [PATCH 6/8] add generate format json --- src/client/galaxy_job_action.cc | 4 +- src/client/galaxy_res_action.cc | 27 ++++++++++++ src/client/galaxy_res_action.h | 1 + src/client/galaxy_res_client.cc | 11 +++++ src/protocol/resman.proto | 5 ++- src/sdk/galaxy_sdk.h | 10 +++++ src/sdk/galaxy_sdk_appmaster.cc | 12 ++--- src/sdk/galaxy_sdk_resman.cc | 77 +++++++++++++++++++++------------ src/sdk/galaxy_sdk_resman.h | 1 + 9 files changed, 110 insertions(+), 38 deletions(-) diff --git a/src/client/galaxy_job_action.cc b/src/client/galaxy_job_action.cc index 8b414e67..1394cd07 100644 --- a/src/client/galaxy_job_action.cc +++ b/src/client/galaxy_job_action.cc @@ -2,6 +2,7 @@ // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. +#include #include #include "galaxy_job_action.h" @@ -179,7 +180,6 @@ bool JobAction::ListJobs() { return false; } - ::baidu::galaxy::sdk::ListContainerGroupsRequest resman_request; ::baidu::galaxy::sdk::ListContainerGroupsResponse resman_response; resman_request.user = user_; @@ -519,7 +519,7 @@ bool JobAction::ExecuteCmd(const std::string& jobid, const std::string& cmd) { request.user = user_; request.jobid = jobid; request.cmd = cmd; - bool ret = app_master_->ExecuteCmd(request, &response); + bool ret = app_master_->ExecuteCmd(request, &response); if (ret) { printf("Execute job %s\n success", jobid.c_str()); } else { diff --git a/src/client/galaxy_res_action.cc b/src/client/galaxy_res_action.cc index b4e6a959..18d73281 100755 --- a/src/client/galaxy_res_action.cc +++ b/src/client/galaxy_res_action.cc @@ -1341,6 +1341,33 @@ bool ResAction::AssignQuota(const std::string& user, return ret; } +bool ResAction::Preempt(const std::string& container_group_id, const std::string& endpoint) { + + if (container_group_id.empty() || endpoint.empty()) { + fprintf(stderr, "container_group_id and endpoint are needed\n"); + return false; + } + + if(!this->Init()) { + return false; + } + + baidu::galaxy::sdk::PreemptRequest request; + baidu::galaxy::sdk::PreemptResponse response; + request.user = user_; + request.container_group_id = container_group_id; + request.endpoint = endpoint; + bool ret = resman_->Preempt(request, &response); + if (ret) { + printf("Preempt %s\n success", container_group_id.c_str()); + } else { + printf("Preempt %s failed for reason %s:%s\n", + container_group_id.c_str(), StringStatus(response.error_code.status).c_str(), response.error_code.reason.c_str()); + } + return ret; + +} + } // end namespace client } // end namespace galaxy } // end namespace baidu diff --git a/src/client/galaxy_res_action.h b/src/client/galaxy_res_action.h index adbcf8a8..105c2315 100644 --- a/src/client/galaxy_res_action.h +++ b/src/client/galaxy_res_action.h @@ -52,6 +52,7 @@ class ResAction { const std::string& ssd, int replica ); + bool Preempt(const std::string& container_group_id, const std::string& endpoint); private: bool Init(); diff --git a/src/client/galaxy_res_client.cc b/src/client/galaxy_res_client.cc index 3caacebe..da1b07a4 100644 --- a/src/client/galaxy_res_client.cc +++ b/src/client/galaxy_res_client.cc @@ -30,6 +30,7 @@ const std::string kGalaxyUsage = "galaxy_res_client.\n" " galaxy_res_client list_agents [-p pool -t tag]\n" " galaxy_res_client online_agent -e endpoint\n" " galaxy_res_client offline_agent -e endpoint\n\n" + " galaxy_res_client preempt -i container_group_id -e endpoint\n\n" " safemode usage:\n" " galaxy_res_client enter_safemode\n" " galaxy_res_client leave_safemode\n\n" @@ -150,6 +151,16 @@ int main(int argc, char** argv) { return -1; } ok = resAction->OfflineAgent(FLAGS_e); + } else if (strcmp(argv[1], "preempt") == 0) { + if (FLAGS_e.empty()) { + fprintf(stderr, "-e is needed\n"); + return -1; + } + if (FLAGS_i.empty()) { + fprintf(stderr, "-i is needed\n"); + return -1; + } + ok = resAction->Preempt(FLAGS_i, FLAGS_e); } else if (strcmp(argv[1], "status") == 0) { ok = resAction->Status(); } else if (strcmp(argv[1], "create_tag") == 0) { diff --git a/src/protocol/resman.proto b/src/protocol/resman.proto index 3671ff17..234fcd0d 100755 --- a/src/protocol/resman.proto +++ b/src/protocol/resman.proto @@ -317,8 +317,9 @@ message ShowAgentResponse { } message PreemptRequest { - optional string container_group_id = 1; - optional string endpoint = 2; + optional User user = 1; + optional string container_group_id = 2; + optional string endpoint = 3; } message PreemptResponse { diff --git a/src/sdk/galaxy_sdk.h b/src/sdk/galaxy_sdk.h index 4973e5bd..28ffd5ee 100755 --- a/src/sdk/galaxy_sdk.h +++ b/src/sdk/galaxy_sdk.h @@ -643,6 +643,16 @@ struct StopJobResponse { ErrorCode error_code; }; +struct PreemptRequest { + User user; + std::string container_group_id; + std::string endpoint; +}; + +struct PreemptResponse { + ErrorCode error_code; +}; + } //namespace sdk } //namespace galaxy } //namespace baidu diff --git a/src/sdk/galaxy_sdk_appmaster.cc b/src/sdk/galaxy_sdk_appmaster.cc index a5cc3260..f21daccd 100644 --- a/src/sdk/galaxy_sdk_appmaster.cc +++ b/src/sdk/galaxy_sdk_appmaster.cc @@ -61,7 +61,7 @@ bool AppMaster::SubmitJob(const SubmitJobRequest& request, SubmitJobResponse* re &::baidu::galaxy::proto::AppMaster_Stub::SubmitJob, &pb_request, &pb_response, 5, 1); if (!ok) { - response->error_code.reason = "Rpc SendRequest failed"; + response->error_code.reason = "AppMaster Rpc SendRequest failed"; return false; } @@ -86,7 +86,7 @@ bool AppMaster::UpdateJob(const UpdateJobRequest& request, UpdateJobResponse* re &::baidu::galaxy::proto::AppMaster_Stub::UpdateJob, &pb_request, &pb_response, 5, 1); if (!ok) { - response->error_code.reason = "Rpc SendRequest failed"; + response->error_code.reason = "AppMaster Rpc SendRequest failed"; return false; } @@ -110,7 +110,7 @@ bool AppMaster::StopJob(const StopJobRequest& request, StopJobResponse* response &::baidu::galaxy::proto::AppMaster_Stub::StopJob, &pb_request, &pb_response, 5, 1); if (!ok) { - response->error_code.reason = "Rpc SendRequest failed"; + response->error_code.reason = "AppMaster Rpc SendRequest failed"; return false; } response->error_code.status = (Status)pb_response.error_code().status(); @@ -131,7 +131,7 @@ bool AppMaster::RemoveJob(const RemoveJobRequest& request, RemoveJobResponse* re &::baidu::galaxy::proto::AppMaster_Stub::RemoveJob, &pb_request, &pb_response, 5, 1); if (!ok) { - response->error_code.reason = "Rpc SendRequest failed"; + response->error_code.reason = "AppMaster Rpc SendRequest failed"; return false; } @@ -188,7 +188,7 @@ bool AppMaster::ShowJob(const ShowJobRequest& request, ShowJobResponse* response &::baidu::galaxy::proto::AppMaster_Stub::ShowJob, &pb_request, &pb_response, 5, 1); if (!ok) { - response->error_code.reason = "Rpc SendRequest failed"; + response->error_code.reason = "AppMaster Rpc SendRequest failed"; return false; } @@ -230,7 +230,7 @@ bool AppMaster::ExecuteCmd(const ExecuteCmdRequest& request, ExecuteCmdResponse* &::baidu::galaxy::proto::AppMaster_Stub::ExecuteCmd, &pb_request, &pb_response, 5, 1); if (!ok) { - response->error_code.reason = "Rpc SendRequest failed"; + response->error_code.reason = "AppMaster Rpc SendRequest failed"; return false; } diff --git a/src/sdk/galaxy_sdk_resman.cc b/src/sdk/galaxy_sdk_resman.cc index 2c469d05..cf615ea6 100644 --- a/src/sdk/galaxy_sdk_resman.cc +++ b/src/sdk/galaxy_sdk_resman.cc @@ -64,7 +64,7 @@ bool ResourceManager::EnterSafeMode(const EnterSafeModeRequest& request, EnterSa &pb_request, &pb_response, 5, 1); if (!ok) { - response->error_code.reason = "Rpc SendRequest failed"; + response->error_code.reason = "ResourceManager Rpc SendRequest failed"; return false; } @@ -86,7 +86,7 @@ bool ResourceManager::LeaveSafeMode(const LeaveSafeModeRequest& request, LeaveSa &::baidu::galaxy::proto::ResMan_Stub::LeaveSafeMode, &pb_request, &pb_response, 5, 1); if (!ok) { - response->error_code.reason = "Rpc SendRequest failed"; + response->error_code.reason = "ResourceManager Rpc SendRequest failed"; return false; } @@ -107,7 +107,7 @@ bool ResourceManager::Status(const StatusRequest& request, StatusResponse* respo &::baidu::galaxy::proto::ResMan_Stub::Status, &pb_request, &pb_response, 5, 1); if (!ok) { - response->error_code.reason = "Rpc SendRequest failed"; + response->error_code.reason = "ResourceManager Rpc SendRequest failed"; return false; } @@ -167,7 +167,7 @@ bool ResourceManager::CreateContainerGroup(const CreateContainerGroupRequest& re &pb_request, &pb_response, 5, 1); if (!ok) { - response->error_code.reason = "Rpc SendRequest failed"; + response->error_code.reason = "ResourceManager Rpc SendRequest failed"; return false; } @@ -192,7 +192,7 @@ bool ResourceManager::RemoveContainerGroup(const RemoveContainerGroupRequest& re &pb_request, &pb_response, 5,1); if (!ok) { - response->error_code.reason = "Rpc SendRequest failed"; + response->error_code.reason = "ResourceManager Rpc SendRequest failed"; return false; } @@ -217,7 +217,7 @@ bool ResourceManager::UpdateContainerGroup(const UpdateContainerGroupRequest& re &::baidu::galaxy::proto::ResMan_Stub::UpdateContainerGroup, &pb_request, &pb_response, 5, 1); if (!ok) { - response->error_code.reason = "Rpc SendRequest failed"; + response->error_code.reason = "ResourceManager Rpc SendRequest failed"; return false; } @@ -238,7 +238,7 @@ bool ResourceManager::ListContainerGroups(const ListContainerGroupsRequest& requ &::baidu::galaxy::proto::ResMan_Stub::ListContainerGroups, &pb_request, &pb_response, 5, 1); if (!ok) { - response->error_code.reason = "Rpc SendRequest failed"; + response->error_code.reason = "ResourceManager Rpc SendRequest failed"; return false; } @@ -291,7 +291,7 @@ bool ResourceManager::ShowContainerGroup(const ShowContainerGroupRequest& reques &::baidu::galaxy::proto::ResMan_Stub::ShowContainerGroup, &pb_request, &pb_response, 5, 1); if (!ok) { - response->error_code.reason = "Rpc SendRequest failed"; + response->error_code.reason = "ResourceManager Rpc SendRequest failed"; return false; } @@ -398,7 +398,7 @@ bool ResourceManager::AddAgent(const AddAgentRequest& request, AddAgentResponse* &pb_request, &pb_response, 5, 1); if (!ok) { - response->error_code.reason = "Rpc SendRequest failed"; + response->error_code.reason = "ResourceManager Rpc SendRequest failed"; return false; } @@ -421,7 +421,7 @@ bool ResourceManager::ShowAgent(const ShowAgentRequest& request, ShowAgentRespon &pb_request, &pb_response, 5, 1); if (!ok) { - response->error_code.reason = "Rpc SendRequest failed"; + response->error_code.reason = "ResourceManager Rpc SendRequest failed"; return false; } @@ -472,7 +472,7 @@ bool ResourceManager::RemoveAgent(const RemoveAgentRequest& request, RemoveAgent &::baidu::galaxy::proto::ResMan_Stub::RemoveAgent, &pb_request, &pb_response, 5, 1); if (!ok) { - response->error_code.reason = "Rpc SendRequest failed"; + response->error_code.reason = "ResourceManager Rpc SendRequest failed"; return false; } @@ -494,7 +494,7 @@ bool ResourceManager::OnlineAgent(const OnlineAgentRequest& request, OnlineAgent &pb_request, &pb_response, 5, 1); if (!ok) { - response->error_code.reason = "Rpc SendRequest failed"; + response->error_code.reason = "ResourceManager Rpc SendRequest failed"; return false; } @@ -515,7 +515,7 @@ bool ResourceManager::OfflineAgent(const OfflineAgentRequest& request, OfflineAg &::baidu::galaxy::proto::ResMan_Stub::OfflineAgent, &pb_request, &pb_response, 5, 1); if (!ok) { - response->error_code.reason = "Rpc SendRequest failed"; + response->error_code.reason = "ResourceManager Rpc SendRequest failed"; return false; } @@ -537,7 +537,7 @@ bool ResourceManager::ListAgents(const ListAgentsRequest& request, ListAgentsRes &pb_request, &pb_response, 5, 1); if (!ok) { - response->error_code.reason = "Rpc SendRequest failed"; + response->error_code.reason = "ResourceManager Rpc SendRequest failed"; return false; } @@ -593,7 +593,7 @@ bool ResourceManager::CreateTag(const CreateTagRequest& request, CreateTagRespon &::baidu::galaxy::proto::ResMan_Stub::CreateTag, &pb_request, &pb_response, 5, 1); if (!ok) { - response->error_code.reason = "Rpc SendRequest failed"; + response->error_code.reason = "ResourceManager Rpc SendRequest failed"; return false; } @@ -614,7 +614,7 @@ bool ResourceManager::ListTags(const ListTagsRequest& request, ListTagsResponse* &::baidu::galaxy::proto::ResMan_Stub::ListTags, &pb_request, &pb_response, 5, 1); if (!ok) { - response->error_code.reason = "Rpc SendRequest failed"; + response->error_code.reason = "ResourceManager Rpc SendRequest failed"; return false; } @@ -640,7 +640,7 @@ bool ResourceManager::ListAgentsByTag(const ListAgentsByTagRequest& request, Lis &::baidu::galaxy::proto::ResMan_Stub::ListAgentsByTag, &pb_request, &pb_response, 5, 1); if (!ok) { - response->error_code.reason = "Rpc SendRequest failed"; + response->error_code.reason = "ResourceManager Rpc SendRequest failed"; return false; } @@ -694,7 +694,7 @@ bool ResourceManager::GetTagsByAgent(const GetTagsByAgentRequest& request, GetTa &::baidu::galaxy::proto::ResMan_Stub::GetTagsByAgent, &pb_request, &pb_response, 5, 1); if (!ok) { - response->error_code.reason = "Rpc SendRequest failed"; + response->error_code.reason = "ResourceManager Rpc SendRequest failed"; return false; } @@ -723,7 +723,7 @@ bool ResourceManager::AddAgentToPool(const AddAgentToPoolRequest& request, AddAg &::baidu::galaxy::proto::ResMan_Stub::AddAgentToPool, &pb_request, &pb_response, 5, 1); if (!ok) { - response->error_code.reason = "Rpc SendRequest failed"; + response->error_code.reason = "ResourceManager Rpc SendRequest failed"; return false; } @@ -745,7 +745,7 @@ bool ResourceManager::RemoveAgentFromPool(const RemoveAgentFromPoolRequest& requ &::baidu::galaxy::proto::ResMan_Stub::RemoveAgentFromPool, &pb_request, &pb_response, 5, 1); if (!ok) { - response->error_code.reason = "Rpc SendRequest failed"; + response->error_code.reason = "ResourceManager Rpc SendRequest failed"; return false; } @@ -768,7 +768,7 @@ bool ResourceManager::ListAgentsByPool(const ListAgentsByPoolRequest& request, L &::baidu::galaxy::proto::ResMan_Stub::ListAgentsByPool, &pb_request, &pb_response, 5, 1); if (!ok) { - response->error_code.reason = "Rpc SendRequest failed"; + response->error_code.reason = "ResourceManager Rpc SendRequest failed"; return false; } @@ -821,7 +821,7 @@ bool ResourceManager::GetPoolByAgent(const GetPoolByAgentRequest& request, GetPo &::baidu::galaxy::proto::ResMan_Stub::GetPoolByAgent, &pb_request, &pb_response, 5, 1); if (!ok) { - response->error_code.reason = "Rpc SendRequest failed"; + response->error_code.reason = "ResourceManager Rpc SendRequest failed"; return false; } @@ -847,7 +847,7 @@ bool ResourceManager::AddUser(const AddUserRequest& request, AddUserResponse* re &pb_request, &pb_response, 5, 1); if (!ok) { - response->error_code.reason = "Rpc SendRequest failed"; + response->error_code.reason = "ResourceManager Rpc SendRequest failed"; return false; } @@ -871,7 +871,7 @@ bool ResourceManager::RemoveUser(const RemoveUserRequest& request, RemoveUserRes &pb_request, &pb_response, 5, 1); if (!ok) { - response->error_code.reason = "Rpc SendRequest failed"; + response->error_code.reason = "ResourceManager Rpc SendRequest failed"; return false; } @@ -895,7 +895,7 @@ bool ResourceManager::ListUsers(const ListUsersRequest& request, ListUsersRespon &pb_request, &pb_response, 5, 1); if (!ok) { - response->error_code.reason = "Rpc SendRequest failed"; + response->error_code.reason = "ResourceManager Rpc SendRequest failed"; return false; } @@ -922,7 +922,7 @@ bool ResourceManager::ShowUser(const ShowUserRequest& request, ShowUserResponse* &pb_request, &pb_response, 5, 1); if (!ok) { - response->error_code.reason = "Rpc SendRequest failed"; + response->error_code.reason = "ResourceManager Rpc SendRequest failed"; return false; } @@ -968,7 +968,7 @@ bool ResourceManager::GrantUser(const GrantUserRequest& request, GrantUserRespon &::baidu::galaxy::proto::ResMan_Stub::GrantUser, &pb_request, &pb_response, 5, 1); if (!ok) { - response->error_code.reason = "Rpc SendRequest failed"; + response->error_code.reason = "ResourceManager Rpc SendRequest failed"; return false; } @@ -998,7 +998,7 @@ bool ResourceManager::AssignQuota(const AssignQuotaRequest& request, AssignQuota &pb_request, &pb_response, 5, 1); if (!ok) { - response->error_code.reason = "Rpc SendRequest failed"; + response->error_code.reason = "ResourceManager Rpc SendRequest failed"; return false; } @@ -1011,6 +1011,27 @@ bool ResourceManager::AssignQuota(const AssignQuotaRequest& request, AssignQuota return true; } +bool ResourceManager::Preempt(const PreemptRequest& request, PreemptResponse* response) { + ::baidu::galaxy::proto::PreemptRequest pb_request; + ::baidu::galaxy::proto::PreemptResponse pb_response; + FillUser(request.user, pb_request.mutable_user()); + pb_request.set_container_group_id(request.container_group_id); + pb_request.set_endpoint(request.endpoint); + bool ok = rpc_client_->SendRequest(res_stub_, + &::baidu::galaxy::proto::ResMan_Stub::Preempt, + &pb_request, &pb_response, 5, 1); + if (!ok) { + response->error_code.reason = "ResourceManager Rpc SendRequest failed"; + return false; + } + response->error_code.status = (::baidu::galaxy::sdk::Status)pb_response.error_code().status(); + response->error_code.reason = pb_response.error_code().reason(); + if (response->error_code.status != kOk) { + return false; + } + return true; +} + } //namespace sdk } //namespace galaxy } //namespace baidu diff --git a/src/sdk/galaxy_sdk_resman.h b/src/sdk/galaxy_sdk_resman.h index 2f02abb1..9a642850 100644 --- a/src/sdk/galaxy_sdk_resman.h +++ b/src/sdk/galaxy_sdk_resman.h @@ -51,6 +51,7 @@ class ResourceManager { bool ShowUser(const ShowUserRequest& request, ShowUserResponse* response); bool GrantUser(const GrantUserRequest& request, GrantUserResponse* response); bool AssignQuota(const AssignQuotaRequest& request, AssignQuotaResponse* response); + bool Preempt(const PreemptRequest& request, PreemptResponse* response); private: ::galaxy::ins::sdk::InsSDK* nexus_; RpcClient* rpc_client_; From b954769af87e8bb6ea15c1d3242e0b6876d9cfbd Mon Sep 17 00:00:00 2001 From: zhoumeiting Date: Fri, 27 May 2016 17:18:34 +0800 Subject: [PATCH 7/8] clear flag --- galaxy.flag | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/galaxy.flag b/galaxy.flag index d469a146..170af61d 100644 --- a/galaxy.flag +++ b/galaxy.flag @@ -1,9 +1,9 @@ ---nexus_servers=wanghaitao01-pc:8868,wanghaitao01-pc:8869,wanghaitao01-pc:8870,wanghaitao01-pc:8871,wanghaitao01-pc:8872 +--nexus_servers= --appworker_endpoint=127.0.0.1:8221 --appworker_container_id=test-1 --logbufsecs=0 ---username=zhoumeiting ---token=1234 ---nexus_addr=cq01-ps-dev377.cq01.baidu.com:8868 +--username=default +--token=default +--nexus_addr= --nexus_root=/galaxy3 --resman_path=/resman From 560bcdab13a856662c4b40cf017380442ed0c2eb Mon Sep 17 00:00:00 2001 From: zhoumeiting Date: Fri, 27 May 2016 17:19:37 +0800 Subject: [PATCH 8/8] clear flag --- galaxy.flag | 1 + 1 file changed, 1 insertion(+) diff --git a/galaxy.flag b/galaxy.flag index 170af61d..fe2a5a5b 100644 --- a/galaxy.flag +++ b/galaxy.flag @@ -7,3 +7,4 @@ --nexus_addr= --nexus_root=/galaxy3 --resman_path=/resman +--appmaster=/appmaster