From 83111b77f96c8b4148d5cb43f5162b0bd09e5828 Mon Sep 17 00:00:00 2001 From: Jim Halfpenny Date: Mon, 13 Jan 2014 12:51:18 +0000 Subject: [PATCH 01/10] Added TLS support for cm_shell There is now a new command line option [-t|-tls] that will enable TLS for the connection to the Cloudera API. Note that enabling TLS will not automatically change the port the client connects to CM on, so -p 7183 (default CM TLS port) needs to be added to the command line for the connection to succeed. jims-air:cm_shell jhalfpenny$ ./cmps.py -H test-cluster.cloudera.com -p 7183 -t Enter Username: admin Enter Password: Welcome to the Cloudera Manager Console Select a cluster using 'show clusters' and 'use' cloudera> (cherry picked from commit 995297ab21b4d858e84b649631a6ea98d81936d3) --- python/src/cm_shell/cmps.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/python/src/cm_shell/cmps.py b/python/src/cm_shell/cmps.py index f728c129..8b90ada8 100755 --- a/python/src/cm_shell/cmps.py +++ b/python/src/cm_shell/cmps.py @@ -593,6 +593,7 @@ def main(): parser.add_argument('--password', action='store', dest='password') parser.add_argument('-e', '--execute', action='store', dest='execute') parser.add_argument('-s', '--seperator', action='store', dest='seperator') + parser.add_argument('-t', '--tls', action='store_const', dest='use_tls', const=True, default=False) args = parser.parse_args() # Check if a username was suplied, if not, prompt the user @@ -605,12 +606,12 @@ def main(): # Attempt to authenticate using the API global api - api = ApiResource(args.hostname, args.port, args.username, args.password) + api = ApiResource(args.hostname, args.port, args.username, args.password, args.use_tls) try: api.echo("ping") except ApiException: try: - api = ApiResource(args.hostname, args.port, args.username, args.password, version=1) + api = ApiResource(args.hostname, args.port, args.username, args.password, args.use_tls, version=1) api.echo("ping") except ApiException: print("Unable to Authenticate") From 8bf307ea99ef637ffde95f7d69343542d9ca62ef Mon Sep 17 00:00:00 2001 From: Jim Halfpenny Date: Mon, 13 Jan 2014 15:29:48 +0000 Subject: [PATCH 02/10] Added TLS parameter to timeseries.py example (cherry picked from commit 33afb301fe226141b9273a59ed4803811103718a) --- python/examples/timeseries.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/python/examples/timeseries.py b/python/examples/timeseries.py index 44823293..d10c94e0 100644 --- a/python/examples/timeseries.py +++ b/python/examples/timeseries.py @@ -69,6 +69,7 @@ CM_HOST = 'localhost' CM_USER = 'admin' CM_PASSWD = 'admin' +CM_USE_TLS = False LOG = logging.getLogger(__name__) @@ -76,7 +77,7 @@ class TimeSeriesQuery(object): """ """ def __init__(self): - self._api = ApiResource(CM_HOST, username=CM_USER, password=CM_PASSWD) + self._api = ApiResource(CM_HOST, username=CM_USER, password=CM_PASSWD, use_tls=CM_USE_TLS) def query(self, query, from_time, to_time): return self._api.query_timeseries(query, from_time, to_time) From d7754d6005008ad72bd4cd4fe19d6e4459eaf5f8 Mon Sep 17 00:00:00 2001 From: Philip Langdale Date: Mon, 30 Mar 2015 14:29:06 -0700 Subject: [PATCH 03/10] Python API: Add Makefile target to generate api docs. We currently generate the epydocs from the publishing script, which means someone just looking at the repo doesn't know what command line to use. Let's add a Makefile rule to do the work. This work is based on: https://github.com/cloudera/cm_api/pull/15 (cherry picked from commit 6b402f82891319b8d63456f2afa21160ab9ca8db) --- python/.gitignore | 1 + python/Makefile | 10 ++++++++++ 2 files changed, 11 insertions(+) diff --git a/python/.gitignore b/python/.gitignore index 3cfea3fd..5489775f 100644 --- a/python/.gitignore +++ b/python/.gitignore @@ -1,5 +1,6 @@ build/ dist/ +doc/ *.egg-info/ .git-hash diff --git a/python/Makefile b/python/Makefile index dfe2a557..e556e230 100644 --- a/python/Makefile +++ b/python/Makefile @@ -14,10 +14,15 @@ # See the License for the specific language governing permissions and # limitations under the License. +DOCDIR ?= doc + .PHONY: help help: @echo 'The build targets are:' @echo ' dist : Create a source distribution tarball' + @echo ' test : Run unit tests' + @echo ' doc : Generate epydoc format documentation' + @echo ' clean : Clean up all generated files' .PHONY: test test: @@ -33,5 +38,10 @@ dist: test clean: rm -rf *.egg-info rm -rf dist + rm -rf doc rm -rf build find . -name *.py[co] -exec rm -f {} \; + +.PHONY: doc +doc: + epydoc -v --html -o $(DOCDIR) -n python-cm_api `find src -name "*.py"` From 56cfd429572dc2aa5911f6bbd8a863daec37084b Mon Sep 17 00:00:00 2001 From: Sean Story Date: Thu, 18 Sep 2014 14:05:06 -0500 Subject: [PATCH 04/10] Update README.md gramatical typo (cherry picked from commit ebea21f4d681cceb47ce8925a37fefd7d2a9da3a) --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index a3999f47..f2129b35 100644 --- a/README.md +++ b/README.md @@ -19,7 +19,7 @@ This client code allows you to interact with Cloudera Manager to: * Upgrade services running on your cluster * Access time-series data on resource utilitization for any activity in the system * Read logs for all processes in the system as well as stdout and stderr -* Programmatically configuration all aspects of your deployment +* Programmatically configure all aspects of your deployment * Collect diagnostic data to aid in debugging issues * Run distributed commands to manage auto-failover, host decommissioning and more * View all events and alerts that have occurred in the system From 5ce9cc6c8ed7286c680a356960c791f577acf38e Mon Sep 17 00:00:00 2001 From: arindamchoudhury Date: Thu, 15 Jan 2015 11:58:16 +0100 Subject: [PATCH 05/10] Update timeseries.py just a typo (cherry picked from commit d55932e645011968c8e3bf830eef110fa91705ad) --- python/examples/timeseries.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/examples/timeseries.py b/python/examples/timeseries.py index d10c94e0..3fa9056d 100644 --- a/python/examples/timeseries.py +++ b/python/examples/timeseries.py @@ -17,7 +17,7 @@ """ -Given a query amd an optional time range, show all timeseries +Given a query and an optional time range, show all timeseries data that matches the given query in the given time range. Usage: %s [options] query From 7df97450dc434dbecfa3be43d376ffc66960e4b4 Mon Sep 17 00:00:00 2001 From: Salman Haq Date: Thu, 24 Apr 2014 08:57:29 -0400 Subject: [PATCH 06/10] Add optional support for SOCKS proxies via PySocks. (cherry picked from commit a1148395554fe54864fec6e9dd841d4f6e401a5a) --- python/setup.py | 4 ++++ python/src/cm_api/http_client.py | 13 +++++++++++++ python/src/cm_api/resource.py | 10 ++++++++++ 3 files changed, 27 insertions(+) diff --git a/python/setup.py b/python/setup.py index e8845c4d..f5a22d15 100644 --- a/python/setup.py +++ b/python/setup.py @@ -33,6 +33,9 @@ if platform == 'darwin': install_requires += ['readline'] +# Optional PySocks support +extras_require = dict(Socks=['PySocks >= 1.5.0']) + setup( name = 'cm_api', version = '10.0.0', # Compatible with API v10 (CM 5.4) @@ -43,6 +46,7 @@ # Project uses simplejson, so ensure that it gets installed or upgraded # on the target machine install_requires = install_requires, + extras_require = extras_require, author = 'Cloudera, Inc.', author_email = 'scm-users@cloudera.org', diff --git a/python/src/cm_api/http_client.py b/python/src/cm_api/http_client.py index 81722403..08b65c3a 100644 --- a/python/src/cm_api/http_client.py +++ b/python/src/cm_api/http_client.py @@ -14,11 +14,24 @@ # See the License for the specific language governing permissions and # limitations under the License. +import os import cookielib import logging import posixpath import types import urllib + +try: + import socks + import socket + socks_server = os.environ.get("SOCKS_SERVER", None) + if socks_server: + host, port = socks_server.split(":") + socks.setdefaultproxy(socks.PROXY_TYPE_SOCKS5, host, int(port)) + socket.socket = socks.socksocket +except ImportError: + pass + import urllib2 __docformat__ = "epytext" diff --git a/python/src/cm_api/resource.py b/python/src/cm_api/resource.py index fda0e53f..f96b9c6a 100644 --- a/python/src/cm_api/resource.py +++ b/python/src/cm_api/resource.py @@ -14,6 +14,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +import os try: import json except ImportError: @@ -22,6 +23,15 @@ import posixpath import time import socket +try: + import socks + socks_server = os.environ.get("SOCKS_SERVER", None) + if socks_server: + host, port = socks_server.split(":") + socks.setdefaultproxy(socks.PROXY_TYPE_SOCKS5, host, int(port)) + socket.socket = socks.socksocket +except ImportError: + pass import urllib2 LOG = logging.getLogger(__name__) From 9996d8b2d1a485cb6be451beadbbe9e186a9fc48 Mon Sep 17 00:00:00 2001 From: Philip Langdale Date: Tue, 7 Apr 2015 10:29:22 -0700 Subject: [PATCH 07/10] [python tests] Make test_yarn consistent with respect to importing 'json' It's unclear if this is futile or not. I don't believe the library works on Python < 2.6 any more, and I'm very sure unit tests don't work on Python < 2.7. (cherry picked from commit 2d6be5425f662895c781e6975eb3c019816ad22e) --- python/src/cm_api_tests/test_yarn.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/python/src/cm_api_tests/test_yarn.py b/python/src/cm_api_tests/test_yarn.py index f6e22a53..ae104b58 100644 --- a/python/src/cm_api_tests/test_yarn.py +++ b/python/src/cm_api_tests/test_yarn.py @@ -15,13 +15,17 @@ # limitations under the License. import datetime -import json import unittest from cm_api.endpoints.clusters import * from cm_api.endpoints.services import * from cm_api.endpoints.types import * from cm_api_tests import utils +try: + import json +except ImportError: + import simplejson as json + class TestYarn(unittest.TestCase): def test_get_yarn_applications(self): From 0d437f99584510b61ebbdca7f226aad708b892c3 Mon Sep 17 00:00:00 2001 From: Philip Zeyliger Date: Mon, 13 Apr 2015 11:15:00 -0700 Subject: [PATCH 08/10] OPSAPS-26038. Conditionalize 'roles' argument. Tests running against older CM versions had the following error: test_support_bundle45 fails with error: Unrecognized property: 'roles' (error 400) As it turns out, we have the capability to set an API version when using the python client, and the client should take care of things like this, so I've done so. To test this, I ran: $ cp ~/src/cm_api/python/src/cm_api/endpoints/cms.py ./target/env/lib/python2.7/site-packages/cm_api/endpoints/cms.py $ systest.py --agents="cdh5+parcels@nightly53-{1..4}.ent.cloudera.com" run_tests -n "test_support_bundle:SupportBundleTests.test_support_bundle_45" --disable-pre-validations --version cm53 Before my change (that I manually cherrypicked in the first hand): ApiException: Unrecognized property: 'roles' (error 400) ---------------------------------------------------------------------- Ran 1 test in 0.499s FAILED (errors=1) False After: *********************************************************************** * End test_support_bundle_45 (test_support_bundle.SupportBundleTests) * *********************************************************************** ---------------------------------------------------------------------- Ran 1 test in 128.929s I also tested against nightly (5.5) to make sure this didn't regress. It passed. $./systest.py --agents="cdh5+parcels@nightly-{1..4}.ent.cloudera.com" run_tests -n "test_support_bundle:SupportBundleTests.test_support_bundle_45" --disable-pre-validations (cherry picked from commit e58b3ccf2a5b63bf42460fde1f9b1c2e596bf440) --- python/src/cm_api/endpoints/cms.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/python/src/cm_api/endpoints/cms.py b/python/src/cm_api/endpoints/cms.py index 6c14e742..55f0196c 100644 --- a/python/src/cm_api/endpoints/cms.py +++ b/python/src/cm_api/endpoints/cms.py @@ -201,9 +201,10 @@ def collect_diagnostic_data_45(self, end_datetime, bundle_size_bytes, cluster_na args = { 'endTime': end_datetime.isoformat(), 'bundleSizeBytes': bundle_size_bytes, - 'clusterName': cluster_name, - 'roles': roles + 'clusterName': cluster_name } + if self._get_resource_root().version >= 10: + args['roles'] = roles return self._cmd('collectDiagnosticData', data=args) def hosts_decommission(self, host_names): From 042a8cdcf953745f8d482a19808c6b6f87cd84c1 Mon Sep 17 00:00:00 2001 From: mthakkar Date: Fri, 24 Apr 2015 11:31:16 -0700 Subject: [PATCH 09/10] [java] Updated with v10 java source --- java/enunciate.xml | 6 +- java/pom.xml | 4 +- .../com/cloudera/api/ApiRootResource.java | 7 + .../api/ClouderaManagerClientBuilder.java | 117 ++++++-- .../com/cloudera/api/model/ApiCluster.java | 1 + .../ApiCollectDiagnosticDataArguments.java | 24 ++ .../model/ApiHdfsReplicationArguments.java | 26 +- .../cloudera/api/model/ApiHealthSummary.java | 2 +- .../api/model/ApiHostInstallArguments.java | 9 + .../api/model/ApiMigrateRolesArguments.java | 98 +++++++ .../model/ApiRollingUpgradeClusterArgs.java | 10 +- .../model/ApiRollingUpgradeServicesArgs.java | 153 ++++++++++ ...nApplicationDiagnosticsCollectionArgs.java | 26 ++ .../cloudera/api/v10/AuditsResourceV10.java | 261 ++++++++++++++++++ .../cloudera/api/v10/ClustersResourceV10.java | 68 +++++ .../cloudera/api/v10/HostsResourceV10.java | 70 +++++ .../api/v10/RoleCommandsResourceV10.java | 55 ++++ .../com/cloudera/api/v10/RootResourceV10.java | 46 +++ .../cloudera/api/v10/ServicesResourceV10.java | 118 ++++++++ .../com/cloudera/api/v10/package-info.java | 21 ++ .../api/v2/ClouderaManagerResourceV2.java | 4 + .../com/cloudera/api/v2/RootResourceV2.java | 2 +- .../cloudera/api/v2/ServicesResourceV2.java | 2 + .../com/cloudera/api/v8/AuditsResourceV8.java | 8 +- .../api/ClouderaManagerClientBuilderTest.java | 21 ++ .../com/cloudera/api/model/ApiModelTest.java | 2 + 26 files changed, 1121 insertions(+), 40 deletions(-) create mode 100644 java/src/main/java/com/cloudera/api/model/ApiMigrateRolesArguments.java create mode 100644 java/src/main/java/com/cloudera/api/model/ApiRollingUpgradeServicesArgs.java create mode 100644 java/src/main/java/com/cloudera/api/v10/AuditsResourceV10.java create mode 100644 java/src/main/java/com/cloudera/api/v10/ClustersResourceV10.java create mode 100644 java/src/main/java/com/cloudera/api/v10/HostsResourceV10.java create mode 100644 java/src/main/java/com/cloudera/api/v10/RoleCommandsResourceV10.java create mode 100644 java/src/main/java/com/cloudera/api/v10/RootResourceV10.java create mode 100644 java/src/main/java/com/cloudera/api/v10/ServicesResourceV10.java create mode 100644 java/src/main/java/com/cloudera/api/v10/package-info.java diff --git a/java/enunciate.xml b/java/enunciate.xml index 963e95e2..b8f56f4e 100644 --- a/java/enunciate.xml +++ b/java/enunciate.xml @@ -2,18 +2,18 @@ xsi:noNamespaceSchemaLocation="http://enunciate.codehaus.org/schemas/enunciate-1.27.xsd"> - + - + com.cloudera.api cloudera-manager-api Cloudera Manager API - 5.3.0 + 5.4.0 2.7.5 14.0 2.1.0 2.1 - 4.8.2 + 4.11 UTF-8 true com.cloudera.api.shaded diff --git a/java/src/main/java/com/cloudera/api/ApiRootResource.java b/java/src/main/java/com/cloudera/api/ApiRootResource.java index eb7ab0fc..f3029e20 100644 --- a/java/src/main/java/com/cloudera/api/ApiRootResource.java +++ b/java/src/main/java/com/cloudera/api/ApiRootResource.java @@ -24,6 +24,7 @@ import com.cloudera.api.v7.RootResourceV7; import com.cloudera.api.v8.RootResourceV8; import com.cloudera.api.v9.RootResourceV9; +import com.cloudera.api.v10.RootResourceV10; import javax.ws.rs.Consumes; import javax.ws.rs.GET; @@ -94,6 +95,12 @@ public interface ApiRootResource { @Path("/v9") RootResourceV9 getRootV9(); + /** + * @return The v10 root resource. + */ + @Path("/v10") + RootResourceV10 getRootV10(); + /** * Fetch the current API version supported by the server. *

diff --git a/java/src/main/java/com/cloudera/api/ClouderaManagerClientBuilder.java b/java/src/main/java/com/cloudera/api/ClouderaManagerClientBuilder.java index 52732514..1b6721d8 100644 --- a/java/src/main/java/com/cloudera/api/ClouderaManagerClientBuilder.java +++ b/java/src/main/java/com/cloudera/api/ClouderaManagerClientBuilder.java @@ -18,15 +18,9 @@ import com.fasterxml.jackson.jaxrs.json.JacksonJsonProvider; import com.google.common.annotations.VisibleForTesting; - -import org.apache.cxf.configuration.jsse.TLSClientParameters; -import org.apache.cxf.feature.AbstractFeature; -import org.apache.cxf.feature.LoggingFeature; -import org.apache.cxf.jaxrs.client.ClientConfiguration; -import org.apache.cxf.jaxrs.client.JAXRSClientFactoryBean; -import org.apache.cxf.jaxrs.client.WebClient; -import org.apache.cxf.transport.http.HTTPConduit; -import org.apache.cxf.transports.http.configuration.HTTPClientPolicy; +import com.google.common.cache.CacheBuilder; +import com.google.common.cache.CacheLoader; +import com.google.common.cache.LoadingCache; import java.net.URI; import java.net.URISyntaxException; @@ -34,9 +28,19 @@ import java.security.cert.X509Certificate; import java.util.Arrays; import java.util.concurrent.TimeUnit; + import javax.net.ssl.TrustManager; import javax.net.ssl.X509TrustManager; +import org.apache.cxf.configuration.jsse.TLSClientParameters; +import org.apache.cxf.feature.AbstractFeature; +import org.apache.cxf.feature.LoggingFeature; +import org.apache.cxf.jaxrs.client.ClientConfiguration; +import org.apache.cxf.jaxrs.client.JAXRSClientFactoryBean; +import org.apache.cxf.jaxrs.client.WebClient; +import org.apache.cxf.transport.http.HTTPConduit; +import org.apache.cxf.transports.http.configuration.HTTPClientPolicy; + public class ClouderaManagerClientBuilder { public static final int DEFAULT_TCP_PORT = 7180; public static final long DEFAULT_CONNECTION_TIMEOUT = 0; @@ -61,6 +65,38 @@ public class ClouderaManagerClientBuilder { private boolean validateCn = true; private TrustManager[] trustManagers = null; + /** + * Cache JAXRSClientFactoryBean per proxyType. + * + * We need a cache because CXF stores stubs + * ({@link org.apache.cxf.jaxrs.model.ClassResourceInfo} objects) as a reference + * inside JAXRSServiceFactoryBean, which is composed within instances of + * {@link JAXRSClientFactoryBean}. + * + * This avoids: + * - creating a lot of temporaries generated during the proxy creation for + * each client + * + * - ensures that different proxies of the same type actually share the same + * ClassResourceInfo, thus reducing aggregate usage + * + * Also, as a useful side effect, generates proxies with cached proxy types faster. + */ + private static final LoadingCache, JAXRSClientFactoryBean> + clientStaticResources = + CacheBuilder.newBuilder() + .softValues() + .build( + new CacheLoader, JAXRSClientFactoryBean>(){ + @Override + public JAXRSClientFactoryBean load(Class proxyType) throws Exception { + JAXRSClientFactoryBean clientFactoryBean = new JAXRSClientFactoryBean(); + clientFactoryBean.setResourceClass(proxyType); + clientFactoryBean.setProvider(new JacksonJsonProvider(new ApiObjectMapper())); + return clientFactoryBean; + } + }); + public ClouderaManagerClientBuilder withBaseURL(URL baseUrl) { this.baseUrl = baseUrl; return this; @@ -154,27 +190,40 @@ public void setTrustManagers(TrustManager[] managers) { trustManagers = managers; } + /** + * Build an ApiRootResource proxy object for communicating with the remote server. + * @return an ApiRootResource proxy object + */ public ApiRootResource build() { return build(ApiRootResource.class); } + /** + * Build a client proxy, for a specific proxy type. + * @param proxyType proxy type class + * @return client proxy stub + */ protected T build(Class proxyType) { - JAXRSClientFactoryBean bean = new JAXRSClientFactoryBean(); - String address = generateAddress(); - boolean isTlsEnabled = address.startsWith("https://"); - bean.setAddress(address); - if (username != null) { - bean.setUsername(username); - bean.setPassword(password); - } - if (enableLogging) { - bean.setFeatures(Arrays.asList(new LoggingFeature())); + T rootResource; + // Synchronized on the class to correlate with the scope of clientStaticResources + // We want to ensure that the shared bean isn't set concurrently in multiple callers + synchronized (ClouderaManagerClientBuilder.class) { + JAXRSClientFactoryBean bean = + cleanFactory(clientStaticResources.getUnchecked(proxyType)); + bean.setAddress(address); + if (username != null) { + bean.setUsername(username); + bean.setPassword(password); + } + + if (enableLogging) { + bean.setFeatures(Arrays.asList(new LoggingFeature())); + } + rootResource = bean.create(proxyType); } - bean.setResourceClass(proxyType); - bean.setProvider(new JacksonJsonProvider(new ApiObjectMapper())); - T rootResource = bean.create(proxyType); + boolean isTlsEnabled = address.startsWith("https://"); ClientConfiguration config = WebClient.getConfig(rootResource); HTTPConduit conduit = (HTTPConduit) config.getConduit(); if (isTlsEnabled) { @@ -197,6 +246,13 @@ else if (trustManagers != null) { return rootResource; } + private static JAXRSClientFactoryBean cleanFactory(JAXRSClientFactoryBean bean) { + bean.setUsername(null); + bean.setPassword(null); + bean.setFeatures(Arrays.asList()); + return bean; + } + /** * Closes the transport level conduit in the client. Reopening a new * connection, requires creating a new client object using the build() @@ -215,6 +271,21 @@ public static void closeClient(ApiRootResource root) { conduit.close(); } + /** + * Clears any cached resources shared during build operations + * across instances of this class. + * + * This includes shared proxy/stub information, that will be automatically + * garbage collected when used heap memory in the JVM + * nears max heap memory in the JVM. + * + * In general, it is unnecessary to invoke this method, unless you are + * concerned with reducing used heap memory in the JVM. + */ + public static void clearCachedResources() { + clientStaticResources.invalidateAll(); + } + /** A trust manager that will accept all certificates. */ private static class AcceptAllTrustManager implements X509TrustManager { @@ -229,7 +300,5 @@ public void checkServerTrusted(X509Certificate[] chain, String authType) { public X509Certificate[] getAcceptedIssuers() { return null; } - } - } diff --git a/java/src/main/java/com/cloudera/api/model/ApiCluster.java b/java/src/main/java/com/cloudera/api/model/ApiCluster.java index afd39265..1874ce08 100644 --- a/java/src/main/java/com/cloudera/api/model/ApiCluster.java +++ b/java/src/main/java/com/cloudera/api/model/ApiCluster.java @@ -23,6 +23,7 @@ import javax.xml.bind.annotation.XmlElementWrapper; import javax.xml.bind.annotation.XmlRootElement; import javax.xml.bind.annotation.XmlType; + import java.util.List; /** diff --git a/java/src/main/java/com/cloudera/api/model/ApiCollectDiagnosticDataArguments.java b/java/src/main/java/com/cloudera/api/model/ApiCollectDiagnosticDataArguments.java index 96a47a72..4b5107cf 100644 --- a/java/src/main/java/com/cloudera/api/model/ApiCollectDiagnosticDataArguments.java +++ b/java/src/main/java/com/cloudera/api/model/ApiCollectDiagnosticDataArguments.java @@ -16,6 +16,9 @@ package com.cloudera.api.model; +import java.util.List; + +import javax.xml.bind.annotation.XmlElement; import javax.xml.bind.annotation.XmlRootElement; /** @@ -31,6 +34,7 @@ public class ApiCollectDiagnosticDataArguments { private String ticketNumber; private String comments; private String clusterName; + private List roles; /** * The maximum approximate bundle size of the output file @@ -115,4 +119,24 @@ public String getClusterName() { public void setClusterName(String clusterName) { this.clusterName = clusterName; } + + /** + * List of roles for which to get logs and metrics. + * + * If set, this restricts the roles for log and metrics collection + * to the list specified. + * + * If empty, the default is to get logs for all roles (in the selected + * cluster, if one is selected). + * + * Introduced in API v10 of the API. + */ + @XmlElement + public List getRoles() { + return roles; + } + + public void setRoles(List roles) { + this.roles = roles; + } } diff --git a/java/src/main/java/com/cloudera/api/model/ApiHdfsReplicationArguments.java b/java/src/main/java/com/cloudera/api/model/ApiHdfsReplicationArguments.java index f0ccecd1..29b4a61b 100644 --- a/java/src/main/java/com/cloudera/api/model/ApiHdfsReplicationArguments.java +++ b/java/src/main/java/com/cloudera/api/model/ApiHdfsReplicationArguments.java @@ -44,6 +44,7 @@ public class ApiHdfsReplicationArguments { private boolean skipChecksumChecks; private Boolean skipTrash; private ReplicationStrategy replicationStrategy; + private Boolean preserveXAttrs; /** * The strategy for distributing the file replication tasks among the mappers @@ -224,6 +225,9 @@ public void setPreserveBlockSize(boolean preserveBlockSize) { /** * Whether to preserve the HDFS owner, group and permissions. Defaults to * false. + * Starting from V10, it also preserves ACLs. Defaults to null (no preserve). + * ACLs is preserved if both clusters enable ACL support, and replication + * ignores any ACL related failures. */ @XmlElement public boolean getPreservePermissions() { @@ -284,6 +288,21 @@ public void setReplicationStrategy(ReplicationStrategy replicationStrategy) { this.replicationStrategy = replicationStrategy; } + /** + * Whether to preserve XAttrs, default to false + * This is introduced in V10. To preserve XAttrs, both CDH versions + * should be >= 5.2. Replication fails if either cluster does not support + * XAttrs. + */ + @XmlElement + public Boolean getPreserveXAttrs() { + return preserveXAttrs; + } + + public void setPreserveXAttrs(Boolean preserveXAttrs) { + this.preserveXAttrs = preserveXAttrs; + } + @Override public String toString() { return Objects.toStringHelper(this) @@ -304,6 +323,7 @@ public String toString() { .add("skipChecksumChecks", skipChecksumChecks) .add("skipTrash", skipTrash) .add("replicationStrategy", replicationStrategy) + .add("preserveXAttrs", preserveXAttrs) .toString(); } @@ -327,7 +347,8 @@ public boolean equals(Object o) { Objects.equal(logPath, other.getLogPath()) && skipChecksumChecks == other.getSkipChecksumChecks() && Objects.equal(skipTrash, other.getSkipTrash()) && - Objects.equal(replicationStrategy, other.getReplicationStrategy())); + Objects.equal(replicationStrategy, other.getReplicationStrategy()) && + Objects.equal(preserveXAttrs, other.getPreserveXAttrs())); } @Override @@ -336,6 +357,7 @@ public int hashCode() { mapreduceServiceName, schedulerPoolName, numMaps, dryRun, bandwidthPerMap, abortOnError, removeMissingFiles, preserveReplicationCount, preserveBlockSize, preservePermissions, - logPath, skipChecksumChecks, skipTrash, replicationStrategy); + logPath, skipChecksumChecks, skipTrash, replicationStrategy, + preserveXAttrs); } } diff --git a/java/src/main/java/com/cloudera/api/model/ApiHealthSummary.java b/java/src/main/java/com/cloudera/api/model/ApiHealthSummary.java index 8bbf3df1..330d4e51 100644 --- a/java/src/main/java/com/cloudera/api/model/ApiHealthSummary.java +++ b/java/src/main/java/com/cloudera/api/model/ApiHealthSummary.java @@ -33,5 +33,5 @@ public enum ApiHealthSummary { /** The subject is in concerning health */ CONCERNING, /** The subject is in bad health */ - BAD + BAD; } diff --git a/java/src/main/java/com/cloudera/api/model/ApiHostInstallArguments.java b/java/src/main/java/com/cloudera/api/model/ApiHostInstallArguments.java index 3bd312a7..d632f756 100644 --- a/java/src/main/java/com/cloudera/api/model/ApiHostInstallArguments.java +++ b/java/src/main/java/com/cloudera/api/model/ApiHostInstallArguments.java @@ -141,6 +141,15 @@ public void setPassword(String password) { /** * The private key to authenticate with the hosts. * Specify either this or a password. + *
+ * The private key, if specified, needs to be a + * standard PEM-encoded key as a single string, with all line breaks + * replaced with the line-feed control character '\n'. + *
+ * A value will typically look like the following string: + *
+ * -----BEGIN RSA PRIVATE KEY-----\n[base-64 encoded key]\n-----END RSA PRIVATE KEY----- + *
*/ @XmlElement public String getPrivateKey() { diff --git a/java/src/main/java/com/cloudera/api/model/ApiMigrateRolesArguments.java b/java/src/main/java/com/cloudera/api/model/ApiMigrateRolesArguments.java new file mode 100644 index 00000000..4182de42 --- /dev/null +++ b/java/src/main/java/com/cloudera/api/model/ApiMigrateRolesArguments.java @@ -0,0 +1,98 @@ +// Licensed to Cloudera, Inc. under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. Cloudera, Inc. licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.cloudera.api.model; + +import com.cloudera.api.ApiUtils; + +import com.google.common.base.Objects; + +import java.util.List; +import javax.xml.bind.annotation.XmlElement; +import javax.xml.bind.annotation.XmlRootElement; + +@XmlRootElement(name="migrateRolesArgs") +public class ApiMigrateRolesArguments { + + private List roleNamesToMigrate; + private String destinationHostId; + private boolean clearStaleRoleData; + + /** + * The list of role names to migrate. + */ + @XmlElement + public List getRoleNamesToMigrate() { + return roleNamesToMigrate; + } + + public void setRoleNamesToMigrate(List roleNamesToMigrate) { + this.roleNamesToMigrate = roleNamesToMigrate; + } + + /** + * The ID of the host to which the roles should be migrated. + */ + @XmlElement + public String getDestinationHostId() { + return destinationHostId; + } + + public void setDestinationHostId(String destinationHostId) { + this.destinationHostId = destinationHostId; + } + + /** + * Delete existing stale role data, if any. For example, when migrating + * a NameNode, if the destination host has stale data in the NameNode data + * directories (possibly because a NameNode role was previously located + * there), this stale data will be deleted before migrating the role. + */ + @XmlElement + public boolean getClearStaleRoleData() { + return clearStaleRoleData; + } + + public void setClearStaleRoleData(boolean clearStaleRoleData) { + this.clearStaleRoleData = clearStaleRoleData; + } + + @Override + public String toString() { + return Objects.toStringHelper(this) + .add("roleNamesToMigrate", roleNamesToMigrate) + .add("destinationHostId", destinationHostId) + .add("clearStaleRoleData", clearStaleRoleData) + .toString(); + } + + @Override + public int hashCode() { + return Objects.hashCode( + roleNamesToMigrate, + destinationHostId, + clearStaleRoleData); + } + + @Override + public boolean equals(Object o) { + ApiMigrateRolesArguments that = ApiUtils.baseEquals(this, o); + return this == that || (that != null + && Objects.equal(this.roleNamesToMigrate, that.roleNamesToMigrate) + && Objects.equal(this.destinationHostId, that.destinationHostId) + && Objects.equal(this.clearStaleRoleData, that.clearStaleRoleData)); + } +} diff --git a/java/src/main/java/com/cloudera/api/model/ApiRollingUpgradeClusterArgs.java b/java/src/main/java/com/cloudera/api/model/ApiRollingUpgradeClusterArgs.java index 2fe330c2..2ecfca09 100644 --- a/java/src/main/java/com/cloudera/api/model/ApiRollingUpgradeClusterArgs.java +++ b/java/src/main/java/com/cloudera/api/model/ApiRollingUpgradeClusterArgs.java @@ -21,6 +21,10 @@ import javax.xml.bind.annotation.XmlElement; import javax.xml.bind.annotation.XmlRootElement; +/** + * Rolling upgrade arguments used in the CDH Upgrade Command. Part of + * ApiCdhUpgradeArgs. + */ @XmlRootElement(name = "rollingUpgradeClusterArgs") public class ApiRollingUpgradeClusterArgs { @@ -41,7 +45,7 @@ public Integer getSlaveBatchSize() { return slaveBatchSize; } - public void setSlaveBatchSize(int slaveBatchSize) { + public void setSlaveBatchSize(Integer slaveBatchSize) { this.slaveBatchSize = slaveBatchSize; } @@ -55,7 +59,7 @@ public Integer getSleepSeconds() { return sleepSeconds; } - public void setSleepSeconds(int sleepSeconds) { + public void setSleepSeconds(Integer sleepSeconds) { this.sleepSeconds = sleepSeconds; } @@ -73,7 +77,7 @@ public Integer getSlaveFailCountThreshold() { return slaveFailCountThreshold; } - public void setSlaveFailCountThreshold(int slaveFailCountThreshold) { + public void setSlaveFailCountThreshold(Integer slaveFailCountThreshold) { this.slaveFailCountThreshold = slaveFailCountThreshold; } diff --git a/java/src/main/java/com/cloudera/api/model/ApiRollingUpgradeServicesArgs.java b/java/src/main/java/com/cloudera/api/model/ApiRollingUpgradeServicesArgs.java new file mode 100644 index 00000000..2ad7e140 --- /dev/null +++ b/java/src/main/java/com/cloudera/api/model/ApiRollingUpgradeServicesArgs.java @@ -0,0 +1,153 @@ +// Licensed to Cloudera, Inc. under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. Cloudera, Inc. licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.cloudera.api.model; + +import java.util.List; + +import javax.xml.bind.annotation.XmlElement; +import javax.xml.bind.annotation.XmlRootElement; + +import com.cloudera.api.ApiUtils; +import com.google.common.base.Objects; + +/** + * Arguments used for Rolling Upgrade command. + */ +@XmlRootElement(name="rollingUpgradeServicesArgs") +public class ApiRollingUpgradeServicesArgs { + + private Integer slaveBatchSize; + private Integer sleepSeconds; + private Integer slaveFailCountThreshold; + private String upgradeFromCdhVersion; + private String upgradeToCdhVersion; + private List upgradeServiceNames; + + /** + * Current CDH Version of the services. Example versions are: + * "5.1.0", "5.2.2" or "5.4.0" + */ + @XmlElement + public String getUpgradeFromCdhVersion() { + return upgradeFromCdhVersion; + } + + public void setUpgradeFromCdhVersion(String upgradeFromCdhVersion) { + this.upgradeFromCdhVersion = upgradeFromCdhVersion; + } + + /** + * Target CDH Version for the services. The CDH version should already + * be present and activated on the nodes. Example versions are: + * "5.1.0", "5.2.2" or "5.4.0" + */ + @XmlElement + public String getUpgradeToCdhVersion() { + return upgradeToCdhVersion; + } + + public void setUpgradeToCdhVersion(String upgradeToCdhVersion) { + this.upgradeToCdhVersion = upgradeToCdhVersion; + } + + /** + * Number of hosts with slave roles to upgrade at a time. + * Must be greater than zero. Default is 1. + */ + @XmlElement + public Integer getSlaveBatchSize() { + return slaveBatchSize; + } + + public void setSlaveBatchSize(Integer slaveBatchSize) { + this.slaveBatchSize = slaveBatchSize; + } + + /** + * Number of seconds to sleep between restarts of slave host batches. + * + * Must be greater than or equal to 0. Default is 0. + */ + @XmlElement + public Integer getSleepSeconds() { + return sleepSeconds; + } + + public void setSleepSeconds(Integer sleepSeconds) { + this.sleepSeconds = sleepSeconds; + } + + /** + * The threshold for number of slave host batches that are allowed to fail + * to restart before the entire command is considered failed. + * + * Must be greater than or equal to 0. Default is 0. + *

+ * This argument is for ADVANCED users only. + *

+ */ + @XmlElement + public Integer getSlaveFailCountThreshold() { + return slaveFailCountThreshold; + } + + public void setSlaveFailCountThreshold(Integer slaveFailCountThreshold) { + this.slaveFailCountThreshold = slaveFailCountThreshold; + } + + /** + * List of services to upgrade. + * Only the services that support rolling upgrade should be included. + */ + @XmlElement + public List getUpgradeServiceNames() { + return upgradeServiceNames; + } + + public void setUpgradeServiceNames(List upgradeServiceNames) { + this.upgradeServiceNames = upgradeServiceNames; + } + + @Override + public String toString() { + return Objects.toStringHelper(this) + .add("slaveBatchSize", slaveBatchSize) + .add("slaveFailCountThreshold", slaveFailCountThreshold) + .add("sleepSeconds", sleepSeconds) + .add("upgradeFromCdhVersion", upgradeFromCdhVersion) + .add("upgradeToCdhVersion", upgradeToCdhVersion) + .add("upgradeServiceNames", upgradeServiceNames) + .toString(); + } + + @Override + public boolean equals(Object o) { + ApiRollingUpgradeServicesArgs other = ApiUtils.baseEquals(this, o); + return this == other || (other != null && + Objects.equal(slaveBatchSize, other.slaveBatchSize) && + Objects.equal(slaveFailCountThreshold, other.slaveFailCountThreshold) && + Objects.equal(sleepSeconds, other.sleepSeconds) && + Objects.equal(upgradeFromCdhVersion, other.upgradeFromCdhVersion) && + Objects.equal(upgradeToCdhVersion, other.upgradeToCdhVersion) && + Objects.equal(upgradeServiceNames, other.upgradeServiceNames)); + } + + @Override + public int hashCode() { + return Objects.hashCode(slaveBatchSize, slaveFailCountThreshold, sleepSeconds, + upgradeFromCdhVersion, upgradeToCdhVersion, upgradeServiceNames); + } +} diff --git a/java/src/main/java/com/cloudera/api/model/ApiYarnApplicationDiagnosticsCollectionArgs.java b/java/src/main/java/com/cloudera/api/model/ApiYarnApplicationDiagnosticsCollectionArgs.java index e2f82b91..a812ca8b 100644 --- a/java/src/main/java/com/cloudera/api/model/ApiYarnApplicationDiagnosticsCollectionArgs.java +++ b/java/src/main/java/com/cloudera/api/model/ApiYarnApplicationDiagnosticsCollectionArgs.java @@ -26,6 +26,8 @@ @XmlRootElement(name = "yarnApplicationDiagnosticsCollectionArgs") public class ApiYarnApplicationDiagnosticsCollectionArgs { private List applicationIds; + private String ticketNumber; + private String comments; /** * Id's of the applications whose diagnostics data has to be collected @@ -38,4 +40,28 @@ public List getApplicationIds() { public void setApplicationIds(List applicationIds) { this.applicationIds = applicationIds; } + + /** + * Ticket Number of the Cloudera Support Ticket + */ + @XmlElement + public String getTicketNumber() { + return ticketNumber; + } + + public void setTicketNumber(String ticketNumber) { + this.ticketNumber = ticketNumber; + } + + /** + * Comments to add to the support bundle + */ + @XmlElement + public String getComments() { + return comments; + } + + public void setComments(String comments) { + this.comments = comments; + } } diff --git a/java/src/main/java/com/cloudera/api/v10/AuditsResourceV10.java b/java/src/main/java/com/cloudera/api/v10/AuditsResourceV10.java new file mode 100644 index 00000000..4b30c50b --- /dev/null +++ b/java/src/main/java/com/cloudera/api/v10/AuditsResourceV10.java @@ -0,0 +1,261 @@ +// Copyright (c) 2014 Cloudera, Inc. All rights reserved. +package com.cloudera.api.v10; + +import com.cloudera.api.v8.AuditsResourceV8; + +import javax.ws.rs.Consumes; +import javax.ws.rs.DefaultValue; +import javax.ws.rs.GET; +import javax.ws.rs.Path; +import javax.ws.rs.Produces; +import javax.ws.rs.QueryParam; +import javax.ws.rs.core.MediaType; +import javax.ws.rs.core.StreamingOutput; + +import static com.cloudera.api.Parameters.DATE_TIME_NOW; +import static com.cloudera.api.Parameters.QUERY; + +@Consumes({ MediaType.APPLICATION_JSON }) +@Produces({ MediaType.APPLICATION_JSON }) + +/** + * Fetches and streams the requested audit events. + *

+ * By default, this call will fetch the first 100 audit events (sorted from most + * recent to oldest) corresponding to a 1 day window based on provided end time + * (which defaults to the current CM server time). The startTime and + * endTime parameters can be used to control the window being queried. + *

+ * Audit events for CM managed services are only retrieved if Cloudera + * Navigator server is running. + * + * @param maxResults Maximum number of audits to return + * @param resultOffset Offset of audits to return + * @param startTime Start of the period to query (defaults to 1 day ago + * relative to endTime) + * @param endTime End of the period to query (defaults to current time) + * @param query + * The query to filter out audits in the system. It accepts + * querying the intersection of a list of constraints, + * joined together with semicolons (without spaces). For example: + *

+ *
+ *
command==listStatus
+ *
looks for audits with listStatus command.
+ *
command==listStatus;username!=foo
+ *
looks for audits with listStatus command but excludes + * audits generated by foo username
+ *
command==listStatus;source==*oozie*
+ *
looks for audits with listStatus command and source that + * contains the string 'oozie'. + *
+ *
+ * + * Following are valid selectors for the query (if applicable to the + * audit): + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + *
Selector Description SCM HDFS HBase Hive Impala Sentry
service Cloudera Manager Service x x x x x x
operation Operation name x x x x x x
username User name x x x x x x
impersonator Impersonator x x x x x
ip_address IP Address x x x x x x
allowed Whether the request was allowed or denied x x x x x x
qualifier Column qualifier x
source Source resource of the operation x x x x x
destination Destination resource of the operation x x x x
hostIpAddress Host IP Address x
role Cloudera Manager Role x
family Column family x
database_name Database name x x x
table_name Table name x x x x
object_type Type of object being handled x x x
operation_text Command/query text x x x
+ *

+ * The only supported operator is ";" (Boolean AND). Boolean OR is + * not supported. + *

+ * The supported comparators are == and != + * Note that "LIKE" comparison is supported using the wild card syntax, + * for example foo==*value*. Asterisk is interpreted as a wild + * card character and must not be part of the value. (LIKE comparison + * queries are converted to standard SQL LIKE syntax, so any % (%25) + * character in a value that also contains a wild card will be + * interpreted as a wild card.) + *

+ * Values for time related query parameters (startTime and + * endTime) should be ISO8601 timestamps. + *

+ * + * @return List of audits in descending order of timestamp + */ +public interface AuditsResourceV10 extends AuditsResourceV8 { + + @GET + @Path("/stream") + public StreamingOutput streamAudits( + @QueryParam(value = "maxResults") + @DefaultValue("100") Integer maxResults, + @QueryParam(value = "resultOffset") + @DefaultValue("0") Integer resultOffset, + @QueryParam("startTime") + String startTime, + @QueryParam("endTime") + @DefaultValue(DATE_TIME_NOW) + String endTime, + @QueryParam(QUERY) String query); +} diff --git a/java/src/main/java/com/cloudera/api/v10/ClustersResourceV10.java b/java/src/main/java/com/cloudera/api/v10/ClustersResourceV10.java new file mode 100644 index 00000000..7b094192 --- /dev/null +++ b/java/src/main/java/com/cloudera/api/v10/ClustersResourceV10.java @@ -0,0 +1,68 @@ +// Licensed to Cloudera, Inc. under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. Cloudera, Inc. licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.cloudera.api.v10; + +import static com.cloudera.api.Parameters.CLUSTER_NAME; + +import com.cloudera.api.model.ApiCommand; +import com.cloudera.api.model.ApiRollingUpgradeServicesArgs; +import com.cloudera.api.v9.ClustersResourceV9; + +import javax.ws.rs.Consumes; +import javax.ws.rs.POST; +import javax.ws.rs.Path; +import javax.ws.rs.PathParam; +import javax.ws.rs.Produces; +import javax.ws.rs.core.MediaType; + +@Consumes({ MediaType.APPLICATION_JSON }) +@Produces({ MediaType.APPLICATION_JSON }) +public interface ClustersResourceV10 extends ClustersResourceV9 { + + /** + * @return The services resource handler. + */ + @Override + @Path("/{clusterName}/services") + public ServicesResourceV10 getServicesResource( + @PathParam(CLUSTER_NAME) String clusterName); + + /** + * Command to do a rolling upgrade of specific services in the given cluster + * + * This command does not handle any services that don't support rolling + * upgrades. The command will throw an error and not start if upgrade of any + * such service is requested. + * + * This command does not upgrade the full CDH Cluster. You should normally + * use the upgradeCDH Command for upgrading the cluster. This is primarily + * helpful if you need to need to recover from an upgrade failure or for + * advanced users to script an alternative to the upgradeCdhCommand. + * + * This command expects the binaries to be available on hosts and activated. + * It does not change any binaries on the hosts. + * + * @param clusterName The name of the cluster. + * @param args Arguments for the rolling upgrade command. + * @return Information about the submitted command. + */ + @POST + @Consumes + @Path("/{clusterName}/commands/rollingUpgrade") + public ApiCommand rollingUpgrade( + @PathParam(CLUSTER_NAME) String clusterName, + ApiRollingUpgradeServicesArgs args); +} diff --git a/java/src/main/java/com/cloudera/api/v10/HostsResourceV10.java b/java/src/main/java/com/cloudera/api/v10/HostsResourceV10.java new file mode 100644 index 00000000..0c07fc5b --- /dev/null +++ b/java/src/main/java/com/cloudera/api/v10/HostsResourceV10.java @@ -0,0 +1,70 @@ +// Licensed to Cloudera, Inc. under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. Cloudera, Inc. licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.cloudera.api.v10; + +import static com.cloudera.api.Parameters.HOST_ID; + +import com.cloudera.api.model.ApiCommand; +import com.cloudera.api.model.ApiMigrateRolesArguments; +import com.cloudera.api.v2.HostsResourceV2; + +import javax.ws.rs.Consumes; +import javax.ws.rs.POST; +import javax.ws.rs.Path; +import javax.ws.rs.PathParam; +import javax.ws.rs.Produces; +import javax.ws.rs.core.MediaType; + +@Consumes({ MediaType.APPLICATION_JSON }) +@Produces({ MediaType.APPLICATION_JSON }) +public interface HostsResourceV10 extends HostsResourceV2 { + + /** + * Migrate roles to a different host. + *

+ * This command applies only to HDFS NameNode, JournalNode, and Failover + * Controller roles. In order to migrate these roles: + *

    + *
  • HDFS High Availability must be enabled, using quorum-based storage.
    • + *
  • HDFS must not be configured to use a federated nameservice.
    • + *
+ * Migrating a NameNode or JournalNode role requires cluster downtime. + * HDFS, along with all of its dependent services, will be stopped at the + * beginning of the migration process, and restarted at its conclusion. + *

If the active NameNode is selected for migration, a manual failover + * will be performed before the role is migrated. The role will remain in + * standby mode after the migration is complete. + *

When migrating a NameNode role, the co-located Failover Controller + * role must be migrated as well if automatic failover is enabled. The + * Failover Controller role name must be included in the list of role + * names to migrate specified in the arguments to this command (it will + * not be included implicitly). This command does not allow a Failover + * Controller role to be moved by itself, although it is possible to move + * a JournalNode independently. + *

+ * Available since API v10. + * + * @param hostId The ID of the host on which the roles to migrate currently + * reside + * @param args Arguments for the command. + * @return Information about the submitted command. + */ + @POST + @Path("/{hostId}/commands/migrateRoles") + public ApiCommand migrateRoles(@PathParam(HOST_ID) String hostId, + ApiMigrateRolesArguments args); +} diff --git a/java/src/main/java/com/cloudera/api/v10/RoleCommandsResourceV10.java b/java/src/main/java/com/cloudera/api/v10/RoleCommandsResourceV10.java new file mode 100644 index 00000000..6bdd83a4 --- /dev/null +++ b/java/src/main/java/com/cloudera/api/v10/RoleCommandsResourceV10.java @@ -0,0 +1,55 @@ +// Licensed to Cloudera, Inc. under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. Cloudera, Inc. licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.cloudera.api.v10; + +import com.cloudera.api.model.ApiBulkCommandList; +import com.cloudera.api.model.ApiRoleNameList; +import com.cloudera.api.v8.RoleCommandsResourceV8; + +import javax.ws.rs.Consumes; +import javax.ws.rs.POST; +import javax.ws.rs.Path; +import javax.ws.rs.Produces; +import javax.ws.rs.core.MediaType; + +@Consumes({ MediaType.APPLICATION_JSON }) +@Produces({ MediaType.APPLICATION_JSON }) +public interface RoleCommandsResourceV10 extends RoleCommandsResourceV8 { + /** + * Refresh a role's data. + *

+ * For MapReduce services, this command should be executed on JobTracker + * roles. It refreshes the role's queue and node information. + *

+ * For HDFS services, this command should be executed on NameNode or + * DataNode roles. For NameNodes, it refreshes the role's node list. + * For DataNodes, it refreshes the role's data directory list. + *

+ * For YARN services, this command should be executed on ResourceManager + * roles. It refreshes the role's queue and node information. + *

+ * Available since API v1. DataNode data directories refresh available + * since API v10. + * + * @param roleNames The names of the roles. + * @return A list of submitted commands. + */ + @POST + @Path("/refresh") + public ApiBulkCommandList refreshCommand( + ApiRoleNameList roleNames); +} diff --git a/java/src/main/java/com/cloudera/api/v10/RootResourceV10.java b/java/src/main/java/com/cloudera/api/v10/RootResourceV10.java new file mode 100644 index 00000000..f8832b82 --- /dev/null +++ b/java/src/main/java/com/cloudera/api/v10/RootResourceV10.java @@ -0,0 +1,46 @@ +// Licensed to Cloudera, Inc. under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. Cloudera, Inc. licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.cloudera.api.v10; + +import com.cloudera.api.v9.RootResourceV9; + +import javax.ws.rs.Path; + +@Path("") +public interface RootResourceV10 extends RootResourceV9 { + + /** + * @return The clusters resource handler. + */ + @Override + @Path("/clusters") + public ClustersResourceV10 getClustersResource(); + + /** + * @return The hosts resource handler. + */ + @Override + @Path("/hosts") + public HostsResourceV10 getHostsResource(); + + /** + * @return Audits resource handler. + */ + @Override + @Path("/audits") + public AuditsResourceV10 getAuditsResource(); + +} diff --git a/java/src/main/java/com/cloudera/api/v10/ServicesResourceV10.java b/java/src/main/java/com/cloudera/api/v10/ServicesResourceV10.java new file mode 100644 index 00000000..db095beb --- /dev/null +++ b/java/src/main/java/com/cloudera/api/v10/ServicesResourceV10.java @@ -0,0 +1,118 @@ +// Licensed to Cloudera, Inc. under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. Cloudera, Inc. licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.cloudera.api.v10; + +import static com.cloudera.api.Parameters.SERVICE_NAME; + +import com.cloudera.api.model.ApiCommand; +import com.cloudera.api.v8.ServicesResourceV8; + +import javax.ws.rs.Consumes; +import javax.ws.rs.POST; +import javax.ws.rs.Path; +import javax.ws.rs.PathParam; +import javax.ws.rs.Produces; +import javax.ws.rs.core.MediaType; + +@Consumes({ MediaType.APPLICATION_JSON }) +@Produces({ MediaType.APPLICATION_JSON }) +public interface ServicesResourceV10 extends ServicesResourceV8 { + + /** + * @return The role command resource handler. + */ + @Override + @Path("/{serviceName}/roleCommands") + public RoleCommandsResourceV10 getRoleCommandsResource( + @PathParam(SERVICE_NAME) String serviceName); + + /** + * Runs Hue's dumpdata command. + * + * Available since API v10. + * + * @param serviceName The name of the service + * @return Information about the submitted command. + */ + @POST + @Path("/{serviceName}/commands/hueDumpDb") + public ApiCommand hueDumpDbCommand( + @PathParam(SERVICE_NAME) String serviceName); + + /** + * Runs Hue's loaddata command. + * + * Available since API v10. + * + * @param serviceName The name of the service + * @return Information about the submitted command. + */ + @POST + @Path("/{serviceName}/commands/hueLoadDb") + public ApiCommand hueLoadDbCommand( + @PathParam(SERVICE_NAME) String serviceName); + + /** + * Runs Hue's syncdb command. + * + * Available since API v10. + * + * @param serviceName The name of the service + * @return Information about the submitted command. + */ + @POST + @Path("/{serviceName}/commands/hueSyncDb") + public ApiCommand hueSyncDbCommand( + @PathParam(SERVICE_NAME) String serviceName); + + /** + * Create the Oozie Server Database. Only works with embedded postgresql + * database. + *

+ * This command is to be run whenever a new user and database need to be + * created in the embedded postgresql database for an Oozie service. This + * command should usually be followed by a call to createOozieDb. + *

+ * Available since API v10. + * + * @param serviceName + * Name of the Oozie service on which to run the command. + * @return Information about the submitted command + */ + @POST + @Consumes() + @Path("/{serviceName}/commands/oozieCreateEmbeddedDatabase") + public ApiCommand oozieCreateEmbeddedDatabaseCommand( + @PathParam(SERVICE_NAME) String serviceName); + + /** + * Create the Sqoop2 Server Database tables. + *

+ * This command is to be run whenever a new database has been specified. Will + * do nothing if tables already exist. Will not perform an upgrade. Only + * available when Sqoop2 Server is stopped. + *

+ * Available since API v10. + * + * @param serviceName Name of the Sentry service on which to run the command. + * @return Information about the submitted command + */ + @POST + @Consumes() + @Path("/{serviceName}/commands/sqoopCreateDatabaseTables") + public ApiCommand sqoopCreateDatabaseTablesCommand( + @PathParam(SERVICE_NAME) String serviceName); +} diff --git a/java/src/main/java/com/cloudera/api/v10/package-info.java b/java/src/main/java/com/cloudera/api/v10/package-info.java new file mode 100644 index 00000000..4e5f703e --- /dev/null +++ b/java/src/main/java/com/cloudera/api/v10/package-info.java @@ -0,0 +1,21 @@ +// Licensed to Cloudera, Inc. under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. Cloudera, Inc. licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +/** + * API version 10, introduced in Cloudera Manager 5.4.0. + */ + +package com.cloudera.api.v10; \ No newline at end of file diff --git a/java/src/main/java/com/cloudera/api/v2/ClouderaManagerResourceV2.java b/java/src/main/java/com/cloudera/api/v2/ClouderaManagerResourceV2.java index 386092d5..e95bfce9 100644 --- a/java/src/main/java/com/cloudera/api/v2/ClouderaManagerResourceV2.java +++ b/java/src/main/java/com/cloudera/api/v2/ClouderaManagerResourceV2.java @@ -80,6 +80,10 @@ public ApiDeployment getDeployment( * all changes will be rolled back leaving the system exactly as it was * before calling this method. The system will never be left in a state * where part of the deployment is created and other parts are not. + *

+ * If the submitted deployment contains entities that require Cloudera + * Enterprise license, then the license should be provided to Cloudera Manager + * before making this API call. * * @param deployment The deployment to create * @param deleteCurrentDeployment If true, the current deployment is deleted diff --git a/java/src/main/java/com/cloudera/api/v2/RootResourceV2.java b/java/src/main/java/com/cloudera/api/v2/RootResourceV2.java index 72ed0dea..f96d3de8 100644 --- a/java/src/main/java/com/cloudera/api/v2/RootResourceV2.java +++ b/java/src/main/java/com/cloudera/api/v2/RootResourceV2.java @@ -39,7 +39,7 @@ public interface RootResourceV2 extends RootResourceV1 { public ClouderaManagerResourceV2 getClouderaManagerResource(); /** - * @return The clusters resource handler. + * @return The hosts resource handler. */ @Override @Path("/hosts") diff --git a/java/src/main/java/com/cloudera/api/v2/ServicesResourceV2.java b/java/src/main/java/com/cloudera/api/v2/ServicesResourceV2.java index 52492c4c..a73d4775 100644 --- a/java/src/main/java/com/cloudera/api/v2/ServicesResourceV2.java +++ b/java/src/main/java/com/cloudera/api/v2/ServicesResourceV2.java @@ -129,6 +129,8 @@ public ApiCommand hdfsCreateTmpDir( /** * Creates the Oozie Database Schema in the configured database. + * This command does not create database. This command creates only tables + * required by Oozie. To create database, please refer to oozieCreateEmbeddedDatabase() * *

* Available since API v2. diff --git a/java/src/main/java/com/cloudera/api/v8/AuditsResourceV8.java b/java/src/main/java/com/cloudera/api/v8/AuditsResourceV8.java index 7853b7c4..05c201cb 100644 --- a/java/src/main/java/com/cloudera/api/v8/AuditsResourceV8.java +++ b/java/src/main/java/com/cloudera/api/v8/AuditsResourceV8.java @@ -38,10 +38,10 @@ public interface AuditsResourceV8 extends AuditsResource { * Fetch audit events from Cloudera Manager (CM) and CM managed services * like HDFS, HBase, Impala, Hive, and Sentry. *

- * By default, this call will fetch all audit events corresponding to a - * 1 day window based on provided end time (which defaults to the current - * CM server time). The startTime and endTime parameters - * can be used to control the window being queried. + * By default, this call will fetch the first 100 audit events (sorted from most + * recent to oldest) corresponding to a 1 day window based on provided end time + * (which defaults to the current CM server time). The startTime and + * endTime parameters can be used to control the window being queried. *

* Audit events for CM managed services are only retrieved if Cloudera * Navigator server is running. diff --git a/java/src/test/java/com/cloudera/api/ClouderaManagerClientBuilderTest.java b/java/src/test/java/com/cloudera/api/ClouderaManagerClientBuilderTest.java index fefb54bb..bbf020d7 100644 --- a/java/src/test/java/com/cloudera/api/ClouderaManagerClientBuilderTest.java +++ b/java/src/test/java/com/cloudera/api/ClouderaManagerClientBuilderTest.java @@ -22,6 +22,7 @@ import java.net.URL; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; import static org.junit.Assert.fail; public class ClouderaManagerClientBuilderTest { @@ -75,4 +76,24 @@ public void testURLGeneration() throws MalformedURLException { // expected } } + + @Test + public void testResourceConfig() { + ClouderaManagerClientBuilder builder = new ClouderaManagerClientBuilder(); + ApiRootResource a = newProxy(builder); + assertNotNull(a); + ApiRootResource b = newProxy(builder); + assertNotNull(b); + + ClouderaManagerClientBuilder.clearCachedResources(); + // test after clear + assertNotNull(newProxy(builder)); + } + + public ApiRootResource newProxy(ClouderaManagerClientBuilder builder) { + return builder.withHost("localhost") + .withPort(1) + .enableLogging() + .build(); + } } diff --git a/java/src/test/java/com/cloudera/api/model/ApiModelTest.java b/java/src/test/java/com/cloudera/api/model/ApiModelTest.java index 8f50ce06..1457c98c 100644 --- a/java/src/test/java/com/cloudera/api/model/ApiModelTest.java +++ b/java/src/test/java/com/cloudera/api/model/ApiModelTest.java @@ -926,6 +926,7 @@ private ApiHdfsReplicationArguments newHdfsReplicationArguments() { hdfsArgs.setLogPath("log1"); hdfsArgs.setSkipChecksumChecks(true); hdfsArgs.setSkipTrash(true); + hdfsArgs.setPreserveXAttrs(true); hdfsArgs.setReplicationStrategy(ReplicationStrategy.DYNAMIC); return hdfsArgs; } @@ -965,6 +966,7 @@ private ApiCluster newCluster() { cluster.setMaintenanceMode(true); cluster.setName("mycluster"); cluster.setDisplayName("mycluster-displayName"); + cluster.setClusterUrl("http://some-url:7180/cmf/clusterRedirect/mycluster"); cluster.setVersion(ApiClusterVersion.CDH4); cluster.setFullVersion("4.1.2"); return cluster; From 695ee0ae369bcede10440778b2a8ddb67858ad3c Mon Sep 17 00:00:00 2001 From: bms231 Date: Thu, 7 Jan 2016 15:49:35 -0500 Subject: [PATCH 10/10] Updating for V11 compatability Adding clusterRef and entityStatus for V11 API --- python/src/cm_api/endpoints/hosts.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/python/src/cm_api/endpoints/hosts.py b/python/src/cm_api/endpoints/hosts.py index 9d109144..a848fd68 100644 --- a/python/src/cm_api/endpoints/hosts.py +++ b/python/src/cm_api/endpoints/hosts.py @@ -72,8 +72,10 @@ class ApiHost(BaseApiResource): 'status' : ROAttr(), 'lastHeartbeat' : ROAttr(datetime.datetime), 'roleRefs' : ROAttr(ApiRoleRef), + 'clusterRef' : ROAttr(ApiClusterRef), 'healthSummary' : ROAttr(), 'healthChecks' : ROAttr(), + 'entityStatus' : ROAttr(), 'hostUrl' : ROAttr(), 'commissionState' : ROAttr(), 'maintenanceMode' : ROAttr(),