Skip to content

Commit

Permalink
fix(cmx): do not fail on 5xx errors from api
Browse files Browse the repository at this point in the history
  • Loading branch information
@emosbaugh
emosbaugh committed Feb 27, 2024
1 parent 3bed7a6 commit 241b6cf
Show file tree
Hide file tree
Showing 40 changed files with 486 additions and 246 deletions.
60 changes: 42 additions & 18 deletions archive-channel/dist/index.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30432,13 +30432,20 @@ async function findChannelDetailsInOutput(channels, { slug, name }) {
"use strict";

Object.defineProperty(exports, "__esModule", ({ value: true }));
exports.getClusterVersions = exports.upgradeCluster = exports.removeCluster = exports.getKubeconfig = exports.pollForStatus = exports.createCluster = exports.ClusterVersion = exports.Cluster = void 0;
exports.getClusterVersions = exports.upgradeCluster = exports.removeCluster = exports.getKubeconfig = exports.pollForStatus = exports.createCluster = exports.StatusError = exports.ClusterVersion = exports.Cluster = void 0;
class Cluster {
}
exports.Cluster = Cluster;
class ClusterVersion {
}
exports.ClusterVersion = ClusterVersion;
class StatusError extends Error {
constructor(message, statusCode) {
super(message);
this.statusCode = statusCode;
}
}
exports.StatusError = StatusError;
async function createCluster(vendorPortalApi, clusterName, k8sDistribution, k8sVersion, clusterTTL, diskGib, nodeCount, instanceType, nodeGroups, tags) {
const http = await vendorPortalApi.client();
const reqBody = {
Expand Down Expand Up @@ -30472,24 +30479,41 @@ async function createCluster(vendorPortalApi, clusterName, k8sDistribution, k8sV
return { name: body.cluster.name, id: body.cluster.id, status: body.cluster.status };
}
exports.createCluster = createCluster;
async function pollForStatus(vendorPortalApi, clusterId, expectedStatus, timeout = 120) {
async function pollForStatus(vendorPortalApi, clusterId, expectedStatus, timeout = 120, sleeptimeMs = 5000) {
// get clusters from the api, look for the status of the id to be ${status}
// if it's not ${status}, sleep for 5 seconds and try again
// if it is ${status}, return the cluster with that status
const sleeptime = 5;
await new Promise(f => setTimeout(f, sleeptime * 1000)); // sleep for 5 seconds before polling as the cluster takes a few seconds to start provisioning
await new Promise(f => setTimeout(f, sleeptimeMs)); // sleep for 5 seconds before polling as the cluster takes a few seconds to start provisioning
// iterate for timeout/sleeptime times
for (let i = 0; i < timeout / sleeptime; i++) {
const clusterDetails = await getClusterDetails(vendorPortalApi, clusterId);
if (clusterDetails.status === expectedStatus) {
return clusterDetails;
const iterations = timeout * 1000 / sleeptimeMs;
for (let i = 0; i < iterations; i++) {
try {
const clusterDetails = await getClusterDetails(vendorPortalApi, clusterId);
if (clusterDetails.status === expectedStatus) {
return clusterDetails;
}
// Once state is "error", it will never change. So we can shortcut polling.
if (clusterDetails.status === "error") {
throw new Error(`Cluster has entered error state.`);
}
console.debug(`Cluster status is ${clusterDetails.status}, sleeping for ${sleeptimeMs / 1000} seconds`);
}
// Once state is "error", it will never change. So we can shortcut polling.
if (clusterDetails.status === "error") {
throw new Error(`Cluster has entered error state.`);
catch (err) {
if (err instanceof StatusError) {
if (err.statusCode >= 500) {
// 5xx errors are likely transient, so we should retry
console.debug(`Got HTTP error with status ${err.statusCode}, sleeping for ${sleeptimeMs / 1000} seconds`);
}
else {
console.debug(`Got HTTP error with status ${err.statusCode}, exiting`);
throw err;
}
}
else {
throw err;
}
}
console.debug(`Cluster status is ${clusterDetails.status}, sleeping for ${sleeptime} seconds`);
await new Promise(f => setTimeout(f, sleeptime * 1000));
await new Promise(f => setTimeout(f, sleeptimeMs));
}
throw new Error(`Cluster did not reach state ${expectedStatus} within ${timeout} seconds`);
}
Expand All @@ -30499,7 +30523,7 @@ async function getClusterDetails(vendorPortalApi, clusterId) {
const uri = `${vendorPortalApi.endpoint}/cluster/${clusterId}`;
const res = await http.get(uri);
if (res.message.statusCode != 200) {
throw new Error(`Failed to get cluster: Server responded with ${res.message.statusCode}`);
throw new StatusError(`Failed to get cluster: Server responded with ${res.message.statusCode}`, res.message.statusCode);
}
const body = JSON.parse(await res.readBody());
return { name: body.cluster.name, id: body.cluster.id, status: body.cluster.status };
Expand All @@ -30509,7 +30533,7 @@ async function getKubeconfig(vendorPortalApi, clusterId) {
const uri = `${vendorPortalApi.endpoint}/cluster/${clusterId}/kubeconfig`;
const res = await http.get(uri);
if (res.message.statusCode != 200) {
throw new Error(`Failed to get kubeconfig: Server responded with ${res.message.statusCode}`);
throw new StatusError(`Failed to get kubeconfig: Server responded with ${res.message.statusCode}`, res.message.statusCode);
}
const body = JSON.parse(await res.readBody());
return atob(body.kubeconfig);
Expand All @@ -30520,7 +30544,7 @@ async function removeCluster(vendorPortalApi, clusterId) {
const uri = `${vendorPortalApi.endpoint}/cluster/${clusterId}`;
const res = await http.del(uri);
if (res.message.statusCode != 200) {
throw new Error(`Failed to remove cluster: Server responded with ${res.message.statusCode}`);
throw new StatusError(`Failed to remove cluster: Server responded with ${res.message.statusCode}`, res.message.statusCode);
}
}
exports.removeCluster = removeCluster;
Expand All @@ -30532,7 +30556,7 @@ async function upgradeCluster(vendorPortalApi, clusterId, k8sVersion) {
const uri = `${vendorPortalApi.endpoint}/cluster/${clusterId}/upgrade`;
const res = await http.post(uri, JSON.stringify(reqBody));
if (res.message.statusCode != 200) {
throw new Error(`Failed to upgrade cluster: Server responded with ${res.message.statusCode}`);
throw new StatusError(`Failed to upgrade cluster: Server responded with ${res.message.statusCode}`, res.message.statusCode);
}
return getClusterDetails(vendorPortalApi, clusterId);
}
Expand All @@ -30542,7 +30566,7 @@ async function getClusterVersions(vendorPortalApi) {
const uri = `${vendorPortalApi.endpoint}/cluster/versions`;
const res = await http.get(uri);
if (res.message.statusCode != 200) {
throw new Error(`Failed to get cluster versions: Server responded with ${res.message.statusCode}`);
throw new StatusError(`Failed to get cluster versions: Server responded with ${res.message.statusCode}`, res.message.statusCode);
}
const body = JSON.parse(await res.readBody());
// 2. Convert body into ClusterVersion[]
Expand Down
2 changes: 1 addition & 1 deletion archive-channel/dist/index.js.map
View file

Large diffs are not rendered by default.

8 changes: 4 additions & 4 deletions archive-channel/package-lock.json
View file

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

2 changes: 1 addition & 1 deletion archive-channel/package.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@
"dependencies": {
"@actions/core": "^1.10.0",
"esbuild-jest": "^0.5.0",
"replicated-lib": "^0.0.1-beta.8",
"replicated-lib": "^0.0.1-beta.9",
"ts-node": "^10.9.1"
},
"devDependencies": {
Expand Down
60 changes: 42 additions & 18 deletions archive-customer/dist/index.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30431,13 +30431,20 @@ async function findChannelDetailsInOutput(channels, { slug, name }) {
"use strict";

Object.defineProperty(exports, "__esModule", ({ value: true }));
exports.getClusterVersions = exports.upgradeCluster = exports.removeCluster = exports.getKubeconfig = exports.pollForStatus = exports.createCluster = exports.ClusterVersion = exports.Cluster = void 0;
exports.getClusterVersions = exports.upgradeCluster = exports.removeCluster = exports.getKubeconfig = exports.pollForStatus = exports.createCluster = exports.StatusError = exports.ClusterVersion = exports.Cluster = void 0;
class Cluster {
}
exports.Cluster = Cluster;
class ClusterVersion {
}
exports.ClusterVersion = ClusterVersion;
class StatusError extends Error {
constructor(message, statusCode) {
super(message);
this.statusCode = statusCode;
}
}
exports.StatusError = StatusError;
async function createCluster(vendorPortalApi, clusterName, k8sDistribution, k8sVersion, clusterTTL, diskGib, nodeCount, instanceType, nodeGroups, tags) {
const http = await vendorPortalApi.client();
const reqBody = {
Expand Down Expand Up @@ -30471,24 +30478,41 @@ async function createCluster(vendorPortalApi, clusterName, k8sDistribution, k8sV
return { name: body.cluster.name, id: body.cluster.id, status: body.cluster.status };
}
exports.createCluster = createCluster;
async function pollForStatus(vendorPortalApi, clusterId, expectedStatus, timeout = 120) {
async function pollForStatus(vendorPortalApi, clusterId, expectedStatus, timeout = 120, sleeptimeMs = 5000) {
// get clusters from the api, look for the status of the id to be ${status}
// if it's not ${status}, sleep for 5 seconds and try again
// if it is ${status}, return the cluster with that status
const sleeptime = 5;
await new Promise(f => setTimeout(f, sleeptime * 1000)); // sleep for 5 seconds before polling as the cluster takes a few seconds to start provisioning
await new Promise(f => setTimeout(f, sleeptimeMs)); // sleep for 5 seconds before polling as the cluster takes a few seconds to start provisioning
// iterate for timeout/sleeptime times
for (let i = 0; i < timeout / sleeptime; i++) {
const clusterDetails = await getClusterDetails(vendorPortalApi, clusterId);
if (clusterDetails.status === expectedStatus) {
return clusterDetails;
const iterations = timeout * 1000 / sleeptimeMs;
for (let i = 0; i < iterations; i++) {
try {
const clusterDetails = await getClusterDetails(vendorPortalApi, clusterId);
if (clusterDetails.status === expectedStatus) {
return clusterDetails;
}
// Once state is "error", it will never change. So we can shortcut polling.
if (clusterDetails.status === "error") {
throw new Error(`Cluster has entered error state.`);
}
console.debug(`Cluster status is ${clusterDetails.status}, sleeping for ${sleeptimeMs / 1000} seconds`);
}
// Once state is "error", it will never change. So we can shortcut polling.
if (clusterDetails.status === "error") {
throw new Error(`Cluster has entered error state.`);
catch (err) {
if (err instanceof StatusError) {
if (err.statusCode >= 500) {
// 5xx errors are likely transient, so we should retry
console.debug(`Got HTTP error with status ${err.statusCode}, sleeping for ${sleeptimeMs / 1000} seconds`);
}
else {
console.debug(`Got HTTP error with status ${err.statusCode}, exiting`);
throw err;
}
}
else {
throw err;
}
}
console.debug(`Cluster status is ${clusterDetails.status}, sleeping for ${sleeptime} seconds`);
await new Promise(f => setTimeout(f, sleeptime * 1000));
await new Promise(f => setTimeout(f, sleeptimeMs));
}
throw new Error(`Cluster did not reach state ${expectedStatus} within ${timeout} seconds`);
}
Expand All @@ -30498,7 +30522,7 @@ async function getClusterDetails(vendorPortalApi, clusterId) {
const uri = `${vendorPortalApi.endpoint}/cluster/${clusterId}`;
const res = await http.get(uri);
if (res.message.statusCode != 200) {
throw new Error(`Failed to get cluster: Server responded with ${res.message.statusCode}`);
throw new StatusError(`Failed to get cluster: Server responded with ${res.message.statusCode}`, res.message.statusCode);
}
const body = JSON.parse(await res.readBody());
return { name: body.cluster.name, id: body.cluster.id, status: body.cluster.status };
Expand All @@ -30508,7 +30532,7 @@ async function getKubeconfig(vendorPortalApi, clusterId) {
const uri = `${vendorPortalApi.endpoint}/cluster/${clusterId}/kubeconfig`;
const res = await http.get(uri);
if (res.message.statusCode != 200) {
throw new Error(`Failed to get kubeconfig: Server responded with ${res.message.statusCode}`);
throw new StatusError(`Failed to get kubeconfig: Server responded with ${res.message.statusCode}`, res.message.statusCode);
}
const body = JSON.parse(await res.readBody());
return atob(body.kubeconfig);
Expand All @@ -30519,7 +30543,7 @@ async function removeCluster(vendorPortalApi, clusterId) {
const uri = `${vendorPortalApi.endpoint}/cluster/${clusterId}`;
const res = await http.del(uri);
if (res.message.statusCode != 200) {
throw new Error(`Failed to remove cluster: Server responded with ${res.message.statusCode}`);
throw new StatusError(`Failed to remove cluster: Server responded with ${res.message.statusCode}`, res.message.statusCode);
}
}
exports.removeCluster = removeCluster;
Expand All @@ -30531,7 +30555,7 @@ async function upgradeCluster(vendorPortalApi, clusterId, k8sVersion) {
const uri = `${vendorPortalApi.endpoint}/cluster/${clusterId}/upgrade`;
const res = await http.post(uri, JSON.stringify(reqBody));
if (res.message.statusCode != 200) {
throw new Error(`Failed to upgrade cluster: Server responded with ${res.message.statusCode}`);
throw new StatusError(`Failed to upgrade cluster: Server responded with ${res.message.statusCode}`, res.message.statusCode);
}
return getClusterDetails(vendorPortalApi, clusterId);
}
Expand All @@ -30541,7 +30565,7 @@ async function getClusterVersions(vendorPortalApi) {
const uri = `${vendorPortalApi.endpoint}/cluster/versions`;
const res = await http.get(uri);
if (res.message.statusCode != 200) {
throw new Error(`Failed to get cluster versions: Server responded with ${res.message.statusCode}`);
throw new StatusError(`Failed to get cluster versions: Server responded with ${res.message.statusCode}`, res.message.statusCode);
}
const body = JSON.parse(await res.readBody());
// 2. Convert body into ClusterVersion[]
Expand Down
2 changes: 1 addition & 1 deletion archive-customer/dist/index.js.map
View file

Large diffs are not rendered by default.

8 changes: 4 additions & 4 deletions archive-customer/package-lock.json
View file

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

2 changes: 1 addition & 1 deletion archive-customer/package.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@
"dependencies": {
"@actions/core": "^1.10.0",
"esbuild-jest": "^0.5.0",
"replicated-lib": "^0.0.1-beta.8",
"replicated-lib": "^0.0.1-beta.9",
"ts-node": "^10.9.1"
},
"devDependencies": {
Expand Down
Loading

0 comments on commit 241b6cf

Please sign in to comment.