From 3935ca68cb0927bf150e15c3b205baebafa5cb98 Mon Sep 17 00:00:00 2001 From: Mathieu LAUDE Date: Tue, 10 Dec 2024 15:25:49 +0100 Subject: [PATCH 1/5] feat: :sparkles: store cluster secret in Vault --- apps/server/src/resources/cluster/queries.ts | 10 +++ apps/server/src/utils/hook-wrapper.ts | 7 +- packages/hooks/src/hooks/index.ts | 1 + plugins/argocd/src/cluster.ts | 9 ++ plugins/argocd/src/functions.ts | 2 +- plugins/argocd/src/index.ts | 6 ++ plugins/argocd/src/zone.ts | 29 +++++++ plugins/gitlab/src/index.ts | 2 +- plugins/vault/src/class.ts | 91 +++++++++++++------- plugins/vault/src/functions.ts | 2 +- plugins/vault/src/index.ts | 10 ++- 11 files changed, 131 insertions(+), 38 deletions(-) create mode 100644 plugins/argocd/src/zone.ts diff --git a/apps/server/src/resources/cluster/queries.ts b/apps/server/src/resources/cluster/queries.ts index 756ff994a..dbd6aef20 100644 --- a/apps/server/src/resources/cluster/queries.ts +++ b/apps/server/src/resources/cluster/queries.ts @@ -133,6 +133,16 @@ export function getPublicClusters() { }) } +export async function getClusterNamesByZoneId(zoneId: string) { + const clusterNames = await prisma.cluster.findMany({ + where: { zoneId }, + select: { + label: true, + }, + }) + return clusterNames.map(({ label }) => label) +} + export function getClusterByLabel(label: Cluster['label']) { return prisma.cluster.findUnique({ where: { label } }) } diff --git a/apps/server/src/utils/hook-wrapper.ts b/apps/server/src/utils/hook-wrapper.ts index 1636bf1c5..aee95e881 100644 --- a/apps/server/src/utils/hook-wrapper.ts +++ b/apps/server/src/utils/hook-wrapper.ts @@ -1,10 +1,10 @@ import type { Cluster, Kubeconfig, Project, ProjectRole, Zone } from '@prisma/client' -import type { ClusterObject, HookResult, KubeCluster, KubeUser, Project as ProjectPayload, RepoCreds, Repository, Store } from '@cpn-console/hooks' +import type { ClusterObject, HookResult, KubeCluster, KubeUser, Project as ProjectPayload, RepoCreds, Repository, Store, ZoneObject } from '@cpn-console/hooks' import { hooks } from '@cpn-console/hooks' import type { AsyncReturnType } from '@cpn-console/shared' import { ProjectAuthorized, getPermsByUserRoles, resourceListToDict } from '@cpn-console/shared' import { genericProxy } from './proxy.js' -import { archiveProject, getAdminPlugin, getClusterByIdOrThrow, getClustersAssociatedWithProject, getHookProjectInfos, getHookRepository, getProjectStore, getZoneByIdOrThrow, saveProjectStore, updateProjectClusterHistory, updateProjectCreated, updateProjectFailed, updateProjectWarning } from '@/resources/queries-index.js' +import { archiveProject, getAdminPlugin, getClusterByIdOrThrow, getClusterNamesByZoneId, getClustersAssociatedWithProject, getHookProjectInfos, getHookRepository, getProjectStore, getZoneByIdOrThrow, saveProjectStore, updateProjectClusterHistory, updateProjectCreated, updateProjectFailed, updateProjectWarning } from '@/resources/queries-index.js' import type { ConfigRecords } from '@/resources/project-service/business.js' import { dbToObj } from '@/resources/project-service/business.js' @@ -121,7 +121,8 @@ const user = { const zone = { upsert: async (zoneId: Zone['id']) => { - const zone = await getZoneByIdOrThrow(zoneId) + const zone: ZoneObject = await getZoneByIdOrThrow(zoneId) + zone.clusterNames = await getClusterNamesByZoneId(zoneId) const store = dbToObj(await getAdminPlugin()) return hooks.upsertZone.execute(zone, store) }, diff --git a/packages/hooks/src/hooks/index.ts b/packages/hooks/src/hooks/index.ts index 41fe06ff4..c6a253e10 100644 --- a/packages/hooks/src/hooks/index.ts +++ b/packages/hooks/src/hooks/index.ts @@ -46,6 +46,7 @@ export interface ZoneObject { id: string slug: string argocdUrl: string + clusterNames?: string[] } export interface ClusterObject { diff --git a/plugins/argocd/src/cluster.ts b/plugins/argocd/src/cluster.ts index da2f336d9..e0ab62050 100644 --- a/plugins/argocd/src/cluster.ts +++ b/plugins/argocd/src/cluster.ts @@ -6,11 +6,19 @@ import { getConfig, getK8sApi } from './utils.js' export const upsertCluster: StepCall = async (payload) => { try { const cluster = payload.args + const { vault } = payload.apis if (cluster.label === inClusterLabel) { await deleteClusterSecret(cluster.secretName) } else { await createClusterSecret(cluster) } + const clusterData = { + name: cluster.label, + clusterResources: cluster.clusterResources.toString(), + server: cluster.cluster.server, + config: JSON.stringify(convertConfig(cluster)), + } + await vault.write(clusterData, `clusters/cluster-${cluster.label}/argocd-cluster-secret`) return { status: { result: 'OK', @@ -32,6 +40,7 @@ export const deleteCluster: StepCall = async (payload) => { try { const secretName = payload.args.secretName await deleteClusterSecret(secretName) + await payload.apis.vault.destroy(`clusters/cluster-${payload.args.label}/argocd-cluster-secret`) return { status: { result: 'OK', diff --git a/plugins/argocd/src/functions.ts b/plugins/argocd/src/functions.ts index 17c564638..35682ed2f 100644 --- a/plugins/argocd/src/functions.ts +++ b/plugins/argocd/src/functions.ts @@ -220,7 +220,7 @@ async function ensureInfraEnvValues(project: Project, environment: Environment, const cluster = getCluster(project, environment) const infraProject = await gitlabApi.getProjectById(repoId) const valueFilePath = getValueFilePath(project, cluster, environment) - const vaultCredentials = await vaultApi.getCredentials() + const vaultCredentials = await vaultApi.Project.getCredentials() const repositories: ArgoRepoSource[] = await Promise.all([ getArgoRepoSource('infra-apps', environment.name, gitlabApi), ...sourceRepos.map(repo => getArgoRepoSource(repo.internalRepoName, environment.name, gitlabApi)), diff --git a/plugins/argocd/src/index.ts b/plugins/argocd/src/index.ts index 09840987b..1def78a5d 100644 --- a/plugins/argocd/src/index.ts +++ b/plugins/argocd/src/index.ts @@ -4,6 +4,7 @@ import monitor from './monitor.js' import { deleteProject, upsertProject } from './functions.js' import { fixLabels } from './label-fix.js' import { deleteCluster, upsertCluster } from './cluster.js' +import { upsertZone } from './zone.js' export const plugin: Plugin = { infos, @@ -28,6 +29,11 @@ export const plugin: Plugin = { main: deleteCluster, }, }, + upsertZone: { + steps: { + main: upsertZone, + }, + }, }, monitor, start: fixLabels, diff --git a/plugins/argocd/src/zone.ts b/plugins/argocd/src/zone.ts new file mode 100644 index 000000000..4329f1352 --- /dev/null +++ b/plugins/argocd/src/zone.ts @@ -0,0 +1,29 @@ +import { type StepCall, type ZoneObject, parseError } from '@cpn-console/hooks' +import { dump } from 'js-yaml' + +export const upsertZone: StepCall = async (payload) => { + try { + const zone = payload.args + const { gitlab, vault } = payload.apis + const values = { + vault: await vault.getCredentials(), + clusters: zone.clusterNames, + } + const zoneRepo = await gitlab.getOrCreateInfraProject(zone.slug) + await gitlab.commitCreateOrUpdate(zoneRepo.id, dump(values), 'argocd-values.yaml') + return { + status: { + result: 'OK', + message: 'Zone argocd configuration created/updated', + }, + } + } catch (error) { + return { + error: parseError(error), + status: { + result: 'KO', + message: 'Failed create/update zone argocd configuration', + }, + } + } +} diff --git a/plugins/gitlab/src/index.ts b/plugins/gitlab/src/index.ts index 7dc8d0d6f..11b685de9 100644 --- a/plugins/gitlab/src/index.ts +++ b/plugins/gitlab/src/index.ts @@ -52,7 +52,7 @@ export const plugin: Plugin = { upsertZone: { api: () => new GitlabZoneApi(), steps: { - main: upsertZone, + pre: upsertZone, post: commitFiles, }, }, diff --git a/plugins/vault/src/class.ts b/plugins/vault/src/class.ts index d16a9e380..687736b33 100644 --- a/plugins/vault/src/class.ts +++ b/plugins/vault/src/class.ts @@ -37,6 +37,16 @@ export class VaultApi extends PluginApi { return this.token } + protected async destroy(path: string, kvName: string) { + if (path.startsWith('/')) + path = path.slice(1) + return await this.axios({ + method: 'delete', + url: `/v1/${kvName}/metadata/${path}`, + headers: { 'X-Vault-Token': await this.getToken() }, + }) + } + Kv = { upsert: async (kvName: string) => { const token = await this.getToken() @@ -132,7 +142,6 @@ export class VaultApi extends PluginApi { headers: { 'X-Vault-Token': await this.getToken() }, }) }, - delete: async (roleName: string) => { await this.axios.delete( `/v1/auth/approle/role/${roleName}`, @@ -141,6 +150,25 @@ export class VaultApi extends PluginApi { }, ) }, + getCredentials: async (roleName: string) => { + const { data: dataRole } = await this.axios.get( + `/v1/auth/approle/role/${roleName}/role-id`, + { + headers: { 'X-Vault-Token': await this.getToken() }, + }, + ) + const { data: dataSecret } = await this.axios.put( + `/v1/auth/approle/role/${roleName}/secret-id`, + 'null', // Force empty data + { + headers: { 'X-Vault-Token': await this.getToken() }, + }, + ) + return { + roleId: dataRole.data?.role_id, + secretId: dataSecret.data?.secret_id, + } + }, } } @@ -233,35 +261,7 @@ export class VaultProjectApi extends VaultApi { public async destroy(path: string = '/') { if (path.startsWith('/')) path = path.slice(1) - return this.axios({ - method: 'delete', - url: `/v1/${this.coreKvName}/metadata/${this.projectRootDir}/${this.basePath}/${path}`, - headers: { 'X-Vault-Token': await this.getToken() }, - - }) - } - - public async getCredentials(): Promise { - const appRoleEnabled = await isAppRoleEnabled(this.axios, await this.getToken()) - if (!appRoleEnabled) return this.defaultAppRoleCredentials - const { data: dataRole } = await this.axios.get( - `/v1/auth/approle/role/${this.roleName}/role-id`, - { - headers: { 'X-Vault-Token': await this.getToken() }, - }, - ) - const { data: dataSecret } = await this.axios.put( - `/v1/auth/approle/role/${this.roleName}/secret-id`, - 'null', // Force empty data - { - headers: { 'X-Vault-Token': await this.getToken() }, - }, - ) - return { - ...this.defaultAppRoleCredentials, - roleId: dataRole.data?.role_id, - secretId: dataSecret.data?.secret_id, - } + return super.destroy(`${this.projectRootDir}/${this.basePath}/${path}`, this.coreKvName) } Project = { @@ -285,6 +285,15 @@ export class VaultProjectApi extends VaultApi { await this.Group.delete() await this.Role.delete(this.roleName) }, + getCredentials: async () => { + const appRoleEnabled = await isAppRoleEnabled(this.axios, await this.getToken()) + if (!appRoleEnabled) return this.defaultAppRoleCredentials + const creds = await this.Role.getCredentials(this.roleName) + return { + ...this.defaultAppRoleCredentials, + ...creds, + } + }, } Group = { @@ -370,4 +379,26 @@ export class VaultZoneApi extends VaultApi { ) return await response.data } + + public async destroy(path: string = '/') { + if (path.startsWith('/')) + path = path.slice(1) + return super.destroy(path, this.kvName) + } + + public async getCredentials() { + const appRoleEnabled = await isAppRoleEnabled(this.axios, await this.getToken()) + if (!appRoleEnabled) { + return { + url: getConfig().publicUrl, + kvName: this.kvName, + } + } + const creds = await this.Role.getCredentials(this.roleName) + return { + url: getConfig().publicUrl, + kvName: this.kvName, + ...creds, + } + } } diff --git a/plugins/vault/src/functions.ts b/plugins/vault/src/functions.ts index 0c0bfcb04..8c8a02084 100644 --- a/plugins/vault/src/functions.ts +++ b/plugins/vault/src/functions.ts @@ -30,7 +30,7 @@ export const deployAuth: StepCall = async (payload) => { if (!payload.apis.vault) { throw new Error('no Vault available') } - const appRoleCreds = await payload.apis.vault.getCredentials() + const appRoleCreds = await payload.apis.vault.Project.getCredentials() if (getConfig().disableVaultSecrets) { return okStatus } diff --git a/plugins/vault/src/index.ts b/plugins/vault/src/index.ts index 63caca760..017ecf14d 100644 --- a/plugins/vault/src/index.ts +++ b/plugins/vault/src/index.ts @@ -1,4 +1,4 @@ -import type { DefaultArgs, Plugin, Project, ProjectLite, ZoneObject } from '@cpn-console/hooks' +import type { ClusterObject, DefaultArgs, Plugin, Project, ProjectLite, ZoneObject } from '@cpn-console/hooks' import { archiveDsoProject, deleteZone, deployAuth, getSecrets, upsertProject, upsertZone } from './functions.js' import infos from './infos.js' import monitor from './monitor.js' @@ -38,6 +38,12 @@ export const plugin: Plugin = { main: deleteZone, }, }, + upsertCluster: { + api: (c: ClusterObject) => new VaultZoneApi(c.zone.slug), + }, + deleteCluster: { + api: (c: ClusterObject) => new VaultZoneApi(c.zone.slug), + }, }, monitor, } @@ -46,7 +52,7 @@ declare module '@cpn-console/hooks' { interface HookPayloadApis { vault: Args extends (ProjectLite | Project) ? VaultProjectApi - : Args extends (ZoneObject) + : Args extends (ZoneObject | ClusterObject) ? VaultZoneApi : never } From 1295ece0312a6c7b319c379e88be1ee570d75499 Mon Sep 17 00:00:00 2001 From: ArnaudTa <33383276+ArnaudTA@users.noreply.github.com> Date: Wed, 8 Jan 2025 11:43:26 +0100 Subject: [PATCH 2/5] fix: :bug: nexus auto disable repo config on delete --- plugins/nexus/src/project.ts | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/plugins/nexus/src/project.ts b/plugins/nexus/src/project.ts index 977434834..b9b7d7ea0 100644 --- a/plugins/nexus/src/project.ts +++ b/plugins/nexus/src/project.ts @@ -31,6 +31,10 @@ export const deleteNexusProject: StepCall = async ({ args: project }) = result: 'OK', message: 'Project deleted from Nexus', }, + store: { + activateNpmRepo: 'disabled', + activateMavenRepo: 'disabled', + }, } } catch (error) { return { From 90faeff2978e0ee2db7624b895550db278aa9c17 Mon Sep 17 00:00:00 2001 From: this-is-tobi Date: Mon, 16 Dec 2024 17:11:25 +0100 Subject: [PATCH 3/5] fix: :bug: downgrade gitlab lib to avoid upsert user warning --- apps/server/package.json | 4 ++-- plugins/gitlab/package.json | 6 ++--- pnpm-lock.yaml | 44 ++++++++++++++++++------------------- 3 files changed, 27 insertions(+), 27 deletions(-) diff --git a/apps/server/package.json b/apps/server/package.json index 39c914d08..3f0b9deb5 100644 --- a/apps/server/package.json +++ b/apps/server/package.json @@ -48,8 +48,8 @@ "@fastify/session": "^10.9.0", "@fastify/swagger": "^8.15.0", "@fastify/swagger-ui": "^4.2.0", - "@gitbeaker/core": "^41.3.0", - "@gitbeaker/rest": "^41.3.0", + "@gitbeaker/core": "~40.6.0", + "@gitbeaker/rest": "~40.6.0", "@kubernetes-models/argo-cd": "^2.6.2", "@kubernetes/client-node": "^0.22.3", "@prisma/client": "^6.0.1", diff --git a/plugins/gitlab/package.json b/plugins/gitlab/package.json index e85e404ce..256f3bc9a 100644 --- a/plugins/gitlab/package.json +++ b/plugins/gitlab/package.json @@ -23,9 +23,9 @@ "@cpn-console/hooks": "workspace:^", "@cpn-console/shared": "workspace:^", "@cpn-console/vault-plugin": "workspace:^", - "@gitbeaker/core": "^41.3.0", - "@gitbeaker/requester-utils": "^41.3.0", - "@gitbeaker/rest": "^41.3.0", + "@gitbeaker/core": "~40.6.0", + "@gitbeaker/requester-utils": "~40.6.0", + "@gitbeaker/rest": "~40.6.0", "axios": "^1.7.9", "js-yaml": "4.1.0" }, diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index be662ed1f..49d26d29a 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -253,11 +253,11 @@ importers: specifier: ^4.2.0 version: 4.2.0 '@gitbeaker/core': - specifier: ^41.3.0 - version: 41.3.0 + specifier: ~40.6.0 + version: 40.6.0 '@gitbeaker/rest': - specifier: ^41.3.0 - version: 41.3.0 + specifier: ~40.6.0 + version: 40.6.0 '@kubernetes-models/argo-cd': specifier: ^2.6.2 version: 2.6.2 @@ -578,14 +578,14 @@ importers: specifier: workspace:^ version: link:../vault '@gitbeaker/core': - specifier: ^41.3.0 - version: 41.3.0 + specifier: ~40.6.0 + version: 40.6.0 '@gitbeaker/requester-utils': - specifier: ^41.3.0 - version: 41.3.0 + specifier: ~40.6.0 + version: 40.6.0 '@gitbeaker/rest': - specifier: ^41.3.0 - version: 41.3.0 + specifier: ~40.6.0 + version: 40.6.0 axios: specifier: ^1.7.9 version: 1.7.9 @@ -2312,16 +2312,16 @@ packages: '@fastify/swagger@8.15.0': resolution: {integrity: sha512-zy+HEEKFqPMS2sFUsQU5X0MHplhKJvWeohBwTCkBAJA/GDYGLGUWQaETEhptiqxK7Hs0fQB9B4MDb3pbwIiCwA==} - '@gitbeaker/core@41.3.0': - resolution: {integrity: sha512-ZPy0v71WTSKdELLhG5FkMCxIWCJwRpA7QkvnULiE36sIynQE0WwBNux+GPjjEh6xQ6PfBAB4E/1Uu2YZXiJlNg==} + '@gitbeaker/core@40.6.0': + resolution: {integrity: sha512-tVVm8ZPrS9YCHEcuPV8vD1IcEf9POpdygWo+kPvkK7LcC36EERVcXagb8snEaGgGLfUaVQh8qP4iDZgPnP3YBQ==} engines: {node: '>=18.20.0'} - '@gitbeaker/requester-utils@41.3.0': - resolution: {integrity: sha512-sNVlp32uaieQ+Giovlu1GJ8hY9jMhY//f3WMHct2GV0U74PkSsixQgQv9XuRKgZalk2uI12iJrBY7gnAA5N8/w==} + '@gitbeaker/requester-utils@40.6.0': + resolution: {integrity: sha512-DQu2l3iXtB+8e1Ye2ekeUHABt4mGMRTLtuVWtFqf74sqJnerHNOxVOjPn19qu/nKdvKR3ZLwSRTtPzEsxgcShg==} engines: {node: '>=18.20.0'} - '@gitbeaker/rest@41.3.0': - resolution: {integrity: sha512-l1jEloxQ4/SCdOjv9ryfbZEvvF90GFllLxULbHiLPI72rAHCjxWtjAdeQXddkvfw/LYCw7CFyu+M9Okt/IjsgA==} + '@gitbeaker/rest@40.6.0': + resolution: {integrity: sha512-sAwYJclU3NlB/gdxqhH6Hnmy5LWzvW7D3W33eShQEnxMhM0VjnFHPHcgJLQCIux3hMiub1uGtTw1hBJTxDc2mQ==} engines: {node: '>=18.20.0'} '@gouvfr/dsfr@1.12.1': @@ -9595,23 +9595,23 @@ snapshots: transitivePeerDependencies: - supports-color - '@gitbeaker/core@41.3.0': + '@gitbeaker/core@40.6.0': dependencies: - '@gitbeaker/requester-utils': 41.3.0 + '@gitbeaker/requester-utils': 40.6.0 qs: 6.13.1 xcase: 2.0.1 - '@gitbeaker/requester-utils@41.3.0': + '@gitbeaker/requester-utils@40.6.0': dependencies: picomatch-browser: 2.2.6 qs: 6.13.1 rate-limiter-flexible: 4.0.1 xcase: 2.0.1 - '@gitbeaker/rest@41.3.0': + '@gitbeaker/rest@40.6.0': dependencies: - '@gitbeaker/core': 41.3.0 - '@gitbeaker/requester-utils': 41.3.0 + '@gitbeaker/core': 40.6.0 + '@gitbeaker/requester-utils': 40.6.0 '@gouvfr/dsfr@1.12.1(browser-sync@2.29.3)': dependencies: From 23d9d16f225e7defcd6da8b7bb797cf635d8d3a2 Mon Sep 17 00:00:00 2001 From: ArnaudTa <33383276+ArnaudTA@users.noreply.github.com> Date: Wed, 8 Jan 2025 17:00:48 +0100 Subject: [PATCH 4/5] Revert "feat: :sparkles: store cluster secret in Vault" This reverts commit 3935ca68cb0927bf150e15c3b205baebafa5cb98. --- apps/server/src/resources/cluster/queries.ts | 10 --- apps/server/src/utils/hook-wrapper.ts | 7 +- packages/hooks/src/hooks/index.ts | 1 - plugins/argocd/src/cluster.ts | 9 -- plugins/argocd/src/functions.ts | 2 +- plugins/argocd/src/index.ts | 6 -- plugins/argocd/src/zone.ts | 29 ------- plugins/gitlab/src/index.ts | 2 +- plugins/vault/src/class.ts | 91 +++++++------------- plugins/vault/src/functions.ts | 2 +- plugins/vault/src/index.ts | 10 +-- 11 files changed, 38 insertions(+), 131 deletions(-) delete mode 100644 plugins/argocd/src/zone.ts diff --git a/apps/server/src/resources/cluster/queries.ts b/apps/server/src/resources/cluster/queries.ts index dbd6aef20..756ff994a 100644 --- a/apps/server/src/resources/cluster/queries.ts +++ b/apps/server/src/resources/cluster/queries.ts @@ -133,16 +133,6 @@ export function getPublicClusters() { }) } -export async function getClusterNamesByZoneId(zoneId: string) { - const clusterNames = await prisma.cluster.findMany({ - where: { zoneId }, - select: { - label: true, - }, - }) - return clusterNames.map(({ label }) => label) -} - export function getClusterByLabel(label: Cluster['label']) { return prisma.cluster.findUnique({ where: { label } }) } diff --git a/apps/server/src/utils/hook-wrapper.ts b/apps/server/src/utils/hook-wrapper.ts index aee95e881..1636bf1c5 100644 --- a/apps/server/src/utils/hook-wrapper.ts +++ b/apps/server/src/utils/hook-wrapper.ts @@ -1,10 +1,10 @@ import type { Cluster, Kubeconfig, Project, ProjectRole, Zone } from '@prisma/client' -import type { ClusterObject, HookResult, KubeCluster, KubeUser, Project as ProjectPayload, RepoCreds, Repository, Store, ZoneObject } from '@cpn-console/hooks' +import type { ClusterObject, HookResult, KubeCluster, KubeUser, Project as ProjectPayload, RepoCreds, Repository, Store } from '@cpn-console/hooks' import { hooks } from '@cpn-console/hooks' import type { AsyncReturnType } from '@cpn-console/shared' import { ProjectAuthorized, getPermsByUserRoles, resourceListToDict } from '@cpn-console/shared' import { genericProxy } from './proxy.js' -import { archiveProject, getAdminPlugin, getClusterByIdOrThrow, getClusterNamesByZoneId, getClustersAssociatedWithProject, getHookProjectInfos, getHookRepository, getProjectStore, getZoneByIdOrThrow, saveProjectStore, updateProjectClusterHistory, updateProjectCreated, updateProjectFailed, updateProjectWarning } from '@/resources/queries-index.js' +import { archiveProject, getAdminPlugin, getClusterByIdOrThrow, getClustersAssociatedWithProject, getHookProjectInfos, getHookRepository, getProjectStore, getZoneByIdOrThrow, saveProjectStore, updateProjectClusterHistory, updateProjectCreated, updateProjectFailed, updateProjectWarning } from '@/resources/queries-index.js' import type { ConfigRecords } from '@/resources/project-service/business.js' import { dbToObj } from '@/resources/project-service/business.js' @@ -121,8 +121,7 @@ const user = { const zone = { upsert: async (zoneId: Zone['id']) => { - const zone: ZoneObject = await getZoneByIdOrThrow(zoneId) - zone.clusterNames = await getClusterNamesByZoneId(zoneId) + const zone = await getZoneByIdOrThrow(zoneId) const store = dbToObj(await getAdminPlugin()) return hooks.upsertZone.execute(zone, store) }, diff --git a/packages/hooks/src/hooks/index.ts b/packages/hooks/src/hooks/index.ts index c6a253e10..41fe06ff4 100644 --- a/packages/hooks/src/hooks/index.ts +++ b/packages/hooks/src/hooks/index.ts @@ -46,7 +46,6 @@ export interface ZoneObject { id: string slug: string argocdUrl: string - clusterNames?: string[] } export interface ClusterObject { diff --git a/plugins/argocd/src/cluster.ts b/plugins/argocd/src/cluster.ts index e0ab62050..da2f336d9 100644 --- a/plugins/argocd/src/cluster.ts +++ b/plugins/argocd/src/cluster.ts @@ -6,19 +6,11 @@ import { getConfig, getK8sApi } from './utils.js' export const upsertCluster: StepCall = async (payload) => { try { const cluster = payload.args - const { vault } = payload.apis if (cluster.label === inClusterLabel) { await deleteClusterSecret(cluster.secretName) } else { await createClusterSecret(cluster) } - const clusterData = { - name: cluster.label, - clusterResources: cluster.clusterResources.toString(), - server: cluster.cluster.server, - config: JSON.stringify(convertConfig(cluster)), - } - await vault.write(clusterData, `clusters/cluster-${cluster.label}/argocd-cluster-secret`) return { status: { result: 'OK', @@ -40,7 +32,6 @@ export const deleteCluster: StepCall = async (payload) => { try { const secretName = payload.args.secretName await deleteClusterSecret(secretName) - await payload.apis.vault.destroy(`clusters/cluster-${payload.args.label}/argocd-cluster-secret`) return { status: { result: 'OK', diff --git a/plugins/argocd/src/functions.ts b/plugins/argocd/src/functions.ts index 35682ed2f..17c564638 100644 --- a/plugins/argocd/src/functions.ts +++ b/plugins/argocd/src/functions.ts @@ -220,7 +220,7 @@ async function ensureInfraEnvValues(project: Project, environment: Environment, const cluster = getCluster(project, environment) const infraProject = await gitlabApi.getProjectById(repoId) const valueFilePath = getValueFilePath(project, cluster, environment) - const vaultCredentials = await vaultApi.Project.getCredentials() + const vaultCredentials = await vaultApi.getCredentials() const repositories: ArgoRepoSource[] = await Promise.all([ getArgoRepoSource('infra-apps', environment.name, gitlabApi), ...sourceRepos.map(repo => getArgoRepoSource(repo.internalRepoName, environment.name, gitlabApi)), diff --git a/plugins/argocd/src/index.ts b/plugins/argocd/src/index.ts index 1def78a5d..09840987b 100644 --- a/plugins/argocd/src/index.ts +++ b/plugins/argocd/src/index.ts @@ -4,7 +4,6 @@ import monitor from './monitor.js' import { deleteProject, upsertProject } from './functions.js' import { fixLabels } from './label-fix.js' import { deleteCluster, upsertCluster } from './cluster.js' -import { upsertZone } from './zone.js' export const plugin: Plugin = { infos, @@ -29,11 +28,6 @@ export const plugin: Plugin = { main: deleteCluster, }, }, - upsertZone: { - steps: { - main: upsertZone, - }, - }, }, monitor, start: fixLabels, diff --git a/plugins/argocd/src/zone.ts b/plugins/argocd/src/zone.ts deleted file mode 100644 index 4329f1352..000000000 --- a/plugins/argocd/src/zone.ts +++ /dev/null @@ -1,29 +0,0 @@ -import { type StepCall, type ZoneObject, parseError } from '@cpn-console/hooks' -import { dump } from 'js-yaml' - -export const upsertZone: StepCall = async (payload) => { - try { - const zone = payload.args - const { gitlab, vault } = payload.apis - const values = { - vault: await vault.getCredentials(), - clusters: zone.clusterNames, - } - const zoneRepo = await gitlab.getOrCreateInfraProject(zone.slug) - await gitlab.commitCreateOrUpdate(zoneRepo.id, dump(values), 'argocd-values.yaml') - return { - status: { - result: 'OK', - message: 'Zone argocd configuration created/updated', - }, - } - } catch (error) { - return { - error: parseError(error), - status: { - result: 'KO', - message: 'Failed create/update zone argocd configuration', - }, - } - } -} diff --git a/plugins/gitlab/src/index.ts b/plugins/gitlab/src/index.ts index 11b685de9..7dc8d0d6f 100644 --- a/plugins/gitlab/src/index.ts +++ b/plugins/gitlab/src/index.ts @@ -52,7 +52,7 @@ export const plugin: Plugin = { upsertZone: { api: () => new GitlabZoneApi(), steps: { - pre: upsertZone, + main: upsertZone, post: commitFiles, }, }, diff --git a/plugins/vault/src/class.ts b/plugins/vault/src/class.ts index 687736b33..d16a9e380 100644 --- a/plugins/vault/src/class.ts +++ b/plugins/vault/src/class.ts @@ -37,16 +37,6 @@ export class VaultApi extends PluginApi { return this.token } - protected async destroy(path: string, kvName: string) { - if (path.startsWith('/')) - path = path.slice(1) - return await this.axios({ - method: 'delete', - url: `/v1/${kvName}/metadata/${path}`, - headers: { 'X-Vault-Token': await this.getToken() }, - }) - } - Kv = { upsert: async (kvName: string) => { const token = await this.getToken() @@ -142,6 +132,7 @@ export class VaultApi extends PluginApi { headers: { 'X-Vault-Token': await this.getToken() }, }) }, + delete: async (roleName: string) => { await this.axios.delete( `/v1/auth/approle/role/${roleName}`, @@ -150,25 +141,6 @@ export class VaultApi extends PluginApi { }, ) }, - getCredentials: async (roleName: string) => { - const { data: dataRole } = await this.axios.get( - `/v1/auth/approle/role/${roleName}/role-id`, - { - headers: { 'X-Vault-Token': await this.getToken() }, - }, - ) - const { data: dataSecret } = await this.axios.put( - `/v1/auth/approle/role/${roleName}/secret-id`, - 'null', // Force empty data - { - headers: { 'X-Vault-Token': await this.getToken() }, - }, - ) - return { - roleId: dataRole.data?.role_id, - secretId: dataSecret.data?.secret_id, - } - }, } } @@ -261,7 +233,35 @@ export class VaultProjectApi extends VaultApi { public async destroy(path: string = '/') { if (path.startsWith('/')) path = path.slice(1) - return super.destroy(`${this.projectRootDir}/${this.basePath}/${path}`, this.coreKvName) + return this.axios({ + method: 'delete', + url: `/v1/${this.coreKvName}/metadata/${this.projectRootDir}/${this.basePath}/${path}`, + headers: { 'X-Vault-Token': await this.getToken() }, + + }) + } + + public async getCredentials(): Promise { + const appRoleEnabled = await isAppRoleEnabled(this.axios, await this.getToken()) + if (!appRoleEnabled) return this.defaultAppRoleCredentials + const { data: dataRole } = await this.axios.get( + `/v1/auth/approle/role/${this.roleName}/role-id`, + { + headers: { 'X-Vault-Token': await this.getToken() }, + }, + ) + const { data: dataSecret } = await this.axios.put( + `/v1/auth/approle/role/${this.roleName}/secret-id`, + 'null', // Force empty data + { + headers: { 'X-Vault-Token': await this.getToken() }, + }, + ) + return { + ...this.defaultAppRoleCredentials, + roleId: dataRole.data?.role_id, + secretId: dataSecret.data?.secret_id, + } } Project = { @@ -285,15 +285,6 @@ export class VaultProjectApi extends VaultApi { await this.Group.delete() await this.Role.delete(this.roleName) }, - getCredentials: async () => { - const appRoleEnabled = await isAppRoleEnabled(this.axios, await this.getToken()) - if (!appRoleEnabled) return this.defaultAppRoleCredentials - const creds = await this.Role.getCredentials(this.roleName) - return { - ...this.defaultAppRoleCredentials, - ...creds, - } - }, } Group = { @@ -379,26 +370,4 @@ export class VaultZoneApi extends VaultApi { ) return await response.data } - - public async destroy(path: string = '/') { - if (path.startsWith('/')) - path = path.slice(1) - return super.destroy(path, this.kvName) - } - - public async getCredentials() { - const appRoleEnabled = await isAppRoleEnabled(this.axios, await this.getToken()) - if (!appRoleEnabled) { - return { - url: getConfig().publicUrl, - kvName: this.kvName, - } - } - const creds = await this.Role.getCredentials(this.roleName) - return { - url: getConfig().publicUrl, - kvName: this.kvName, - ...creds, - } - } } diff --git a/plugins/vault/src/functions.ts b/plugins/vault/src/functions.ts index 8c8a02084..0c0bfcb04 100644 --- a/plugins/vault/src/functions.ts +++ b/plugins/vault/src/functions.ts @@ -30,7 +30,7 @@ export const deployAuth: StepCall = async (payload) => { if (!payload.apis.vault) { throw new Error('no Vault available') } - const appRoleCreds = await payload.apis.vault.Project.getCredentials() + const appRoleCreds = await payload.apis.vault.getCredentials() if (getConfig().disableVaultSecrets) { return okStatus } diff --git a/plugins/vault/src/index.ts b/plugins/vault/src/index.ts index 017ecf14d..63caca760 100644 --- a/plugins/vault/src/index.ts +++ b/plugins/vault/src/index.ts @@ -1,4 +1,4 @@ -import type { ClusterObject, DefaultArgs, Plugin, Project, ProjectLite, ZoneObject } from '@cpn-console/hooks' +import type { DefaultArgs, Plugin, Project, ProjectLite, ZoneObject } from '@cpn-console/hooks' import { archiveDsoProject, deleteZone, deployAuth, getSecrets, upsertProject, upsertZone } from './functions.js' import infos from './infos.js' import monitor from './monitor.js' @@ -38,12 +38,6 @@ export const plugin: Plugin = { main: deleteZone, }, }, - upsertCluster: { - api: (c: ClusterObject) => new VaultZoneApi(c.zone.slug), - }, - deleteCluster: { - api: (c: ClusterObject) => new VaultZoneApi(c.zone.slug), - }, }, monitor, } @@ -52,7 +46,7 @@ declare module '@cpn-console/hooks' { interface HookPayloadApis { vault: Args extends (ProjectLite | Project) ? VaultProjectApi - : Args extends (ZoneObject | ClusterObject) + : Args extends (ZoneObject) ? VaultZoneApi : never } From 227baf6802687c9555d3931071a7e049d2756db1 Mon Sep 17 00:00:00 2001 From: ArnaudTa <33383276+ArnaudTA@users.noreply.github.com> Date: Wed, 8 Jan 2025 15:49:10 +0100 Subject: [PATCH 5/5] feat: track project version provisioning --- .../migration.sql | 2 + apps/server/src/prisma/schema/project.prisma | 40 ++++++++++--------- apps/server/src/resources/log/queries.ts | 2 +- apps/server/src/utils/hook-wrapper.spec.ts | 1 + apps/server/src/utils/hook-wrapper.ts | 1 + packages/hooks/src/hooks/hook-project.ts | 5 +++ plugins/kubernetes/package.json | 2 +- plugins/kubernetes/src/class.ts | 8 ++-- plugins/kubernetes/src/namespace.ts | 10 ++--- 9 files changed, 41 insertions(+), 30 deletions(-) create mode 100644 apps/server/src/prisma/migrations/20241112102015_add_provisionning_version/migration.sql diff --git a/apps/server/src/prisma/migrations/20241112102015_add_provisionning_version/migration.sql b/apps/server/src/prisma/migrations/20241112102015_add_provisionning_version/migration.sql new file mode 100644 index 000000000..b143cbeb9 --- /dev/null +++ b/apps/server/src/prisma/migrations/20241112102015_add_provisionning_version/migration.sql @@ -0,0 +1,2 @@ +-- AlterTable +ALTER TABLE "Project" ADD COLUMN "lastSuccessProvisionningVersion" TEXT; diff --git a/apps/server/src/prisma/schema/project.prisma b/apps/server/src/prisma/schema/project.prisma index 485ffe907..f9ac7f472 100644 --- a/apps/server/src/prisma/schema/project.prisma +++ b/apps/server/src/prisma/schema/project.prisma @@ -65,25 +65,27 @@ model ProjectRole { } model Project { - id String @id @unique @default(uuid()) @db.Uuid - name String - organizationId String @db.Uuid - description String @default("") - status ProjectStatus @default(initializing) - locked Boolean @default(false) - createdAt DateTime @default(now()) - updatedAt DateTime @updatedAt - everyonePerms BigInt @default(896) - ownerId String @db.Uuid - environments Environment[] - logs Log[] - organization Organization @relation(fields: [organizationId], references: [id]) - owner User @relation(fields: [ownerId], references: [id]) - members ProjectMembers[] - plugins ProjectPlugin[] - roles ProjectRole[] - repositories Repository[] - clusters Cluster[] @relation("ClusterToProject") + id String @id @unique @default(uuid()) @db.Uuid + name String + organizationId String @db.Uuid + description String @default("") + status ProjectStatus @default(initializing) + locked Boolean @default(false) + createdAt DateTime @default(now()) + updatedAt DateTime @updatedAt + everyonePerms BigInt @default(896) + ownerId String @db.Uuid + environments Environment[] + logs Log[] + organization Organization @relation(fields: [organizationId], references: [id]) + owner User @relation(fields: [ownerId], references: [id]) + members ProjectMembers[] + plugins ProjectPlugin[] + roles ProjectRole[] + repositories Repository[] + clusters Cluster[] @relation("ClusterToProject") + + lastSuccessProvisionningVersion String? } enum ProjectStatus { diff --git a/apps/server/src/resources/log/queries.ts b/apps/server/src/resources/log/queries.ts index 6d6db4609..3851a8f13 100644 --- a/apps/server/src/resources/log/queries.ts +++ b/apps/server/src/resources/log/queries.ts @@ -38,7 +38,7 @@ export function addLogs({ action, data, requestId, userId = null, projectId }: A data: { action, userId, - data: exclude(data, ['cluster', 'user', 'newCreds']), + data: exclude(data, ['cluster', 'user', 'newCreds', 'apis']), requestId, projectId, }, diff --git a/apps/server/src/utils/hook-wrapper.spec.ts b/apps/server/src/utils/hook-wrapper.spec.ts index 45a842a28..5beac71af 100644 --- a/apps/server/src/utils/hook-wrapper.spec.ts +++ b/apps/server/src/utils/hook-wrapper.spec.ts @@ -233,6 +233,7 @@ describe('transformToHookProject', () => { stage: stage.name, permissions: [{ permissions: { rw: true, ro: true }, userId: project.ownerId }], ...environment, + apis: {}, }))) // Assert sur la transformation des repositories diff --git a/apps/server/src/utils/hook-wrapper.ts b/apps/server/src/utils/hook-wrapper.ts index 1636bf1c5..49138b0eb 100644 --- a/apps/server/src/utils/hook-wrapper.ts +++ b/apps/server/src/utils/hook-wrapper.ts @@ -199,6 +199,7 @@ export function transformToHookProject(project: ProjectInfos, store: Store, repo })), ], ...environment, + apis: {}, })), repositories: project.repositories.map(repo => ({ ...repo, newCreds: reposCreds[repo.internalRepoName] })), store, diff --git a/packages/hooks/src/hooks/hook-project.ts b/packages/hooks/src/hooks/hook-project.ts index 5cf2cfc1a..508a676d5 100644 --- a/packages/hooks/src/hooks/hook-project.ts +++ b/packages/hooks/src/hooks/hook-project.ts @@ -1,3 +1,4 @@ +import type { PluginApi } from '@/utils/utils.js' import type { Hook } from './hook.js' import { createHook } from './hook.js' import type { ClusterObject, ExternalRepoUrl, InternalRepoName, IsInfra, IsPrivate, UserObject } from './index.js' @@ -17,6 +18,9 @@ export interface Role { role: 'owner' | 'user' } +export interface EnvironmentApis { + [x: string]: PluginApi +} export interface Environment { id: string name: string @@ -30,6 +34,7 @@ export interface Environment { rw: boolean } }[] + apis: EnvironmentApis } export interface Repository { diff --git a/plugins/kubernetes/package.json b/plugins/kubernetes/package.json index 703879822..2ea4bdaee 100644 --- a/plugins/kubernetes/package.json +++ b/plugins/kubernetes/package.json @@ -1,7 +1,7 @@ { "name": "@cpn-console/kubernetes-plugin", "type": "module", - "version": "2.0.7", + "version": "2.1.0", "private": false, "description": "", "main": "dist/index.js", diff --git a/plugins/kubernetes/src/class.ts b/plugins/kubernetes/src/class.ts index 46bf2aa3a..4f37ad372 100644 --- a/plugins/kubernetes/src/class.ts +++ b/plugins/kubernetes/src/class.ts @@ -40,12 +40,12 @@ class KubernetesNamespace { anyObjectApi: AnyObjectsApi | undefined apisApi: ApisApi | undefined - constructor(organizationName: string, projectName: string, environmentName: string, owner: UserObject, cluster: ClusterObject, projectId: string) { + constructor(organizationName: string, projectName: string, environment: Environment, owner: UserObject, cluster: ClusterObject, projectId: string) { this.coreV1Api = createCoreV1Api(cluster) this.apisApi = createApisApi(cluster) this.anyObjectApi = createCustomObjectApi(cluster) - this.nsObjectExpected = getNsObject(organizationName, projectName, environmentName, owner, cluster.zone.slug, projectId) - this.nsObject = getNsObject(organizationName, projectName, environmentName, owner, cluster.zone.slug, projectId) + this.nsObjectExpected = getNsObject(organizationName, projectName, environment, owner, cluster.zone.slug, projectId) + this.nsObject = getNsObject(organizationName, projectName, environment, owner, cluster.zone.slug, projectId) } public async create() { @@ -131,7 +131,7 @@ export class KubernetesProjectApi extends PluginApi { const owner = project.owner this.namespaces = project.environments.reduce((acc, env) => { const cluster = project.clusters.find(cluster => cluster.id === env.clusterId) as ClusterObject - acc[env.name] = new KubernetesNamespace(project.organization.name, project.name, env.name, owner, cluster, project.id) + acc[env.name] = new KubernetesNamespace(project.organization.name, project.name, env, owner, cluster, project.id) return acc }, {} as Record) } diff --git a/plugins/kubernetes/src/namespace.ts b/plugins/kubernetes/src/namespace.ts index 95806a823..0924be6be 100644 --- a/plugins/kubernetes/src/namespace.ts +++ b/plugins/kubernetes/src/namespace.ts @@ -1,6 +1,6 @@ import { createHmac } from 'node:crypto' import { Namespace } from 'kubernetes-models/v1' -import type { Project, ProjectLite, StepCall, UserObject } from '@cpn-console/hooks' +import type { Environment, Project, ProjectLite, StepCall, UserObject } from '@cpn-console/hooks' import { parseError } from '@cpn-console/hooks' import type { CoreV1Api, V1NamespaceList } from '@kubernetes/client-node' import { createCoreV1Api } from './api.js' @@ -83,19 +83,19 @@ export const deleteNamespaces: StepCall = async (payload) => { } // Utils -export function getNsObject(organization: string, project: string, environment: string, owner: UserObject, zone: string, projectId: string): V1NamespacePopulated { +export function getNsObject(organization: string, project: string, environment: Environment, owner: UserObject, zone: string, projectId: string): V1NamespacePopulated { const nsObject = new Namespace({ metadata: { - name: generateNamespaceName(organization, project, environment), + name: generateNamespaceName(organization, project, environment.name), labels: { 'dso/organization': organization, - 'dso/projet': project, 'dso/project': project, - 'dso/environment': environment, + 'dso/environment': environment.name, 'dso/owner.id': owner.id, 'app.kubernetes.io/managed-by': 'dso-console', 'dso/zone': zone, 'dso/project.id': projectId, + 'dso/environment.id': environment.id, }, }, })