plan a charm file resource built by local script (#454)

.github/workflows/integration_test.yaml

@@ -178,7 +178,7 @@ on:
       zap-target-port:
         description: ZAP target port
         type: string
-        default: 80
+        default: '80'
       zap-target-protocol:
         description: ZAP target protocol
         type: string

.github/workflows/integration_test_run.yaml

@@ -143,7 +143,7 @@ on:
       zap-target-port:
         description: ZAP target port
         type: string
-        default: 80
+        default: '80'
       zap-target-protocol:
         description: ZAP target protocol
         type: string

README.md

@@ -28,6 +28,8 @@ The following secrets are available for this workflow:
 When running the integration tests, the following posargs will be automatically passed to the `integration` target:
 
 * --charm-file [charm_file_name]: The name of the charm artifact generated prior to the integration tests run, this argument can be supplied multiple times for charm with multiple bases.
+* --{image-name}-image: The name of the image artifact built prior to the integration tests run, this argument may be supplied multiple times or not at all depending on the plan
+* --{resource-name}-resource: The name of the charm file resources built prior to the integration tests run, this argument may be supplied multiple times or not at all depending on the plan
 * --series [series]: As defined in the `series` configuration described option above
 * -k [module]: As defined in the `modules` configuration option described above
 * --keep-models

src/build.ts

@@ -135,6 +135,35 @@ interface BuildDockerImageParams {
   token: string
 }
 
+async function buildFileResource(plan: BuildPlan): Promise<void> {
+  core.startGroup(`Build resource {plan.name}`)
+  if (!plan.build_target) {
+    throw new Error('build_target is required for file resources')
+  }
+  await exec.exec(`./${plan.source_file}`, [plan.build_target], {
+    cwd: plan.source_directory
+  })
+  core.endGroup()
+  const resourceFiles = await (
+    await glob.create(path.join(plan.source_directory, plan.build_target))
+  ).glob()
+  const artifact = new DefaultArtifactClient()
+  const manifestFile = path.join(plan.source_directory, 'manifest.json')
+  fs.writeFileSync(
+    manifestFile,
+    JSON.stringify(
+      { name: plan.name, files: resourceFiles.map(f => path.basename(f)) },
+      null,
+      2
+    )
+  )
+  await artifact.uploadArtifact(
+    plan.output,
+    [...resourceFiles, manifestFile],
+    plan.source_directory
+  )
+}
+
 async function buildDockerImage({
   plan,
   user,
@@ -366,6 +395,10 @@ export async function run(): Promise<void> {
           user: github.context.actor,
           token: core.getInput('github-token')
         })
+        break
+      case 'file':
+        await buildFileResource(plan)
+        break
     }
   } catch (error) {
     // Fail the workflow run if an error occurs

src/model.ts

@@ -7,10 +7,18 @@ export interface Plan {
 }
 
 export interface BuildPlan {
-  type: 'charm' | 'rock' | 'docker-image'
+  type: 'charm' | 'rock' | 'docker-image' | 'file'
   name: string
   source_file: string
   source_directory: string
+  build_target: string | undefined
   output_type: 'file' | 'registry'
   output: string
 }
+
+export interface CharmResource {
+  type: 'file' | 'oci-image'
+  description?: string
+  filename?: string
+  'upstream-source'?: string
+}

src/plan-integration.ts

@@ -23,17 +23,21 @@ export async function run(): Promise<void> {
       const manifest = JSON.parse(
         fs.readFileSync(path.join(tmp, 'manifest.json'), { encoding: 'utf-8' })
       ) as object
-      if (build.type === 'charm') {
+      if (build.type === 'charm' || build.type === 'file') {
         // @ts-ignore
-        for (const file of manifest.files) {
+        for (const file of manifest.files as string[]) {
           fs.renameSync(
             path.join(tmp, file),
             path.join(plan.working_directory, file)
           )
-          args.push(`--charm-file=./${file}`)
+          const file_path = path.resolve(plan.working_directory, file)
+          // @ts-ignore
+          const name = manifest.name as string
+          let argName: string =
+            build.type === 'charm' ? 'charm-file' : `${name}-resource`
+          args.push(`--${argName}=${file_path}`)
         }
-      }
-      if (build.type === 'rock' || build.type == 'docker-image') {
+      } else if (build.type === 'rock' || build.type == 'docker-image') {
         // @ts-ignore
         const name = manifest.name as string
         if ('files' in manifest) {

src/plan.ts

@@ -7,7 +7,7 @@ import * as path from 'path'
 import * as yaml from 'js-yaml'
 import * as fs from 'fs'
 import * as github from '@actions/github'
-import { Plan, BuildPlan } from './model'
+import { Plan, BuildPlan, CharmResource } from './model'
 import { DefaultArtifactClient } from '@actions/artifact'
 import * as os from 'os'
 
@@ -71,6 +71,7 @@ async function planBuildCharm(
       name,
       source_file: file,
       source_directory: path.dirname(file),
+      build_target: undefined,
       output_type: 'file',
       output: sanitizeArtifactName(`${id}__build__output__charm__${name}`)
     }
@@ -97,6 +98,7 @@ async function planBuildRock(
       name,
       source_file: file,
       source_directory: path.dirname(file),
+      build_target: undefined,
       output_type: outputType,
       output: sanitizeArtifactName(`${id}__build__output__rock__${name}`)
     }
@@ -119,6 +121,7 @@ async function planBuildDockerImage(
       name,
       source_file: file,
       source_directory: path.dirname(file),
+      build_target: undefined,
       output_type: outputType,
       output: sanitizeArtifactName(
         `${id}__build__output__docker-image__${name}`
@@ -127,6 +130,76 @@ async function planBuildDockerImage(
   })
 }
 
+async function planBuildFileResource(
+  workingDir: string,
+  id: string
+): Promise<BuildPlan[]> {
+  const allCharmcraftFiles = await (
+    await glob.create(path.join(workingDir, '**', 'charmcraft.yaml'))
+  ).glob()
+  const charmcraftFiles = allCharmcraftFiles.filter(
+    file =>
+      !path.normalize(path.relative(workingDir, file)).startsWith('tests/')
+  )
+  return charmcraftFiles.flatMap((charmcraftFile: string) => {
+    const file = path.join(
+      workingDir,
+      path.relative(workingDir, charmcraftFile)
+    )
+    const charmcraft = yaml.load(
+      fs.readFileSync(charmcraftFile, { encoding: 'utf-8' })
+    ) as object
+    const metadataFile = path.join(
+      path.dirname(charmcraftFile),
+      'metadata.yaml'
+    )
+    const metadataExists = fs.existsSync(metadataFile)
+    const metadata = metadataExists
+      ? (yaml.load(
+          fs.readFileSync(metadataFile, { encoding: 'utf-8' })
+        ) as object)
+      : {}
+
+    let charmName: string = ''
+    if ('name' in charmcraft) {
+      charmName = charmcraft['name'] as string
+    } else if ('name' in metadata) {
+      charmName = metadata.name as string
+    } else {
+      throw new Error(`unknown charm name (${workingDir})`)
+    }
+
+    let resources: Map<string, CharmResource> = new Map()
+    if ('resources' in charmcraft) {
+      resources = charmcraft['resources'] as Map<string, CharmResource>
+    }
+    if ('resources' in metadata) {
+      resources = metadata['resources'] as Map<string, CharmResource>
+    }
+
+    return Object.entries(resources).reduce(
+      (acc, [resourceName, resource]: [string, CharmResource]) => {
+        if (resource.type === 'file' && resource.filename) {
+          let parent = path.dirname(file)
+          acc.push({
+            type: 'file',
+            name: resourceName,
+            source_file: `build-${resourceName}.sh`,
+            build_target: resource.filename,
+            source_directory: parent,
+            output_type: 'file',
+            output: sanitizeArtifactName(
+              `${id}__build__output__file__${charmName}__${resourceName}`
+            )
+          })
+        }
+        return acc
+      },
+      [] as BuildPlan[]
+    )
+  })
+}
+
 async function planBuild(
   workingDir: string,
   id: string,
@@ -135,7 +208,8 @@ async function planBuild(
   return [
     ...(await planBuildCharm(workingDir, id)),
     ...(await planBuildRock(workingDir, id, imageOutputType)),
-    ...(await planBuildDockerImage(workingDir, id, imageOutputType))
+    ...(await planBuildDockerImage(workingDir, id, imageOutputType)),
+    ...(await planBuildFileResource(workingDir, id))
   ]
 }
 
@@ -172,7 +246,7 @@ export async function run(): Promise<void> {
       working_directory: workingDir,
       build: buildPlans
     }
-    core.info(`generated workflow plan: ${JSON.stringify(plan, null, 2)}`)
+    core.info(`Generated workflow plan: ${JSON.stringify(plan, null, 2)}`)
     const artifact = new DefaultArtifactClient()
     const tmp = fs.mkdtempSync(path.join(os.tmpdir(), 'plan-'))
     const pathFile = path.join(tmp, 'plan.json')

src/publish.ts

@@ -146,11 +146,12 @@ class Publish {
     throw new Error(`can't find plan artifact for workflow run ${runId}`)
   }
 
-  async getImageResources(): Promise<string[]> {
+  async getCharmResources(): Promise<[string[], string[]]> {
     interface Metadata {
       resources?: {
         [name: string]: {
           type: string
+          filename?: string
           'upstream-source'?: string
         }
       }
@@ -185,15 +186,69 @@ class Publish {
     }
     const resources = metadata.resources
     if (resources === undefined) {
-      return []
+      return [[], []]
     }
-    return Object.keys(resources).filter(
+    let images = Object.keys(resources).filter(
       k => resources[k].type === 'oci-image' && !resources[k]['upstream-source']
     )
+    let files = Object.keys(resources).filter(k => resources[k].type === 'file')
+    return [images, files]
+  }
+
+  async getFiles(): Promise<Map<string, string>> {
+    const [, resources] = await this.getCharmResources()
+    core.info(`required resources: ${resources}`)
+    const upload: Map<string, string> = new Map()
+    if (resources.length === 0) {
+      return upload
+    }
+    const runId = await this.findWorkflowRunId()
+    const plan = await this.getPlan(runId)
+    for (const build of plan.build) {
+      if (build.type === 'file') {
+        const resourceName = this.resourceMapping[build.name]
+        if (!resources.includes(resourceName)) {
+          core.info(`skip uploading file: ${build.name}`)
+          continue
+        }
+        const tmp = this.mkdtemp()
+        const artifact = await this.artifact.getArtifact(build.output, {
+          findBy: {
+            token: this.token,
+            repositoryOwner: github.context.repo.owner,
+            repositoryName: github.context.repo.repo,
+            workflowRunId: runId
+          }
+        })
+        await this.artifact.downloadArtifact(artifact.artifact.id, {
+          path: tmp,
+          findBy: {
+            token: this.token,
+            repositoryOwner: github.context.repo.owner,
+            repositoryName: github.context.repo.repo,
+            workflowRunId: runId
+          }
+        })
+        const manifest = JSON.parse(
+          fs.readFileSync(path.join(tmp, 'manifest.json'), {
+            encoding: 'utf-8'
+          })
+        )
+        const files = manifest.files as string[]
+        if (files.length !== 1) {
+          throw new Error(
+            `file resource ${build.name} contain multiple candidates: ${files}`
+          )
+        }
+        const file = files[0]
+        upload.set(resourceName, file)
+      }
+    }
+    return upload
   }
 
   async getImages(): Promise<Map<string, string>> {
-    const resources = await this.getImageResources()
+    const [resources] = await this.getCharmResources()
     core.info(`required resources: ${resources}`)
     const upload: Map<string, string> = new Map()
     if (resources.length === 0) {
@@ -203,7 +258,7 @@ class Publish {
     const plan = await this.getPlan(runId)
     let dockerLogin = false
     for (const build of plan.build) {
-      if (build.type === 'charm') {
+      if (build.type === 'charm' || build.type === 'file') {
         continue
       }
       const resourceName = this.resourceMapping.hasOwnProperty(build.name)
@@ -346,7 +401,8 @@ class Publish {
   async run() {
     try {
       core.startGroup('retrieve image info')
-      const images = await this.getImages()
+      const imageResources = await this.getImages()
+      const fileResources = await this.getFiles()
       core.endGroup()
       core.startGroup('retrieve charm info')
       const {
@@ -355,15 +411,36 @@ class Publish {
         files: charms
       } = await this.getCharms()
       core.endGroup()
-      core.info(
-        `start uploading image resources: ${JSON.stringify(Object.fromEntries([...images]))}`
-      )
-      for (const resource of images.keys()) {
+      if (fileResources.size !== 0) {
+        core.info(
+          `start uploading file resources: ${JSON.stringify(Object.fromEntries([...fileResources]))}`
+        )
+      }
+      for (const [resource, filePath] of fileResources) {
+        core.info(`upload resource ${resource}`)
+        await exec.exec(
+          'charmcraft',
+          [
+            'upload-resource',
+            charmName,
+            resource,
+            `--filepath=${filePath}`,
+            '--verbosity=brief'
+          ],
+          { env: { CHARMCRAFT_AUTH: this.charmhubToken } }
+        )
+      }
+      if (imageResources.size !== 0) {
+        core.info(
+          `start uploading image resources: ${JSON.stringify(Object.fromEntries([...imageResources]))}`
+        )
+      }
+      for (const [resource, image] of imageResources) {
         core.info(`upload resource ${resource}`)
         const imageId = (
           await exec.getExecOutput('docker', [
             'images',
-            images.get(resource) as string,
+            image,
             '--format',
             '{{.ID}}'
           ])