Merge pull request #46 from machacekondra/remove_collector

Remove collector container and merge it with planner
kubev2v · Oct 23, 2024 · 2482490 · 2482490
2 parents d44ee36 + eb3c530
commit 2482490
Show file tree

Hide file tree

Showing 11 changed files with 123 additions and 196 deletions.
diff --git a/Containerfile.agent b/Containerfile.agent
@@ -15,7 +15,7 @@ RUN go mod download
 COPY . .
 
 USER 0
-RUN CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -buildvcs=false -o /planner-agent cmd/planner-agent/main.go
+RUN CGO_ENABLED=1 GOOS=linux GOARCH=amd64 go build -buildvcs=false -o /planner-agent cmd/planner-agent/main.go
 
 FROM registry.access.redhat.com/ubi9/ubi-micro
 

diff --git a/Containerfile.collector b/Containerfile.collector
diff --git a/Makefile b/Makefile
@@ -7,7 +7,6 @@ GO_CACHE := -v $${HOME}/go/migration-planner-go-cache:/opt/app-root/src/go:Z -v
 TIMEOUT ?= 30m
 VERBOSE ?= false
 MIGRATION_PLANNER_AGENT_IMAGE ?= quay.io/kubev2v/migration-planner-agent
-MIGRATION_PLANNER_COLLECTOR_IMAGE ?= quay.io/kubev2v/migration-planner-collector
 MIGRATION_PLANNER_API_IMAGE ?= quay.io/kubev2v/migration-planner-api
 MIGRATION_PLANNER_UI_IMAGE ?= quay.io/kubev2v/migration-planner-ui
 DOWNLOAD_RHCOS ?= true
@@ -81,23 +80,18 @@ build-api: bin
 bin/.migration-planner-agent-container: bin Containerfile.agent go.mod go.sum $(GO_FILES)
 	podman build -f Containerfile.agent -t $(MIGRATION_PLANNER_AGENT_IMAGE):latest
 
-bin/.migration-planner-collector-container: bin Containerfile.collector go.mod go.sum $(GO_FILES)
-	podman build -f Containerfile.collector -t $(MIGRATION_PLANNER_COLLECTOR_IMAGE):latest
-
 bin/.migration-planner-api-container: bin Containerfile.api go.mod go.sum $(GO_FILES)
 	podman build -f Containerfile.api -t $(MIGRATION_PLANNER_API_IMAGE):latest
 
 migration-planner-api-container: bin/.migration-planner-api-container
-migration-planner-collector-container: bin/.migration-planner-collector-container
 migration-planner-agent-container: bin/.migration-planner-agent-container
 
-build-containers: migration-planner-api-container migration-planner-agent-container migration-planner-collector-container
+build-containers: migration-planner-api-container migration-planner-agent-container
 
 .PHONY: build-containers
 
 push-containers: build-containers
 	podman push $(MIGRATION_PLANNER_API_IMAGE):latest
-	podman push $(MIGRATION_PLANNER_COLLECTOR_IMAGE):latest
 	podman push $(MIGRATION_PLANNER_AGENT_IMAGE):latest
 
 deploy-on-openshift:

diff --git a/cmd/collector/README.md → cmd/planner-agent/COLLECTOR.md b/cmd/collector/README.md → cmd/planner-agent/COLLECTOR.md
@@ -4,11 +4,29 @@ To run the collector localy here are the steps.
 ## Prepare
 Prepare the dependencies.
 
+### Configuration 
+Create the planner-agent configuration file:
+
+```
+$ mkdir /tmp/config
+$ mkdir /tmp/data
+$ cat <<EOF > ~/.planner-agent/config.yaml
+config-dir: /tmp/config
+data-dir: /tmp/data
+log-level: debug
+source-id: 9195e61d-e56d-407d-8b29-ff2fb7986928
+update-interval: 5s
+planner-service:
+  service:
+    server: http://127.0.0.1:7443
+EOF
+```
+
 ### Credentials
 Create VMware credentials file.
 
 ```
-cat <<EOF > /tmp/creds.json
+cat <<EOF > /tmp/data/creds.json
 {
   "username": "[email protected]",
   "password": "userpassword",
@@ -28,7 +46,7 @@ podman run -p 8181:8181 -d --name opa --entrypoint '/usr/bin/opa' quay.io/kubev2
 Build & run the collector code specifying credentials file as first argument and as second path to invetory file, where data should be written.
 
 ```
-go run cmd/collector/main.go /tmp/creds.json /tmp/inventory.json
+go run cmd/planner-agent/main.go -config -config ~/.planner-agent/config.yaml
 ```
 
-Explore `/tmp/inventory.json`
+Explore `/tmp/data/inventory.json`
diff --git a/data/ignition.template b/data/ignition.template
@@ -22,19 +22,19 @@ storage:
         name: core
       group:
         name: core
-    - path: /home/core/vol
+    - path: /home/core/.migration-planner
       overwrite: true
       user:
         name: core
       group:
         name: core
-    - path: /home/core/vol/config
+    - path: /home/core/.migration-planner/config
       overwrite: true
       user:
         name: core
       group:
         name: core
-    - path: /home/core/vol/data
+    - path: /home/core/.migration.planner/data
       overwrite: true
       user:
         name: core
@@ -46,7 +46,7 @@ storage:
       contents:
         inline: |
           PasswordAuthentication yes
-    - path: /home/core/vol/config.yaml
+    - path: /home/core/.migration-planner/config/config.yaml
       contents:
         inline: |
           config-dir: /agent/config
@@ -63,58 +63,32 @@ storage:
         name: core
       group:
         name: core
-    - path: /home/core/.config/containers/systemd/collector.network
+    - path: /home/core/.config/containers/systemd/agent.network
       contents:
         inline: |
           [Network]
       user:
         name: core
       group:
         name: core
-    - path: /home/core/.config/containers/systemd/planner.volume
-      contents:
-        inline: |
-          [Volume]
-          VolumeName=planner.volume
-      user:
-        name: core
-      group:
-        name: core
-    - path: /home/core/.config/containers/systemd/planner-setup.container
-      mode: 0644
-      contents:
-        inline: |
-          [Unit]
-          Description=Prepare data volume for the container
-          Before=planner-agent.service
-
-          [Container]
-          Image=registry.access.redhat.com/ubi9/ubi-micro
-          Exec=sh -c "cp -r /mnt/* /agent/ && chmod -R a+rwx /agent"
-          Volume=planner.volume:/agent
-          Volume=/home/core/vol:/mnt:Z
-
-          [Service]
-          Type=oneshot
-          RemainAfterExit=yes
-
-          [Install]
-          WantedBy=multi-user.target default.target
     - path: /home/core/.config/containers/systemd/planner-agent.container
       mode: 0644
       contents:
         inline: |
           [Unit]
           Description=Planner agent quadlet
-          Wants=planner-setup.service
+          Wants=planner-agent-opa.service
 
           [Container]
           Image={{.MigrationPlannerAgentImage}}
           ContainerName=planner-agent
           AutoUpdate=registry
-          Exec= -config /agent/config.yaml
+          Exec= -config /agent/config/config.yaml
           PublishPort=3333:3333
-          Volume=planner.volume:/agent
+          Volume=/home/core/.migration-planner:/agent:Z
+          Environment=OPA_SERVER=opa:8181
+          Network=agent.network
+          UserNS=keep-id:uid=1001
 
           [Install]
           WantedBy=multi-user.target default.target
@@ -123,39 +97,16 @@ storage:
       contents:
         inline: |
           [Unit]
-          Description=Collector quadlet
-          Before=planner-agent-collector.service
+          Description=OPA quadlet
+          Before=planner-agent.service
 
           [Container]
           ContainerName=opa
           Image=quay.io/kubev2v/forklift-validation:release-v2.6.4
           Entrypoint=/usr/bin/opa
           PublishPort=8181:8181
           Exec=run --server /usr/share/opa/policies
-          Network=collector.network
-
-          [Install]
-          WantedBy=multi-user.target default.target
-
-    - path: /home/core/.config/containers/systemd/planner-agent-collector.container
-      mode: 0644
-      contents:
-        inline: |
-          [Unit]
-          Description=Collector quadlet
-          Wants=planner-agent-opa.service
-
-          [Container]
-          Image={{.MigrationPlannerCollectorImage}}
-          ContainerName=migration-planner-collector
-          AutoUpdate=registry
-          Exec=/vol/data/credentials.json /vol/data/inventory.json
-          Volume=planner.volume:/vol
-          Environment=OPA_SERVER=opa:8181
-          Network=collector.network
-
-          [Service]
-          Restart=on-failure
+          Network=agent.network
 
           [Install]
           WantedBy=multi-user.target default.target
diff --git a/doc/agentvm.md b/doc/agentvm.md
@@ -4,23 +4,17 @@ The VM is initialized using ignition, which configures multiple containers that
 
 ## Systemd services
 The following are a list of systemd services that can be found on agent virtual machines. All of the services
-are defined as quadlets. Quadlet configuration can be found in the [ignition template file](../data/config.ign.template).
-The Agent containerfile can be found [here](../Containerfile.agent). The collector containerfile is [here](../Containerfile.collector).
-
-### planner-setup
-Planner-setup service is responsible for initializing the volume with data that is shared between the `planner-agent` and the `planner-agent-collector`.
+are defined as quadlets. Quadlet configuration can be found in the [ignition template file](../data/ignition.template).
+The Agent containerfile can be found [here](../Containerfile.agent).
 
 ### planner-agent
-Planner-agent is a service that reports the status to the Agent service. The URL of the Agent service is configured in the file `$HOME/vol/config.yaml`, which is injected via ignition.
+Planner-agent is a service that reports the status to the Agent service. The URL of the Agent service is configured in the file `$HOME/.migration-planner/config/config.yaml`, which is injected via ignition.
 
-The Planner-agent contains a web application that is exposed via tcp port 3333. Once the user accesses the web application and enters the credentials of their vCenter, the `credentials.json` file is created on the shared volume and the `planner-agent-collector` container is spawned.
+The Planner-agent contains a web application that is exposed via tcp port 3333. Once the user accesses the web application and enters the credentials of their vCenter, the `credentials.json` file is created on the shared volume and the `collector` goroutine is spawned, which fetches the vCenter data. The data is stored in `$HOME/.migration-planner/data/inventory.json`. Once `inventory.json` is created, the `planner-agent` service sends the data over to Agent service.
 
 ### planner-agent-opa
 Planner-agent-opa is a service that re-uses the [forklift validation](https://github.com/kubev2v/forklift/blob/main/validation/README.adoc) container. The forklift validation container is responsible for vCenter data validation. When the `planner-agent-collector` fetches vCenter data, it's validated against the OPA server and the report is shared back to the Agent Service.
 
-### planner-agent-collector
-Planner-agent-collector service waits until the user enters the vCenter credentials in the `planner-agent` web application. Once the credentials are entered, the vCenter data is collected. The data is stored in `$HOME/vol/data/inventory.json`. Once `inventory.json` is created, the `planner-agent` service sends the data over to Agent service.
-
 ### podman-auto-update
 Podman auto update is responsible for updating the image of the containers in case there is a new image release. The default `podman-auto-update.timer` is used, which executes `podman-auto-update` every 24 hours.
 
@@ -32,36 +26,21 @@ Useful commands to troubleshoot the Agent VM. Note that all the containers are r
 $ podman ps
 ```
 
-### Checking the status of all our services
-```
-$ systemctl --user status planner-*
-```
-
-### Inspecting the shared volume
-A shared volume is created between containers, so that information can be shared between the `planner-agent-collector` and `planner-agent` containers.
-In order to export the data stored in the volume, find the mountpoint of the volume:
-```
-$ podman volume inspect planner.volume | jq .[0].Mountpoint
-```
-
-And then the relevant data can be explored, such as: `config.yaml`, `credentials.json`, `inventory.json`, etc.
+### Checking the status of planner-agent service
 ```
-$ ls /var/home/core/.local/share/containers/storage/volumes/planner.volume/_data
-$ cat /var/home/core/.local/share/containers/storage/volumes/planner.volume/_data/config.yaml
-$ cat /var/home/core/.local/share/containers/storage/volumes/planner.volume/_data/data/credentials.json
-$ cat /var/home/core/.local/share/containers/storage/volumes/planner.volume/_data/data/inventory.json
+$ systemctl --user status planner-agent
 ```
 
 ### Inspecting the host directory with data
-The ignition create a `vol` directory in `core` user home directory.
+The ignition create a `.migration-planner` directory in `core` user home directory.
 This directory should contain all relevant data, so in order to find misconfiguration please search in this directory.
 ```
-$ ls -l vol
+$ ls -l .migration-planner
 ```
 
 ### Check logs of the services
 ```
-$ journalctl --user -f -u planner-*
+$ journalctl --user -f -u planner-agent
 ```
 
 ### Status is `Not connected` after VM is booted.

diff --git a/doc/deployment.md b/doc/deployment.md
@@ -31,8 +31,6 @@ Agent images are defined in the ignition file. In order to modify the images of
 
 ```
 env:
-  - name: MIGRATION_PLANNER_COLLECTOR_IMAGE
-    value: quay.io/$USER/migration-planner-collector
   - name: MIGRATION_PLANNER_AGENT_IMAGE
     value: quay.io/$USER/migration-planner-agent
-```
+```
diff --git a/internal/agent/agent.go b/internal/agent/agent.go
@@ -92,6 +92,9 @@ func (a *Agent) Run(ctx context.Context) error {
 	}
 	healthChecker.Start(healthCheckCh)
 
+	collector := NewCollector(a.log, a.config.DataDir)
+	collector.collect(ctx)
+
 	inventoryUpdater := NewInventoryUpdater(a.log, a.config, client)
 	inventoryUpdater.UpdateServiceWithInventory(ctx)