Create self-hosted runner for integration(-ish) CI tests

.github/workflows/deploy-test-runner.yml

@@ -0,0 +1,46 @@
+name: Deploy the test runner vm to azure
+
+on:
+    workflow_call:
+    workflow_dispatch:
+
+jobs:
+  deploy_test_vm:
+    name: Deploy VM to azure
+    runs-on: ubuntu-latest
+    env:
+        ARM_CLIENT_ID: ${{ secrets.ARM_CLIENT_ID }}
+        ARM_CLIENT_SECRET: ${{ secrets.ARM_CLIENT_SECRET }}
+        ARM_SUBSCRIPTION_ID: ${{ secrets.ARM_SUBSCRIPTION_ID }}
+        ARM_TENANT_ID: ${{ secrets.ARM_TENANT_ID }}
+    permissions:
+      contents: "read"
+      id-token: "write"
+
+    defaults:
+      run:
+        working-directory: ./infrastructure
+
+    steps:
+      - name: Checkout the Code
+        uses: actions/checkout@v3
+
+      - name: Setup Terraform
+        uses: hashicorp/setup-terraform@v2
+
+      - name: Terraform fmt
+        id: fmt
+        run: terraform fmt -check
+        continue-on-error: true
+
+      - name: Terraform Init
+        id: init
+        run: terraform init
+
+      - name: Terraform Validate
+        id: validate
+        run: terraform validate -no-color
+
+      - run: terraform apply -auto-approve
+        env:
+          TF_VAR_github_runner_token: ${{ secrets.runner_token }}

.github/workflows/destroy-test-runner.yml

@@ -0,0 +1,46 @@
+name: Destroy the test runner vm to azure
+
+on:
+    workflow_call:
+    workflow_dispatch:
+
+jobs:
+  destroy_test_vm:
+    name: Destroy VM to azure
+    runs-on: ubuntu-latest
+    env:
+        ARM_CLIENT_ID: ${{ secrets.ARM_CLIENT_ID }}
+        ARM_CLIENT_SECRET: ${{ secrets.ARM_CLIENT_SECRET }}
+        ARM_SUBSCRIPTION_ID: ${{ secrets.ARM_SUBSCRIPTION_ID }}
+        ARM_TENANT_ID: ${{ secrets.ARM_TENANT_ID }}
+    permissions:
+      contents: "read"
+      id-token: "write"
+
+    defaults:
+      run:
+        working-directory: ./infrastructure
+
+    steps:
+      - name: Checkout the Code
+        uses: actions/checkout@v3
+
+      - name: Setup Terraform
+        uses: hashicorp/setup-terraform@v2
+
+      - name: Terraform fmt
+        id: fmt
+        run: terraform fmt -check
+        continue-on-error: true
+
+      - name: Terraform Init
+        id: init
+        run: terraform init
+
+      - name: Terraform Validate
+        id: validate
+        run: terraform validate -no-color
+
+      - run: terraform destroy -auto-approve -refresh=False
+        env:
+          TF_VAR_github_runner_token: ${{ secrets.runner_token }}

infrastructure/deploy.tf

@@ -0,0 +1,96 @@
+resource "azurerm_resource_group" "example" {
+  name     = "zenml-test"
+  location = "West Europe"
+}
+
+resource "azurerm_virtual_network" "example" {
+  name                = "mlstack-test-network"
+  address_space       = ["10.0.0.0/16"]
+  location            = azurerm_resource_group.example.location
+  resource_group_name = azurerm_resource_group.example.name
+}
+
+resource "azurerm_subnet" "example" {
+  name                 = "mlstack-subnet"
+  resource_group_name  = azurerm_resource_group.example.name
+  virtual_network_name = azurerm_virtual_network.example.name
+  address_prefixes     = ["10.0.2.0/24"]
+}
+
+resource "azurerm_network_interface" "example" {
+  name                = "mlstack-nic"
+  location            = azurerm_resource_group.example.location
+  resource_group_name = azurerm_resource_group.example.name
+
+  ip_configuration {
+    name                          = "mlstack-ip"
+    subnet_id                     = azurerm_subnet.example.id
+    private_ip_address_allocation = "Dynamic"
+    public_ip_address_id          = azurerm_public_ip.example.id
+  }
+}
+
+resource "azurerm_public_ip" "example" {
+  name                = "mlstack-pip"
+  location            = azurerm_resource_group.example.location
+  resource_group_name = azurerm_resource_group.example.name
+  allocation_method   = "Dynamic"
+}
+
+resource "azurerm_network_security_group" "example" {
+  name                = "mlstack-nsg"
+  location            = azurerm_resource_group.example.location
+  resource_group_name = azurerm_resource_group.example.name
+}
+
+resource "azurerm_network_security_rule" "example" {
+  name                        = "SSH"
+  priority                    = 1001
+  direction                   = "Inbound"
+  access                      = "Allow"
+  protocol                    = "Tcp"
+  source_port_range           = "*"
+  destination_port_range      = "22"
+  source_address_prefix       = "*"
+  destination_address_prefix  = "*"
+  resource_group_name         = azurerm_resource_group.example.name
+  network_security_group_name = azurerm_network_security_group.example.name
+}
+
+resource "azurerm_network_interface_security_group_association" "example" {
+  network_interface_id      = azurerm_network_interface.example.id
+  network_security_group_id = azurerm_network_security_group.example.id
+}
+
+data "azurerm_ssh_public_key" "example" {
+  name                = "mlstack-test-vm"
+  resource_group_name = "zenml-developers"
+}
+
+data "azurerm_image" "example" {
+  name                = "mlstack-github-runner-machine-image-20230819162059"
+  resource_group_name = "zenml-developers"
+}
+
+resource "azurerm_linux_virtual_machine" "example" {
+  name                = "mlstack-test-machine"
+  resource_group_name = azurerm_resource_group.example.name
+  location            = azurerm_resource_group.example.location
+  size                = "Standard_D8s_v3"
+  admin_username      = "mlstackuser"
+  network_interface_ids = [
+    azurerm_network_interface.example.id,
+  ]
+
+  admin_ssh_key {
+    username   = "mlstackuser"
+    public_key = data.azurerm_ssh_public_key.example.public_key
+  }
+
+  os_disk {
+    caching              = "ReadWrite"
+    storage_account_type = "StandardSSD_LRS"
+  }
+
+  source_image_id = data.azurerm_image.example.id
+}

infrastructure/terraform.tf

@@ -0,0 +1,22 @@
+# defining the providers for the recipe module
+terraform {
+  required_providers {
+    azurerm = {
+      source  = "hashicorp/azurerm"
+      version = ">=3.16.0"
+    }
+  }
+
+  required_version = ">= 0.14.8"
+
+  backend "azurerm" {
+    resource_group_name  = "zenml-developers"
+    storage_account_name = "zenmlstorageaccount"
+    container_name       = "github-runner-tf"
+    key                  = "terraform.tfstate"
+  }
+}
+
+provider "azurerm" {
+  features {}
+}

infrastructure/variables.tf

@@ -0,0 +1,4 @@
+variable "github_runner_token" {
+  description = "GitHub token"
+  type        = string
+}