diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile new file mode 100644 index 000000000..e3dfd0351 --- /dev/null +++ b/.devcontainer/Dockerfile @@ -0,0 +1,20 @@ +# See here for image contents: https://github.com/devcontainers/images/tree/main/src/go + +# [Choice] Go version (use -bullseye variants on local arm64/Apple Silicon): 1, 1.16, 1.17, 1-bullseye, 1.16-bullseye, 1.17-bullseye, 1-buster, 1.16-buster, 1.17-buster +ARG VARIANT="1.18-bullseye" +FROM mcr.microsoft.com/vscode/devcontainers/go:1-${VARIANT} + +# [Optional] Uncomment this section to install additional OS packages. +# graphviz for pprof +RUN apt-get update && export DEBIAN_FRONTEND=noninteractive \ + && apt-get -y install --no-install-recommends graphviz + +RUN curl -Lo skaffold https://storage.googleapis.com/skaffold/releases/v2.3.1/skaffold-linux-amd64 && \ + install skaffold /usr/local/bin/ + +# [Optional] Uncomment the next lines to use go get to install anything else you need +USER vscode +RUN go install github.com/google/pprof@latest + +# [Optional] Uncomment this line to install global node packages. +# RUN su vscode -c "source /usr/local/share/nvm/nvm.sh && npm install -g " 2>&1 diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json new file mode 100644 index 000000000..5306e6e15 --- /dev/null +++ b/.devcontainer/devcontainer.json @@ -0,0 +1,66 @@ +// For format details, see https://aka.ms/devcontainer.json. For config options, see the README at: +// https://github.com/devcontainers/images/tree/main/src/go +{ + "name": "Go-karpenter", + "build": { + "dockerfile": "Dockerfile", + "args": { + "VARIANT": "1.21-bullseye", + } + }, + "runArgs": [ "--cap-add=SYS_PTRACE", "--security-opt", "seccomp=unconfined" ], + + // Configure tool-specific properties. + "customizations": { + // Configure properties specific to VS Code. + "vscode": { + // Set *default* container specific settings.json values on container create. + "settings": { + "go.toolsManagement.checkForUpdates": "local", + "go.useLanguageServer": true, + "go.gopath": "/go" + }, + + // Add the IDs of extensions you want installed when the container is created. + "extensions": [ + "golang.Go", + "ms-kubernetes-tools.vscode-kubernetes-tools", + "ms-kubernetes-tools.vscode-aks-tools", + "timonwong.shellcheck", + "mutantdino.resourcemonitor", + "Gruntfuggly.todo-tree", + "ms-azuretools.vscode-bicep", + "GitHub.vscode-pull-request-github", + "jinliming2.vscode-go-template" + ] + } + }, + + "containerEnv": { + "KUBEBUILDER_ASSETS": "/home/vscode/.kubebuilder/bin", + }, + + // Use 'forwardPorts' to make a list of ports inside the container available locally. + // "forwardPorts": [], + + // Use 'postCreateCommand' to run commands after the container is created. + "postCreateCommand": { + "disable metrics": "skaffold config set --global collect-metrics false", + "make toolchain": "make toolchain" + }, + + // Comment out to connect as root instead. More info: https://aka.ms/vscode-remote/containers/non-root. + "remoteUser": "vscode", + + "features": { + "ghcr.io/devcontainers/features/docker-outside-of-docker:1": { + "version": "latest" + }, + "ghcr.io/devcontainers/features/kubectl-helm-minikube:1": { + "version": "latest" + }, + "ghcr.io/devcontainers/features/azure-cli:1": { + "version": "latest" + } + } +} diff --git a/.github/ISSUE_TEMPLATE/bug.yaml b/.github/ISSUE_TEMPLATE/bug.yaml new file mode 100644 index 000000000..82e38deda --- /dev/null +++ b/.github/ISSUE_TEMPLATE/bug.yaml @@ -0,0 +1,68 @@ +name: Bug Report +description: Report a bug in AKS Karpenter provider +labels: bug +body: + - type: textarea + id: version + attributes: + label: Version + description: | + https://github.com/Azure/karpenter/releases + value: | + + __Karpenter Version:__ v0.0.0 + + __Kubernetes Version:__ v1.0.0 + validations: + required: true + + - type: textarea + id: expected + attributes: + label: Expected Behavior + description: | + Briefly describe what you expected to happen + validations: + required: true + + - type: textarea + id: actual + attributes: + label: Actual Behavior + description: | + Briefly describe what is actually happening + validations: + required: true + + - type: textarea + id: steps + attributes: + label: Steps to Reproduce the Problem + description: | + How can a maintainer reproduce this issue (be detailed)? + validations: + required: true + + - type: textarea + id: specs-logs + attributes: + label: Resource Specs and Logs + description: | + Include Provisioner spec(s), pod spec(s), and Karpenter controller logs when you experienced the bug + + Tip: You can attach images or log files by clicking this area to highlight it and then dragging files in. + validations: + required: true + + - type: textarea + id: community-note + attributes: + label: Community Note + description: | + Please keep this note for the community + value: | + * Please vote on this issue by adding a 👍 [reaction](https://blog.github.com/2016-03-10-add-reactions-to-pull-requests-issues-and-comments/) to the original issue to help the community and maintainers prioritize this request + * Please do not leave "+1" or "me too" comments, they generate extra noise for issue followers and do not help prioritize the request + * If you are interested in working on this issue or have submitted a pull request, please leave a comment + validations: + required: true diff --git a/.github/ISSUE_TEMPLATE/docs-request.yaml b/.github/ISSUE_TEMPLATE/docs-request.yaml new file mode 100644 index 000000000..8b2b70a0a --- /dev/null +++ b/.github/ISSUE_TEMPLATE/docs-request.yaml @@ -0,0 +1,36 @@ +name: Docs Request/Question +description: All things related to docs! Bugs, requests, suggestions, etc. +labels: Documentation +body: + - type: input + id: existing-page + attributes: + label: Is an existing page relevant? + placeholder: ex. https://karpenter.sh/v0.16.1/upgrade-guide/ + + - type: textarea + id: relevant-features + attributes: + label: What karpenter features are relevant? + validations: + required: true + + - type: textarea + id: docs-improvement + attributes: + label: How should the docs be improved? + validations: + required: true + + - type: textarea + id: community-note + attributes: + label: Community Note + description: | + Please keep this note for the community + value: | + * Please vote on this issue by adding a 👍 [reaction](https://blog.github.com/2016-03-10-add-reactions-to-pull-requests-issues-and-comments/) to the original issue to help the community and maintainers prioritize this request + * Please do not leave "+1" or "me too" comments, they generate extra noise for issue followers and do not help prioritize the request + * If you are interested in working on this issue or have submitted a pull request, please leave a comment + validations: + required: true \ No newline at end of file diff --git a/.github/ISSUE_TEMPLATE/request.yaml b/.github/ISSUE_TEMPLATE/request.yaml new file mode 100644 index 000000000..f84138a84 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/request.yaml @@ -0,0 +1,59 @@ +name: Request +description: Suggest an idea for the roadmap! +labels: feature +body: + - type: textarea + id: feature + attributes: + label: Tell us about your request + description: | + What do you want us to build? + validations: + required: true + + - type: textarea + id: rationale + attributes: + label: Tell us about the problem you're trying to solve. What are you trying to do, and why is it hard? + description: | + What outcome are you trying to achieve, ultimately, and why is it hard/impossible to do right now? What is the impact of not having this problem solved? The more details you can provide, the better we'll be able to understand and solve the problem. + validations: + required: true + + - type: textarea + id: workaround + attributes: + label: Are you currently working around this issue? + description: | + How are you currently solving this problem? + validations: + required: true + + - type: textarea + id: additional + attributes: + label: Additional Context + description: | + Anything else we should know? + + - type: textarea + id: attachments + attributes: + label: Attachments + description: | + If you think you might have additional information that you'd like to include via an attachment, please do - we'll take a look. (Remember to remove any personally-identifiable information.) + + Tip: You can attach images or log files by clicking this area to highlight it and then dragging files in. + + - type: textarea + id: community-note + attributes: + label: Community Note + description: | + Please keep this note for the community + value: | + * Please vote on this issue by adding a 👍 [reaction](https://blog.github.com/2016-03-10-add-reactions-to-pull-requests-issues-and-comments/) to the original issue to help the community and maintainers prioritize this request + * Please do not leave "+1" or "me too" comments, they generate extra noise for issue followers and do not help prioritize the request + * If you are interested in working on this issue or have submitted a pull request, please leave a comment + validations: + required: true \ No newline at end of file diff --git a/.github/actions/e2e/cleanup/action.yaml b/.github/actions/e2e/cleanup/action.yaml new file mode 100644 index 000000000..3220a252b --- /dev/null +++ b/.github/actions/e2e/cleanup/action.yaml @@ -0,0 +1,46 @@ +name: Cleanup +description: 'Runs all the cleanup tasks to cleanup resources deployed during E2E' +inputs: + client-id: + description: + required: true + tenant-id: + description: + required: true + subscription-id: + description: + required: true + # region: + # description: "Region to create aks cluster" + # required: true + resource_group: + description: "Name of the resource group to create the cluster within" + required: true + cluster_name: + description: 'Name of the cluster to be created' + required: true + acr_name: + description: "Name of the acr holding the karpenter image" + required: true +runs: + using: "composite" + steps: + - uses: actions/checkout@v3 + - name: az login + uses: azure/login@v1 + with: + client-id: ${{ inputs.client-id }} + tenant-id: ${{ inputs.tenant-id }} + subscription-id: ${{ inputs.subscription-id }} + - name: az set sub + shell: bash + run: az account set --subscription ${{ inputs.subscription-id }} + - name: delete cluster ${{ inputs.cluster_name }} + shell: bash + run: az aks delete --name ${{ inputs.cluster_name }} --resource-group ${{ inputs.resource_group }} --yes + - name: delete acr ${{ inputs.acr_name }} + shell: bash + run: az acr delete --name ${{ inputs.acr_name }} --resource-group ${{ inputs.resource_group }} --yes + - name: delete rg ${{ inputs.resource_group }} + shell: bash + run: az group delete --name ${{ inputs.resource_group }} --yes diff --git a/.github/actions/e2e/create-acr/action.yaml b/.github/actions/e2e/create-acr/action.yaml new file mode 100644 index 000000000..d61c7fd0c --- /dev/null +++ b/.github/actions/e2e/create-acr/action.yaml @@ -0,0 +1,34 @@ +name: CreateACR +description: 'Creates ACR' +inputs: + client-id: + description: + required: true + tenant-id: + description: + required: true + subscription-id: + description: + required: true + resource_group: + description: "Name of the resource group to create the cluster within" + required: true + acr_name: + description: "Name of the acr holding the karpenter image" + required: true +runs: + using: "composite" + steps: + - uses: actions/checkout@v3 + - name: az login + uses: azure/login@v1 + with: + client-id: ${{ inputs.client-id }} + tenant-id: ${{ inputs.tenant-id }} + subscription-id: ${{ inputs.subscription-id }} + - name: az set sub + shell: bash + run: az account set --subscription ${{ inputs.subscription-id }} + - name: create ACR + shell: bash + run: AZURE_RESOURCE_GROUP=${{ inputs.resource_group }} AZURE_ACR_NAME=${{ inputs.acr_name }} make az-mkacr diff --git a/.github/actions/e2e/create-cluster/action.yaml b/.github/actions/e2e/create-cluster/action.yaml new file mode 100644 index 000000000..4a28e1b0c --- /dev/null +++ b/.github/actions/e2e/create-cluster/action.yaml @@ -0,0 +1,53 @@ +name: CreateCluster +description: 'Installs Go Downloads and installs Karpenter Dependencies' +inputs: +# k8s_version: +# description: 'Version of Kubernetes to use for the launched cluster' +# required: false +# default: "1.27" + client-id: + description: + required: true + tenant-id: + description: + required: true + subscription-id: + description: + required: true + # region: + # description: "Region to create aks cluster" + # required: true + resource_group: + description: "Name of the resource group to create the cluster within" + required: true + cluster_name: + description: 'Name of the cluster to be created' + required: true + acr_name: + description: "Name of the acr holding the karpenter image" + required: true +runs: + using: "composite" + steps: + - uses: actions/checkout@v3 + - name: az login + uses: azure/login@v1 + with: + client-id: ${{ inputs.client-id }} + tenant-id: ${{ inputs.tenant-id }} + subscription-id: ${{ inputs.subscription-id }} + - name: az set sub + shell: bash + run: az account set --subscription ${{ inputs.subscription-id }} + - name: create cluster + shell: bash + run: AZURE_CLUSTER_NAME=${{ inputs.cluster_name }} AZURE_RESOURCE_GROUP=${{ inputs.resource_group }} AZURE_ACR_NAME=${{ inputs.acr_name }} make az-mkaks-cilium + - name: az login 2 + uses: azure/login@v1 + with: + client-id: ${{ inputs.client-id }} + tenant-id: ${{ inputs.tenant-id }} + subscription-id: ${{ inputs.subscription-id }} + - name: update azure perms + shell: bash + run: AZURE_CLUSTER_NAME=${{ inputs.cluster_name }} AZURE_RESOURCE_GROUP=${{ inputs.resource_group }} make az-perm diff --git a/.github/actions/e2e/dump-logs/action.yaml b/.github/actions/e2e/dump-logs/action.yaml new file mode 100644 index 000000000..64ffe13f6 --- /dev/null +++ b/.github/actions/e2e/dump-logs/action.yaml @@ -0,0 +1,51 @@ +name: DumpLogs +description: 'Dump logs and debug information from the cluster after a test run' +inputs: + client-id: + description: + required: true + tenant-id: + description: + required: true + subscription-id: + description: + required: true + resource_group: + description: "Name of the resource group for the cluster" + required: true + cluster_name: + description: 'Name of the cluster' + required: true +runs: + using: "composite" + steps: + - uses: actions/checkout@v3 + - name: az login + uses: azure/login@v1 + with: + client-id: ${{ inputs.client-id }} + tenant-id: ${{ inputs.tenant-id }} + subscription-id: ${{ inputs.subscription-id }} + - name: az set sub + shell: bash + run: az account set --subscription ${{ inputs.subscription-id }} + - name: controller-logs + shell: bash + run: | + echo "step: controller-logs" + AZURE_CLUSTER_NAME=${{ inputs.cluster_name }} AZURE_RESOURCE_GROUP=${{ inputs.resource_group }} make az-creds + POD_NAME=$(kubectl get pods -n karpenter --no-headers -o custom-columns=":metadata.name" | tail -n 1) + echo "logs from pod ${POD_NAME}" + kubectl logs "${POD_NAME}" -n karpenter -c controller + - name: describe-karpenter-pods + shell: bash + run: | + echo "step: describe-karpenter-pods" + AZURE_CLUSTER_NAME=${{ inputs.cluster_name }} AZURE_RESOURCE_GROUP=${{ inputs.resource_group }} make az-creds + kubectl describe pods -n karpenter + - name: describe-nodes + shell: bash + run: | + echo "step: describe-nodes" + AZURE_CLUSTER_NAME=${{ inputs.cluster_name }} AZURE_RESOURCE_GROUP=${{ inputs.resource_group }} make az-creds + kubectl describe nodes \ No newline at end of file diff --git a/.github/actions/e2e/install-helm/action.yaml b/.github/actions/e2e/install-helm/action.yaml new file mode 100644 index 000000000..d369c8bac --- /dev/null +++ b/.github/actions/e2e/install-helm/action.yaml @@ -0,0 +1,15 @@ +name: InstallHelm +description: 'Installs helm' +runs: + using: "composite" + steps: + - name: install helm + shell: bash + run: | + curl -fsSL -o get_helm.sh https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 + chmod 700 get_helm.sh + ./get_helm.sh + - name: install helm-diff + shell: bash + run: | + helm plugin install https://github.com/databus23/helm-diff || true \ No newline at end of file diff --git a/.github/actions/e2e/install-karpenter/action.yaml b/.github/actions/e2e/install-karpenter/action.yaml new file mode 100644 index 000000000..2a122f93c --- /dev/null +++ b/.github/actions/e2e/install-karpenter/action.yaml @@ -0,0 +1,52 @@ +name: InstallKarpenter +description: 'Installs Karpenter on the aks cluster' +inputs: + client-id: + description: + required: true + tenant-id: + description: + required: true + subscription-id: + description: + required: true + # region: + # description: "Region to create aks cluster" + # required: true + resource_group: + description: "Name of the resource group to create the cluster within" + required: true + cluster_name: + description: 'Name of the cluster to be created' + required: true + acr_name: + description: "Name of the acr holding the karpenter image" + required: true +runs: + using: "composite" + steps: + - uses: actions/checkout@v3 + - name: az login + uses: azure/login@v1 + with: + client-id: ${{ inputs.client-id }} + tenant-id: ${{ inputs.tenant-id }} + subscription-id: ${{ inputs.subscription-id }} + - name: az set sub + shell: bash + run: az account set --subscription ${{ inputs.subscription-id }} + - name: patch skaffold and cilium + shell: bash + run: AZURE_CLUSTER_NAME=${{ inputs.cluster_name }} AZURE_RESOURCE_GROUP=${{ inputs.resource_group }} make az-patch-skaffold-azureoverlay + - name: deploy karpenter to cluster + shell: bash + run: AZURE_ACR_NAME=${{ inputs.acr_name }} make az-run + - name: wait for pods to settle + shell: bash + run: | + ${GITHUB_WORKSPACE}/.github/actions/e2e/install-karpenter/waitforgatekeeper.sh + sleep 10 + - name: taint nodes as karpenter-system + shell: bash + run: | + kubectl taint nodes CriticalAddonsOnly=true:NoSchedule --all diff --git a/.github/actions/e2e/install-karpenter/waitforgatekeeper.sh b/.github/actions/e2e/install-karpenter/waitforgatekeeper.sh new file mode 100755 index 000000000..ae888fde1 --- /dev/null +++ b/.github/actions/e2e/install-karpenter/waitforgatekeeper.sh @@ -0,0 +1,44 @@ +#!/usr/bin/env bash + +# wait for the namespace to exist +for i in $(seq 1 60); +do +namespaceoutput=$(kubectl get namespaces -A | grep gatekeeper-system) +if [ "$namespaceoutput" == "" ]; then + echo "no namespace for gatekeeper-system yet"; + sleep 10; +else + echo "namespace found" + break; +fi +done + +sleep 1 + +# wait for a pod to exist +for j in $(seq 1 60); +do +namespaceoutput=$(kubectl get po -n gatekeeper-system --no-headers) +if [ "$namespaceoutput" == "" ]; then + echo "waiting on gatekeeper-system a pod to exist"; + sleep 1; +else + echo "pod found" + break; +fi +done + +sleep 1 + +# waiting for pods to be ready +for k in $(seq 1 60); +do +namespaceoutput=$(kubectl get po -n gatekeeper-system --no-headers | grep -v Running | grep -v Completed) +if [ "$namespaceoutput" != "" ]; then + echo "waiting on gatekeeper-system pods to be ready"; + sleep 1; +else + echo "pods ready/completed" + break; +fi +done diff --git a/.github/actions/install-deps/action.yml b/.github/actions/install-deps/action.yml new file mode 100644 index 000000000..9114044cc --- /dev/null +++ b/.github/actions/install-deps/action.yml @@ -0,0 +1,24 @@ +name: InstallDependencies +description: 'Installs Go Downloads and installs Karpenter Dependencies' +inputs: + k8sVersion: + description: Kubernetes version to use when installing the toolchain + default: "1.27.x" +runs: + using: "composite" + steps: + - uses: actions/setup-go@v4 + with: + go-version-file: go.mod + check-latest: true + cache-dependency-path: "**/go.sum" + - uses: actions/cache@v3 + id: cache-toolchain + with: + path: | + ~/.kubebuilder/bin + ~/go/bin + key: ${{ runner.os }}-${{ inputs.k8sVersion }}-toolchain-cache-${{ hashFiles('hack/toolchain.sh') }} + - if: ${{ steps.cache-toolchain.outputs.cache-hit != 'true' }} + shell: bash + run: K8S_VERSION=${{ inputs.k8sVersion }} make toolchain diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 000000000..c63fc8e03 --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,27 @@ +# See https://docs.github.com/en/github/administering-a-repository/configuration-options-for-dependency-updates#package-ecosystem +version: 2 +updates: + - package-ecosystem: "gomod" + directory: "/" + schedule: + interval: "monthly" + # Interdependent + ignore: + - dependency-name: "k8s.io/api" + - dependency-name: "k8s.io/apimachinery" + - dependency-name: "k8s.io/client-go" + - dependency-name: "sigs.k8s.io/controller-runtime" + - package-ecosystem: "gomod" + directory: "/test" + schedule: + interval: "monthly" + # Interdependent + ignore: + - dependency-name: "k8s.io/api" + - dependency-name: "k8s.io/apimachinery" + - dependency-name: "k8s.io/client-go" + - dependency-name: "sigs.k8s.io/controller-runtime" + - package-ecosystem: "github-actions" + directory: "/.github" + schedule: + interval: "monthly" diff --git a/.github/pull-request-template.md b/.github/pull-request-template.md new file mode 100644 index 000000000..8d2ccf77f --- /dev/null +++ b/.github/pull-request-template.md @@ -0,0 +1,36 @@ + + + + +Fixes # + +**Description** + +**How was this change tested?** + +* + +**Does this change impact docs?** +- [ ] Yes, PR includes docs updates +- [ ] Yes, issue opened: # +- [ ] No + +**Release Note** + + + +```release-note + +``` diff --git a/.github/workflows/README.md b/.github/workflows/README.md new file mode 100644 index 000000000..aca279915 --- /dev/null +++ b/.github/workflows/README.md @@ -0,0 +1,15 @@ +# workflows + +## E2E + +### Making a new E2E test +1. Create your new E2E testing suite `` within the `test/suites/` package. See: `test/README.md` +2. Update the `workflows/e2e-matrix.yaml` workflow to include your E2E test case on line 24: `suite: [Nonbehavioral, Utilization]`. + - Add in the name of your folder within the `test/suites/` package to the comma seperated list. Casing does not matter. + +### Running the test case +1. Make a draft PR +2. Anytime you want to run the E2E test suite submit a review comment `/test` +3. Each time a the given review comment is submited it will trigger the [E2EMatrixTrigger](https://github.com/Azure/karpenter/actions/workflows/e2e-matrix-trigger.yaml) workflow which will contain your test suite. + + \ No newline at end of file diff --git a/.github/workflows/build-publish-mcr.yml b/.github/workflows/build-publish-mcr.yml new file mode 100644 index 000000000..02f5ecbc9 --- /dev/null +++ b/.github/workflows/build-publish-mcr.yml @@ -0,0 +1,71 @@ +# This Github action builds and publishes image to Azure Container Registry (ACR), +# from where it is automatically propagated to Microsoft Container Registry (MCR). + +name: Build and publish to MCR +on: + workflow_dispatch: + inputs: + releaseTag: + description: 'Release tag to publish, defaults to the latest one' + type: string + +permissions: + id-token: write # This is required for requesting the JWT + contents: read # This is required for actions/checkout + +env: + REGISTRY_REPO: unlisted/aks/karpenter + +jobs: + prepare-variables: + runs-on: ubuntu-latest + outputs: + release_tag: ${{ steps.vars.outputs.release_tag }} + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + - name: 'Set output variables' + id: vars + run: | + RELEASE_TAG=${{ inputs.releaseTag }} + if [ -z "$RELEASE_TAG" ]; then + RELEASE_TAG=`git describe --tags $(git rev-list --tags --max-count=1)` + echo "The user input release tag is empty, will use the latest tag $RELEASE_TAG." + fi + echo "release_tag=$RELEASE_TAG" >> $GITHUB_OUTPUT + + publish-images: + runs-on: ubuntu-latest + needs: prepare-variables + steps: + - uses: actions/checkout@v4 + with: + ref: ${{ needs.prepare-variables.outputs.release_tag }} + + - uses: ./.github/actions/install-deps + + # reference: https://docs.github.com/en/actions/deployment/security-hardening-your-deployments/configuring-openid-connect-in-azure + - name: Login to Azure + uses: azure/login@v1 + with: + client-id: ${{ secrets.AZURE_CLIENT_ID }} + tenant-id: ${{ secrets.AZURE_TENANT_ID }} + subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }} + + - name: Build and publish image + env: + KO_DOCKER_REPO: ${{ secrets.AZURE_REGISTRY }}/${{ env.REGISTRY_REPO }} + run: | + az acr login -n ${{ secrets.AZURE_REGISTRY }} + cat >.ko.yaml <> $GITHUB_OUTPUT + e2e: + needs: [initialize-generative-params] + strategy: + fail-fast: false + matrix: + suite: [Nonbehavioral, Utilization, GPU, Drift, Integration] + uses: ./.github/workflows/e2e.yaml + with: + suite: ${{ matrix.suite }} + hash: ${{ needs.initialize-generative-params.outputs.E2E_HASH }} + # region: ${{ inputs.region }} + # k8s_version: ${{ inputs.k8s_version }} + secrets: + E2E_CLIENT_ID: ${{ secrets.E2E_CLIENT_ID }} + E2E_TENANT_ID: ${{ secrets.E2E_TENANT_ID }} + E2E_SUBSCRIPTION_ID: ${{ secrets.E2E_SUBSCRIPTION_ID }} diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml new file mode 100644 index 000000000..d8c26a52a --- /dev/null +++ b/.github/workflows/e2e.yaml @@ -0,0 +1,122 @@ +name: E2E +on: + workflow_call: + inputs: + # region: + # type: string + # default: "us-east-2" + suite: + type: string + required: true + hash: + type: string + required: true + # k8s_version: + # type: string + # default: "1.27" + secrets: + E2E_CLIENT_ID: + required: true + E2E_TENANT_ID: + required: true + E2E_SUBSCRIPTION_ID: + required: true +permissions: + id-token: write # This is required for requesting the JWT + contents: read # This is required for actions/checkout +jobs: + run-suite: + name: suite-${{ inputs.suite }} + runs-on: ubuntu-latest + env: + AZURE_SUBSCRIPTION_ID: ${{ secrets.E2E_SUBSCRIPTION_ID }} + steps: + - uses: actions/checkout@v3 + - uses: ./.github/actions/install-deps + - name: az login + uses: azure/login@v1 + with: + client-id: ${{ secrets.E2E_CLIENT_ID }} + tenant-id: ${{ secrets.E2E_TENANT_ID }} + subscription-id: ${{ secrets.E2E_SUBSCRIPTION_ID }} + - name: az set sub + shell: bash + run: az account set --subscription ${{ secrets.E2E_SUBSCRIPTION_ID }} + - name: install skaffold + run: | + curl -Lo skaffold https://storage.googleapis.com/skaffold/releases/v2.3.1/skaffold-linux-amd64 + install skaffold /usr/local/bin/ + - name: generate rg name + run: | + RG_NAME=$(echo karpenter-e2e-${{ inputs.suite }}-rg-${{ inputs.hash }} | awk '{print tolower($0)}') + echo "Using resource group name \"$RG_NAME\"" + echo RG_NAME=$RG_NAME >> $GITHUB_ENV + - name: generate acr name + run: | + ACR_NAME=$(echo ${{ inputs.suite }}acr${{ inputs.hash }} | awk '{print tolower($0)}') + echo "Using acr name \"$ACR_NAME\"" + echo ACR_NAME=$ACR_NAME >> $GITHUB_ENV + - name: create acr '${{ env.ACR_NAME }}' + uses: ./.github/actions/e2e/create-acr + with: + client-id: ${{ secrets.E2E_CLIENT_ID }} + tenant-id: ${{ secrets.E2E_TENANT_ID }} + subscription-id: ${{ secrets.E2E_SUBSCRIPTION_ID }} + resource_group: ${{ env.RG_NAME }} + acr_name: ${{ env.ACR_NAME }} + - name: add jitter on cluster creation + run: | + # Creating jitter so that we can stagger cluster creation to avoid throttling + sleep $(( $RANDOM % 60 + 1 )) + - name: generate cluster name + run: | + CLUSTER_NAME=$(echo ${{ inputs.suite }}-mc-${{ inputs.hash }} | awk '{print tolower($0)}') + echo "Using cluster name \"$CLUSTER_NAME\"" + echo CLUSTER_NAME=$CLUSTER_NAME >> $GITHUB_ENV + - name: create aks cluster '${{ env.CLUSTER_NAME }}' + uses: ./.github/actions/e2e/create-cluster + with: + client-id: ${{ secrets.E2E_CLIENT_ID }} + tenant-id: ${{ secrets.E2E_TENANT_ID }} + subscription-id: ${{ secrets.E2E_SUBSCRIPTION_ID }} + # region: westus2 + resource_group: ${{ env.RG_NAME }} + cluster_name: ${{ env.CLUSTER_NAME }} + acr_name: ${{ env.ACR_NAME }} + - name: build and publish karpenter + shell: bash + run: AZURE_ACR_NAME=${{ env.ACR_NAME }} make az-build + - name: install karpenter + uses: ./.github/actions/e2e/install-karpenter + with: + client-id: ${{ secrets.E2E_CLIENT_ID }} + tenant-id: ${{ secrets.E2E_TENANT_ID }} + subscription-id: ${{ secrets.E2E_SUBSCRIPTION_ID }} + resource_group: ${{ env.RG_NAME }} + cluster_name: ${{ env.CLUSTER_NAME }} + acr_name: ${{ env.ACR_NAME }} + - name: run the ${{ inputs.suite }} test suite + if: inputs.suite != 'Nonbehavioral' + run: | + AZURE_CLUSTER_NAME=${{ env.CLUSTER_NAME }} AZURE_RESOURCE_GROUP=${{ env.RG_NAME }} make az-creds + CLUSTER_NAME=${{ env.CLUSTER_NAME }} TEST_SUITE="${{ inputs.suite }}" GIT_REF="$(git rev-parse HEAD)" make e2etests + - name: dump logs on failure + uses: ./.github/actions/e2e/dump-logs + if: failure() || cancelled() + with: + client-id: ${{ secrets.E2E_CLIENT_ID }} + tenant-id: ${{ secrets.E2E_TENANT_ID }} + subscription-id: ${{ secrets.E2E_SUBSCRIPTION_ID }} + resource_group: ${{ env.RG_NAME }} + cluster_name: ${{ env.CLUSTER_NAME }} + - name: cleanup resources + uses: ./.github/actions/e2e/cleanup + if: always() + with: + client-id: ${{ secrets.E2E_CLIENT_ID }} + tenant-id: ${{ secrets.E2E_TENANT_ID }} + subscription-id: ${{ secrets.E2E_SUBSCRIPTION_ID }} + # region: westus2 + resource_group: ${{ env.RG_NAME }} + cluster_name: ${{ env.CLUSTER_NAME }} + acr_name: ${{ env.ACR_NAME }} diff --git a/.gitignore b/.gitignore new file mode 100644 index 000000000..f88fe4539 --- /dev/null +++ b/.gitignore @@ -0,0 +1,22 @@ +# Go toolchain +coverage.out +coverage.html +*.test +*.cpuprofile +*.heapprofile +go.work +go.work.sum + +# Environment +.idea +# .vscode - do check in to share settings +.DS_Store + +# Go Benchmarking +*.cpuprofile +*.heapprofile + +# Temp perf logs +/logs +# Project Specific +*.csv diff --git a/.golangci.yaml b/.golangci.yaml new file mode 100644 index 000000000..0d0fff5bd --- /dev/null +++ b/.golangci.yaml @@ -0,0 +1,85 @@ +# See https://github.com/golangci/golangci-lint/blob/master/.golangci.example.yml +run: + tests: true + allow-parallel-runners: true + timeout: 15m + skip-dirs: + - tools + - hack + - charts + - designs + +linters: + enable: + - asciicheck + - bidichk + - errorlint + - errcheck + - exportloopref + - gosec + - revive + - stylecheck + - tparallel + - unconvert + - unparam + - gocyclo + - whitespace + - govet + - goimports + - goheader + - misspell + - nilerr + disable: + - prealloc + +linters-settings: + dupl: + threshold: 100 + gocyclo: + min-complexity: 11 + govet: + check-shadowing: true + revive: + rules: + - name: dot-imports + disabled: true + stylecheck: + dot-import-whitelist: + - "github.com/onsi/ginkgo/v2" + - "github.com/onsi/gomega" + misspell: + locale: US + ignore-words: [karpenter] + goimports: + local-prefixes: github.com/azure/karpenter + goheader: + template: |- + Portions Copyright (c) Microsoft Corporation. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +issues: + fix: true + exclude: ['declaration of "(err|ctx)" shadows declaration at'] + exclude-rules: + - linters: + - goheader + path: 'zz_(.+)\.go' + - linters: + - goheader + source: "^//go:build" + +service: + golangci-lint-version: 1.50.x # use the fixed version to not introduce new linters unexpectedly + prepare: + - echo "here I can run custom commands, but no preparation needed for this repo" diff --git a/.vscode/launch.json b/.vscode/launch.json new file mode 100644 index 000000000..2d6a2a2c0 --- /dev/null +++ b/.vscode/launch.json @@ -0,0 +1,25 @@ +{ + // Use IntelliSense to learn about possible attributes. + // Hover to view descriptions of existing attributes. + // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387 + "version": "0.2.0", + "configurations": [ + { + "name": "scaffold debug 56268", + "type": "go", + "request": "attach", + "debugAdapter": "dlv-dap", + "mode": "remote", + "host": "127.0.0.1", + "cwd": "${workspaceFolder}", + // observed variation in mapping ("dlv sources" in DEBUG CONSOLE), reason unknown for now; + // sometimes the below is needed, and somtimes not + // "substitutePath": [ + // {"from": "${workspaceFolder}", "to": "github.com/Azure/karpenter/"} + // {"from": "/go/pkg/mod/", "to": ""} // source mapping for external modules, if needed + //], + "port": 56268, + "trace": "verbose" + }, + ] +} diff --git a/.vscode/settings.json b/.vscode/settings.json new file mode 100644 index 000000000..9b96b1f80 --- /dev/null +++ b/.vscode/settings.json @@ -0,0 +1,31 @@ +{ + "go.testTimeout": "120s", + "gopls": { + "build.directoryFilters": [ + "-hack" + ] + }, + "githubPullRequests": { + "remotes": [ + "origin" + ], + "queries": [ + { + "label": "Waiting For My Review", + "query": "is:open review-requested:${user}" + }, + { + "label": "Assigned To Me", + "query": "is:open assignee:${user}" + }, + { + "label": "Created By Me", + "query": "is:open author:${user}" + }, + { + "label": "All Open (-dependabot)", + "query": "is:open -label:dependencies" + } + ] + } +} \ No newline at end of file diff --git a/ADOPTERS.md b/ADOPTERS.md new file mode 100644 index 000000000..92f036512 --- /dev/null +++ b/ADOPTERS.md @@ -0,0 +1,16 @@ +# Who is using Karpenter? +Karpenter has a variety of users and use cases for scaling Kubernetes. +Many customers want to learn from others who have already implemented Karpenter in their environments. + +The following is a self-reported list of users to help identify adoption and points of contact. + +## Community +If you would like to ask question from the community please join the [Karpenter slack channel in the Kubernetes Slack](https://kubernetes.slack.com/archives/C02SFFZSA2K) or join the [Karpenter working group](https://karpenter.sh/docs/contributing/working-group/) bi-weekly calls. + +## Add yourself +Please add yourself if you or your organization is leveraging azure karpenter! +## Adopters (Alphabetical) + +| Organization | Description | Contacts | Link | +| --- | --- | --- | --- | +| Microsoft | Scaling production workloads and batch jobs in all Azure Workloads | `@Alex Leites`, `@Bryce-Soghigian`, `@Charlie McBride` | [Introducing Karpenter](TBD) diff --git a/CODE_OF_CONDUCT.md b/CODE_OF_CONDUCT.md new file mode 100644 index 000000000..b56544768 --- /dev/null +++ b/CODE_OF_CONDUCT.md @@ -0,0 +1,9 @@ +# Code of Conduct +The Karpenter project follows the [CNCF Community Code of Conduct](https://github.com/cncf/foundation/blob/master/code-of-conduct.md) +and the Microsoft Code of Conduct [Microsoft Open Source Code of Conduct](https://opensource.microsoft.com/codeofconduct/). + +Resources: + +- [Microsoft Open Source Code of Conduct](https://opensource.microsoft.com/codeofconduct/) +- [Microsoft Code of Conduct FAQ](https://opensource.microsoft.com/codeofconduct/faq/) +- Contact [opencode@microsoft.com](mailto:opencode@microsoft.com) with questions or concerns diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 000000000..0680fa9a6 --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,7 @@ +# Contributing + +This project welcomes contributions and suggestions. Most contributions require you to agree to a Contributor License Agreement (CLA) declaring that you have the right to, and actually do, grant us the rights to use your contribution. For details, visit https://cla.microsoft.com. + +When you submit a pull request, a CLA-bot will automatically determine whether you need to provide a CLA and decorate the PR appropriately (e.g., label, comment). Simply follow the instructions provided by the bot. You will only need to do this once across all repositories using our CLA. + +This project has adopted the [Microsoft Open Source Code of Conduct](https://opensource.microsoft.com/codeofconduct/). For more information see the [Code of Conduct FAQ](https://opensource.microsoft.com/codeofconduct/faq/) or contact [opencode@microsoft.com](mailto:opencode@microsoft.com) with any additional questions or comments. diff --git a/LICENSE b/LICENSE new file mode 100644 index 000000000..d64569567 --- /dev/null +++ b/LICENSE @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/LICENSE.txt b/LICENSE.txt new file mode 100644 index 000000000..d64569567 --- /dev/null +++ b/LICENSE.txt @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/Makefile b/Makefile new file mode 100644 index 000000000..0d0cafe82 --- /dev/null +++ b/Makefile @@ -0,0 +1,110 @@ +include Makefile-az.mk + +export K8S_VERSION ?= 1.27.x +export KUBEBUILDER_ASSETS ?= ${HOME}/.kubebuilder/bin + +# # CR for local builds of Karpenter +SYSTEM_NAMESPACE ?= karpenter + +# Common Directories +# TODO: revisit testing tools (temporarily excluded here, for make verify) +MOD_DIRS = $(shell find . -name go.mod -type f ! -path "./test/*" | xargs dirname) +KARPENTER_CORE_DIR = $(shell go list -m -f '{{ .Dir }}' github.com/aws/karpenter-core) + +# TEST_SUITE enables you to select a specific test suite directory to run "make e2etests" or "make test" against +TEST_SUITE ?= "..." +TEST_TIMEOUT ?= "3h" + +help: ## Display help + @awk 'BEGIN {FS = ":.*##"; printf "Usage:\n make \033[36m\033[0m\n"} /^[a-zA-Z_0-9-]+:.*?##/ { printf " \033[36m%-15s\033[0m %s\n", $$1, $$2 } /^##@/ { printf "\n\033[1m%s\033[0m\n", substr($$0, 5) } ' $(MAKEFILE_LIST) + +presubmit: verify test ## Run all steps in the developer loop + +ci-test: battletest coverage ## Runs tests and submits coverage + +ci-non-test: verify vulncheck ## Runs checks other than tests + +test: ## Run tests + ginkgo -v --focus="${FOCUS}" ./pkg/$(shell echo $(TEST_SUITE) | tr A-Z a-z) + +battletest: ## Run randomized, racing, code-covered tests + ginkgo -v \ + -race \ + -cover -coverprofile=coverage.out -output-dir=. -coverpkg=./pkg/... \ + --focus="${FOCUS}" \ + --randomize-all \ + -tags random_test_delay \ + ./pkg/... + +e2etests: ## Run the e2e suite against your local cluster + # Notes: + # -p: the number of programs, such as build commands or test binaries, that can be run in parallel? + # -count 1: prevents caching + # -timeout: If a test binary runs longer than TEST_TIMEOUT, panic + # -v: verbose output + cd test && CLUSTER_NAME=${CLUSTER_NAME} go test \ + -p 1 \ + -count 1 \ + -timeout ${TEST_TIMEOUT} \ + -v \ + ./suites/$(shell echo $(TEST_SUITE) | tr A-Z a-z)/... \ + --ginkgo.focus="${FOCUS}" \ + --ginkgo.timeout=${TEST_TIMEOUT} \ + --ginkgo.grace-period=3m \ + --ginkgo.vv + +benchmark: + go test -tags=test_performance -run=NoTests -bench=. ./... + +deflake: ## Run randomized, racing, code-covered tests to deflake failures + for i in $(shell seq 1 5); do make battletest || exit 1; done + +deflake-until-it-fails: ## Run randomized, racing tests until the test fails to catch flakes + ginkgo \ + --race \ + --focus="${FOCUS}" \ + --randomize-all \ + --until-it-fails \ + -v \ + ./pkg/... + +coverage: + go tool cover -html coverage.out -o coverage.html + +verify: toolchain tidy download ## Verify code. Includes dependencies, linting, formatting, etc + go generate ./... + hack/boilerplate.sh + cp $(KARPENTER_CORE_DIR)/pkg/apis/crds/* pkg/apis/crds + yq -i '(.spec.versions[0].additionalPrinterColumns[] | select (.name=="Zone")) .jsonPath=".metadata.labels.karpenter\.azure\.com/zone"' \ + pkg/apis/crds/karpenter.sh_nodeclaims.yaml + $(foreach dir,$(MOD_DIRS),cd $(dir) && golangci-lint run $(newline)) + @git diff --quiet ||\ + { echo "New file modification detected in the Git working tree. Please check in before commit."; git --no-pager diff --name-only | uniq | awk '{print " - " $$0}'; \ + if [ "${CI}" = true ]; then\ + exit 1;\ + fi;} + # TODO: restore codegen if needed; decide on the future of docgen + #@echo "Validating codegen/docgen build scripts..." + #@find hack/code hack/docs -name "*.go" -type f -print0 | xargs -0 -I {} go build -o /dev/null {} + +vulncheck: ## Verify code vulnerabilities + @govulncheck ./pkg/... + +codegen: ## Auto generate files based on Azure API responses + ./hack/codegen.sh + +toolchain: ## Install developer toolchain + ./hack/toolchain.sh + +tidy: ## Recursively "go mod tidy" on all directories where go.mod exists + $(foreach dir,$(MOD_DIRS),cd $(dir) && go mod tidy $(newline)) + +download: ## Recursively "go mod download" on all directories where go.mod exists + $(foreach dir,$(MOD_DIRS),cd $(dir) && go mod download $(newline)) + +.PHONY: help test battletest e2etests verify tidy download codegen toolchain vulncheck + +define newline + + +endef diff --git a/Makefile-az.mk b/Makefile-az.mk new file mode 100755 index 000000000..8ba0d0972 --- /dev/null +++ b/Makefile-az.mk @@ -0,0 +1,285 @@ +AZURE_LOCATION ?= westus2 +ifeq ($(CODESPACES),true) + AZURE_RESOURCE_GROUP ?= $(CODESPACE_NAME) + AZURE_ACR_NAME ?= $(subst -,,$(CODESPACE_NAME)) +else + AZURE_RESOURCE_GROUP ?= karpenter + AZURE_ACR_NAME ?= karpenter +endif + +AZURE_CLUSTER_NAME ?= karpenter +AZURE_RESOURCE_GROUP_MC = MC_$(AZURE_RESOURCE_GROUP)_$(AZURE_CLUSTER_NAME)_$(AZURE_LOCATION) + +az-all: az-login az-mkaks-cilium az-perm az-patch-skaffold-azureoverlay az-build az-run az-run-sample ## Provision the infra (ACR,AKS); build and deploy Karpenter; deploy sample Provisioner and workload +az-all-savm: az-login az-mkaks-savm az-perm az-patch-skaffold-azure az-build az-run az-run-sample ## Provision the infra (ACR,AKS); build and deploy Karpenter; deploy sample Provisioner and workload - StandaloneVirtualMachines + +az-login: ## Login into Azure + az login + az account set --subscription $(AZURE_SUBSCRIPTION_ID) + +az-mkrg: ## Create resource group + az group create --name $(AZURE_RESOURCE_GROUP) --location $(AZURE_LOCATION) -o none + +az-mkacr: az-mkrg ## Create test ACR + az acr create --name $(AZURE_ACR_NAME) --resource-group $(AZURE_RESOURCE_GROUP) --sku Basic --admin-enabled -o none + az acr login --name $(AZURE_ACR_NAME) + +az-mkaks: az-mkacr ## Create test AKS cluster (with --vm-set-type AvailabilitySet for compatibility with standalone VMs) + az aks create --name $(AZURE_CLUSTER_NAME) --resource-group $(AZURE_RESOURCE_GROUP) --attach-acr $(AZURE_ACR_NAME) \ + --enable-managed-identity --node-count 3 --generate-ssh-keys --vm-set-type AvailabilitySet -o none + az aks get-credentials --name $(AZURE_CLUSTER_NAME) --resource-group $(AZURE_RESOURCE_GROUP) --overwrite-existing + skaffold config set default-repo $(AZURE_ACR_NAME).azurecr.io/karpenter + +az-mkaks-cilium: az-mkacr ## Create test AKS cluster (with --network-dataplane cilium, --network-plugin cilium, and --network-plugin-mode overlay) + az aks create --name $(AZURE_CLUSTER_NAME) --resource-group $(AZURE_RESOURCE_GROUP) --attach-acr $(AZURE_ACR_NAME) \ + --enable-managed-identity --node-count 3 --generate-ssh-keys -o none --network-dataplane cilium --network-plugin azure --network-plugin-mode overlay + az aks get-credentials --name $(AZURE_CLUSTER_NAME) --resource-group $(AZURE_RESOURCE_GROUP) --overwrite-existing + skaffold config set default-repo $(AZURE_ACR_NAME).azurecr.io/karpenter + +az-mkaks-savm: az-mkrg ## Create experimental cluster with standalone VMs (+ ACR) + az deployment group create --resource-group $(AZURE_RESOURCE_GROUP) --template-file hack/azure/aks-savm.bicep --parameters aksname=$(AZURE_CLUSTER_NAME) acrname=$(AZURE_ACR_NAME) + az aks get-credentials --resource-group $(AZURE_RESOURCE_GROUP) --name $(AZURE_CLUSTER_NAME) --overwrite-existing + skaffold config set default-repo $(AZURE_ACR_NAME).azurecr.io/karpenter + +az-rmrg: ## Destroy test ACR and AKS cluster by deleting the resource group (use with care!) + az group delete --name $(AZURE_RESOURCE_GROUP) + +az-patch-skaffold: ## Update Azure client env vars and settings in skaffold config + $(eval AZURE_CLIENT_ID=$(shell az aks show --name $(AZURE_CLUSTER_NAME) --resource-group $(AZURE_RESOURCE_GROUP) | jq -r ".identityProfile.kubeletidentity.clientId")) + $(eval CLUSTER_ENDPOINT=$(shell kubectl config view --minify -o jsonpath='{.clusters[0].cluster.server}')) + # bootstrap token + $(eval TOKEN_SECRET_NAME=$(shell kubectl get -n kube-system secrets --field-selector=type=bootstrap.kubernetes.io/token -o jsonpath='{.items[0].metadata.name}')) + $(eval TOKEN_ID=$(shell kubectl get -n kube-system secret $(TOKEN_SECRET_NAME) -o jsonpath='{.data.token-id}' | base64 -d)) + $(eval TOKEN_SECRET=$(shell kubectl get -n kube-system secret $(TOKEN_SECRET_NAME) -o jsonpath='{.data.token-secret}' | base64 -d)) + $(eval BOOTSTRAP_TOKEN=$(TOKEN_ID).$(TOKEN_SECRET)) + # ssh key + $(eval SSH_PUBLIC_KEY=$(shell cat ~/.ssh/id_rsa.pub) azureuser) + yq -i '(.manifests.helm.releases[0].overrides.controller.env[] | select(.name=="ARM_SUBSCRIPTION_ID")) .value = "$(AZURE_SUBSCRIPTION_ID)"' skaffold.yaml + yq -i '(.manifests.helm.releases[0].overrides.controller.env[] | select(.name=="LOCATION")) .value = "$(AZURE_LOCATION)"' skaffold.yaml + yq -i '(.manifests.helm.releases[0].overrides.controller.env[] | select(.name=="ARM_USER_ASSIGNED_IDENTITY_ID")) .value = "$(AZURE_CLIENT_ID)"' skaffold.yaml + yq -i '(.manifests.helm.releases[0].overrides.controller.env[] | select(.name=="AZURE_NODE_RESOURCE_GROUP")) .value = "$(AZURE_RESOURCE_GROUP_MC)"' skaffold.yaml + yq -i '.manifests.helm.releases[0].overrides.settings.azure.clusterName = "$(AZURE_CLUSTER_NAME)"' skaffold.yaml + yq -i '.manifests.helm.releases[0].overrides.settings.azure.clusterEndpoint = "$(CLUSTER_ENDPOINT)"' skaffold.yaml + yq -i '.manifests.helm.releases[0].overrides.settings.azure.networkPlugin = "azure"' skaffold.yaml + yq -i '.manifests.helm.releases[0].overrides.settings.azure.kubeletClientTLSBootstrapToken = "$(BOOTSTRAP_TOKEN)"' skaffold.yaml + yq -i '.manifests.helm.releases[0].overrides.settings.azure.sshPublicKey = "$(SSH_PUBLIC_KEY)"' skaffold.yaml + +az-patch-skaffold-kubenet: az-patch-skaffold az-fetch-network-info + $(eval AZURE_SUBNET_ID=$(shell az network vnet list --resource-group $(AZURE_RESOURCE_GROUP_MC) | jq -r ".[0].subnets[0].id")) + yq -i '(.manifests.helm.releases[0].overrides.controller.env[] | select(.name=="AZURE_SUBNET_ID")) .value = "$(AZURE_SUBNET_ID)"' skaffold.yaml + yq -i '.manifests.helm.releases[0].overrides.settings.azure.networkPlugin = "kubenet"' skaffold.yaml + +az-patch-skaffold-azure: az-patch-skaffold az-fetch-network-info + $(eval AZURE_SUBNET_ID=$(shell az aks show --name $(AZURE_CLUSTER_NAME) --resource-group $(AZURE_RESOURCE_GROUP) | jq -r ".agentPoolProfiles[0].vnetSubnetId")) + yq -i '(.manifests.helm.releases[0].overrides.controller.env[] | select(.name=="AZURE_SUBNET_ID")) .value = "$(AZURE_SUBNET_ID)"' skaffold.yaml + +az-patch-skaffold-azureoverlay: az-patch-skaffold az-fetch-network-info + $(eval AZURE_SUBNET_ID=$(shell az network vnet list --resource-group $(AZURE_RESOURCE_GROUP_MC) | jq -r ".[0].subnets[0].id")) + yq -i '(.manifests.helm.releases[0].overrides.controller.env[] | select(.name=="AZURE_SUBNET_ID")) .value = "$(AZURE_SUBNET_ID)"' skaffold.yaml + yq -i '.manifests.helm.releases[0].overrides.settings.azure.networkPlugin = "azure"' skaffold.yaml + +az-fetch-network-info: + $(eval AZURE_VNET_NAME=$(shell az network vnet list --resource-group $(AZURE_RESOURCE_GROUP_MC) | jq -r ".[0].name")) + yq -i '(.manifests.helm.releases[0].overrides.controller.env[] | select(.name=="AZURE_VNET_NAME")) .value = "$(AZURE_VNET_NAME)"' skaffold.yaml + $(eval AZURE_SUBNET_NAME=$(shell az network vnet list --resource-group $(AZURE_RESOURCE_GROUP_MC) | jq -r ".[0].subnets[0].name")) + yq -i '(.manifests.helm.releases[0].overrides.controller.env[] | select(.name=="AZURE_SUBNET_NAME")) .value = "$(AZURE_SUBNET_NAME)"' skaffold.yaml + +az-mkvmssflex: ## Create VMSS Flex (optional, only if creating VMs referencing this VMSS) + az vmss create --name $(AZURE_CLUSTER_NAME)-vmss --resource-group $(AZURE_RESOURCE_GROUP_MC) --location $(AZURE_LOCATION) \ + --instance-count 0 --orchestration-mode Flexible --platform-fault-domain-count 1 --zones 1 2 3 + +az-rmvmss-vms: ## Delete all VMs in VMSS Flex (use with care!) + az vmss delete-instances --name $(AZURE_CLUSTER_NAME)-vmss --resource-group $(AZURE_RESOURCE_GROUP_MC) --instance-ids '*' + +az-perm: ## Create role assignments to let Karpenter manage VMs and Network + # Note (charliedmcb): need to be objectId for E2E workflow as the pipeline identity doesn't have permissions to "query Graph API" + $(eval AZURE_OBJECT_ID=$(shell az aks show --name $(AZURE_CLUSTER_NAME) --resource-group $(AZURE_RESOURCE_GROUP) | jq -r ".identityProfile.kubeletidentity.objectId")) + az role assignment create --assignee $(AZURE_OBJECT_ID) --resource-group $(AZURE_RESOURCE_GROUP_MC) --role "Virtual Machine Contributor" + az role assignment create --assignee $(AZURE_OBJECT_ID) --resource-group $(AZURE_RESOURCE_GROUP_MC) --role "Network Contributor" + az role assignment create --assignee $(AZURE_OBJECT_ID) --resource-group $(AZURE_RESOURCE_GROUP_MC) --role "Managed Identity Operator" + az role assignment create --assignee $(AZURE_OBJECT_ID) --resource-group $(AZURE_RESOURCE_GROUP) --role "Network Contributor" # in some case we create vnet here + @echo Consider "make az-patch-skaffold"! + +az-perm-acr: + $(eval AZURE_CLIENT_ID=$(shell az aks show --name $(AZURE_CLUSTER_NAME) --resource-group $(AZURE_RESOURCE_GROUP) | jq -r ".identityProfile.kubeletidentity.clientId")) + $(eval AZURE_ACR_ID=$(shell az acr show --name $(AZURE_ACR_NAME) --resource-group $(AZURE_RESOURCE_GROUP) | jq -r ".id")) + az role assignment create --assignee $(AZURE_CLIENT_ID) --scope $(AZURE_ACR_ID) --role "AcrPull" + +az-aks-check-acr: + az aks check-acr --name $(AZURE_CLUSTER_NAME) --resource-group $(AZURE_RESOURCE_GROUP) --acr $(AZURE_ACR_NAME) + +az-build: ## Build the Karpenter controller and webhook images using skaffold build (which uses ko build) + az acr login -n $(AZURE_ACR_NAME) + skaffold build + +az-creds: ## Get cluster credentials + az aks get-credentials --name $(AZURE_CLUSTER_NAME) --resource-group $(AZURE_RESOURCE_GROUP) + +az-run: ## Deploy the controller from the current state of your git repository into your ~/.kube/config cluster using skaffold run + az acr login -n $(AZURE_ACR_NAME) + skaffold run + +az-run-sample: ## Deploy sample Provisioner and workload (with 0 replicas, to be scaled manually) + kubectl apply -f examples/v1beta1/general-purpose.yaml + kubectl apply -f examples/workloads/inflate.yaml + +az-mc-show: ## show managed cluster + az aks show --name $(AZURE_CLUSTER_NAME) --resource-group $(AZURE_RESOURCE_GROUP) + +az-mc-upgrade: ## upgrade managed cluster + $(eval UPGRADE_K8S_VERSION=$(shell az aks get-upgrades --name $(AZURE_CLUSTER_NAME) --resource-group $(AZURE_RESOURCE_GROUP) | jq -r ".controlPlaneProfile.upgrades[0].kubernetesVersion")) + az aks upgrade --name $(AZURE_CLUSTER_NAME) --resource-group $(AZURE_RESOURCE_GROUP) --kubernetes-version $(UPGRADE_K8S_VERSION) + +az-dev: ## Deploy and develop using skaffold dev + skaffold dev + +az-debug: ## Rebuild, deploy and debug using skaffold debug + az acr login -n $(AZURE_ACR_NAME) + skaffold delete || true + skaffold debug # --platform=linux/arm64 + +az-debug-bootstrap: ## Debug bootstrap (target first privateIP of the first NIC with Karpenter tag) + $(eval JUMP_NODE=$(shell kubectl get nodes -o name | head -n 1)) + $(eval JUMP_POD=$(shell kubectl debug $(JUMP_NODE) --image kroniak/ssh-client -- sh -c "mkdir /root/.ssh; sleep 1h" | cut -d' ' -f4)) + kubectl wait --for=condition=Ready pod/$(JUMP_POD) + kubectl cp ~/.ssh/id_rsa $(JUMP_POD):/root/.ssh/id_rsa + $(eval NODE_IP=$(shell az network nic list -g $(AZURE_RESOURCE_GROUP_MC) \ + --query '[?tags."karpenter.azure.com_cluster"]|[0].ipConfigurations[0].privateIPAddress')) + kubectl exec $(JUMP_POD) -it -- ssh -o StrictHostKeyChecking=accept-new azureuser@$(NODE_IP) + +az-cleanup: ## Delete the deployment + skaffold delete || true + +az-mon-deploy: ## Deploy monitoring stack (w/o node-exporter) + helm repo add grafana-charts https://grafana.github.io/helm-charts + helm repo add prometheus-community https://prometheus-community.github.io/helm-charts + helm repo update + kubectl create namespace monitoring || true + helm install --namespace monitoring prometheus prometheus-community/prometheus \ + --values examples/prometheus-values.yaml \ + --set nodeExporter.enabled=false + helm install --namespace monitoring grafana grafana-charts/grafana \ + --values examples/grafana-values.yaml + +az-mon-access: ## Get Grafana admin password and forward port + kubectl get secret --namespace monitoring grafana -o jsonpath="{.data.admin-password}" | base64 --decode; echo + @echo Consider running port forward outside of codespace ... + kubectl port-forward --namespace monitoring svc/grafana 3000:80 + +az-mon-cleanup: ## Delete monitoring stack + helm delete --namespace monitoring grafana + helm delete --namespace monitoring prometheus + +az-mkgohelper: ## Build and configure custom go-helper-image for skaffold + cd hack/go-helper-image; docker build . --tag $(AZURE_ACR_NAME).azurecr.io/skaffold-debug-support/go # --platform=linux/arm64 + az acr login -n $(AZURE_ACR_NAME) + docker push $(AZURE_ACR_NAME).azurecr.io/skaffold-debug-support/go + skaffold config set --global debug-helpers-registry $(AZURE_ACR_NAME).azurecr.io/skaffold-debug-support + +az-rmnodes-fin: ## Remove Karpenter finalizer from all nodes (use with care!) + for node in $$(kubectl get nodes -l karpenter.sh/nodepool --output=jsonpath={.items..metadata.name}); do \ + kubectl patch node $$node -p '{"metadata":{"finalizers":null}}'; \ + done + +az-rmnodes: ## kubectl delete all Karpenter-provisioned nodes; don't wait for finalizers (use with care!) + kubectl delete --wait=false nodes -l karpenter.sh/nodepool + # kubectl wait --for=delete nodes -l karpenter.sh/nodepool --timeout=10m + +az-rmnodeclaims-fin: ## Remove Karpenter finalizer from all nodeclaims (use with care!) + for nodeclaim in $$(kubectl get nodeclaims --output=jsonpath={.items..metadata.name}); do \ + kubectl patch nodeclaim $$nodeclaim --type=json -p '[{"op": "remove", "path": "/metadata/finalizers"}]'; \ + done + +az-rmnodeclaims: ## kubectl delete all nodeclaims; don't wait for finalizers (use with care!) + kubectl delete --wait=false nodeclaims --all + +az-e2etests: ## Run e2etests + kubectl taint nodes CriticalAddonsOnly=true:NoSchedule --all --overwrite + TEST_SUITE=Utilization make e2etests + kubectl taint nodes CriticalAddonsOnly=true:NoSchedule- --all + +az-perftest1: ## Test scaling out/in (1 VM) + hack/azure/perftest.sh 1 + +az-perftest5: ## Test scaling out/in (5 VMs) + hack/azure/perftest.sh 5 + +az-perftest20: ## Test scaling out/in (20 VMs) + hack/azure/perftest.sh 20 + +az-perftest100: ## Test scaling out/in (100 VMs) + hack/azure/perftest.sh 100 + +az-perftest300: ## Test scaling out/in (300 VMs) + hack/azure/perftest.sh 300 + +az-perftest400: ## Test scaling out/in (400 VMs) + hack/azure/perftest.sh 400 + +az-perftest500: ## Test scaling out/in (500 VMs) + hack/azure/perftest.sh 500 + +az-perftest1000: ## Test scaling out/in (1000 VMs) + hack/azure/perftest.sh 1000 + +az-resg: ## List resources in MC rg + az resource list -o table -g $(AZURE_RESOURCE_GROUP_MC) + +RESK=az resource list --tag=karpenter.sh_nodepool --query "[?resourceGroup=='$(AZURE_RESOURCE_GROUP_MC)']" +az-res: ## List resources created by Karpenter + $(RESK) -o table + +az-resc: ## Count resources created by Karpenter + $(RESK) -o tsv | wc -l + +az-rmres: ## Delete (az resource delete) all resources created by Karpenter. Use with extra care! + $(RESK) -o yaml | yq eval '.[]|.id' | xargs --verbose -r -n 5 az resource delete --ids + +az-rmcluster: + az aks delete --name $(AZURE_CLUSTER_NAME) --resource-group $(AZURE_RESOURCE_GROUP) --yes + +az-portal: ## Get Azure Portal links for relevant resource groups + @echo https://ms.portal.azure.com/#@microsoft.onmicrosoft.com/asset/HubsExtension/ResourceGroups/subscriptions/$(AZURE_SUBSCRIPTION_ID)/resourceGroups/$(AZURE_RESOURCE_GROUP) + @echo https://ms.portal.azure.com/#@microsoft.onmicrosoft.com/asset/HubsExtension/ResourceGroups/subscriptions/$(AZURE_SUBSCRIPTION_ID)/resourceGroups/$(AZURE_RESOURCE_GROUP_MC) + +az-list-skus: ## List all public VM images from microsoft-aks + az vm image list-skus --publisher microsoft-aks --location $(AZURE_LOCATION) --offer aks -o table + +az-list-usage: ## List VM usage/quotas + az vm list-usage --location $(AZURE_LOCATION) -o table | grep "Family vCPU" + +az-ratelimits: ## Show remaining ARM requests for subscription + @az group create --name $(AZURE_RESOURCE_GROUP) --location $(AZURE_LOCATION) --debug 2>&1 | grep x-ms-ratelimit-remaining-subscription-writes + @az group show --name $(AZURE_RESOURCE_GROUP) --debug 2>&1 | grep x-ms-ratelimit-remaining-subscription-reads + +az-kdebug: ## Inject ephemeral debug container (kubectl debug) into Karpenter pod + $(eval POD=$(shell kubectl get pods -l app.kubernetes.io/name=karpenter -n karpenter -o name)) + kubectl debug -n karpenter $(POD) --image wbitt/network-multitool -it -- sh + +az-klogs: ## Karpenter logs + $(eval POD=$(shell kubectl get pods -l app.kubernetes.io/name=karpenter -n karpenter -o name)) + kubectl logs -f -n karpenter $(POD) + +az-kevents: ## Karpenter events + kubectl get events -A --field-selector source=karpenter + +az-provision-gpus: + kubectl apply -f examples/workloads/device-plugin.yaml + kubectl apply -f examples/provisioner/gpu-provisioner.yaml + kubectl apply -f examples/workloads/samples-mnist.yaml + + +az-node-viewer: ## Watch nodes using eks-node-viewer + eks-node-viewer --disable-pricing --node-selector "karpenter.sh/nodepool" # --resources cpu,memory + +az-argvmlist: ## List current VMs owned by Karpenter + az graph query -q "Resources | where type =~ 'microsoft.compute/virtualmachines' | where resourceGroup == tolower('$(AZURE_RESOURCE_GROUP_MC)') | where tags has_cs 'karpenter.sh_nodepool'" \ + --subscriptions $(AZURE_SUBSCRIPTION_ID) \ + | jq '.data[] | .id' + +az-pprof-enable: ## Enable profiling + yq -i '.manifests.helm.releases[0].overrides.controller.env += [{"name":"ENABLE_PROFILING","value":"true"}]' skaffold.yaml + +az-pprof: ## Profile + kubectl port-forward service/karpenter -n karpenter 8000 & + sleep 2 && go tool pprof -http 0.0.0.0:9000 localhost:8000/debug/pprof/heap diff --git a/NOTICE.txt b/NOTICE.txt new file mode 100644 index 000000000..b37e9bb9d --- /dev/null +++ b/NOTICE.txt @@ -0,0 +1,23 @@ +Karpenter +Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + +AKS Karpenter Provider +Copyright (c) Microsoft Corporation. + +NOTICES AND INFORMATION +Do Not Translate or Localize + +This software incorporates material from third parties. Microsoft makes certain +open source code available at https://3rdpartysource.microsoft.com, or you may +send a check or money order for US $5.00, including the product name, the open +source component name, and version number, to: + +Source Code Compliance Team +Microsoft Corporation +One Microsoft Way +Redmond, WA 98052 +USA + +Notwithstanding any other terms, you may reverse engineer this software to the +extent required to debug changes to any libraries licensed under the GNU Lesser +General Public License. diff --git a/README.md b/README.md new file mode 100644 index 000000000..2a0d421b9 --- /dev/null +++ b/README.md @@ -0,0 +1,87 @@ +[![GitHub License](https://img.shields.io/badge/License-Apache%202.0-ff69b4.svg)](https://github.com/Azure/karpenter/blob/main/LICENSE.txt) +[![CI](https://github.com/Azure/karpenter/actions/workflows/ci.yml/badge.svg?branch=main)](https://github.com/Azure/karpenter/actions/workflows/ci.yml) +![GitHub stars](https://img.shields.io/github/stars/Azure/karpenter) +![GitHub forks](https://img.shields.io/github/forks/Azure/karpenter) +[![Go Report Card](https://goreportcard.com/badge/github.com/Azure/karpenter)](https://goreportcard.com/report/github.com/Azure/karpenter) +[![contributions welcome](https://img.shields.io/badge/contributions-welcome-brightgreen.svg?style=flat)](https://github.com/Azure/karpenter/issues) + +--- + +# AKS Karpenter Provider + +The AKS Karpenter Provider enables node autoprovisioning using [Karpenter](https://karpenter.sh/) on your AKS cluster. + +## Status of Project: + +The API for AKS Karpenter Provider is currently alpha (`v1alpha2`). + +## Development + +A [GitHub Codespaces]((https://github.com/features/codespaces)) development flow is described below, which you can use to test karpenter functionality on your own cluster, and to aid rapid development of this project. + +1. **Install VSCode**: Go [here](https://code.visualstudio.com/download) to download VSCode for your platform. After installation, in your VSCode app install the "GitHub Codespaces" Extension. See [here](https://code.visualstudio.com/docs/remote/codespaces) for more information about this extension. + +2. **Create Codespace** (~2min): In browser, click Code / "Create a codespace on main" (for better experience customize to use 4cores/8GB), wait for codespace to be created. It is created with everything needed for development (Go, Azure CLI, kubectl, skaffold, useful plugins, etc.) Now you can open up the Codespace in VSCode: Click on Codespaces in the lower left corner in the browser status bar region, choose "Open in VSCode Desktop". (Pretty much everything except for `az login` and some `az role assignment` works in browser; but VSCode is a better experience anyway.) + +More information on GitHub Codespaces is [here](https://github.com/features/codespaces). + +3. **Provision cluster, build and deploy Karpenter** (~5min): Set `AZURE_SUBSCRIPTION_ID` to your subscription (and customize region in `Makefile-az.mk` if desired). Then at the VSCode command line run `make az-all`. This logs into Azure (follow the prompts), provisions AKS and ACR (using resource group `$CODESPACE_NAME`, so everything is unique / scoped to codespace), builds and deploys Karpenter, deploys sample `default` Provisioner and `inflate` Deployment workload. + +Manually scale the `inflate` Deployment workload, watch Karpenter controller log and Nodes in the cluster. Explore further with `make help` (mostly `az-*` targets). + +To debug Karpenter in-cluster, use `make az-debug`, wait for it to deploy, and attach from VSCode using Start Debugging (F5). After that you should be able to set breakpoints, examine variables, single step, etc. (Behind the scenes, besides building and deploying Karpenter, `skaffold debug` automatically and transparently applies the necessary flags during build, instruments the deployment with Delve, adjusts health probe timeouts - to allow for delays introduced by breakpoints, sets up port-forwarding, etc.; more on how this works is [here](https://skaffold.dev/docs/workflows/debug/). + +Once done, you can delete all infra with `make az-rmrg` (it deletes the resource group), and can delete the codespace (though it will be automatically suspended when not used, and deleted after 30 days.) + +#### Developer notes +- During step 1 you will observe `Running postCreateCommand...` which takes ~10+ minutes. You don't have to wait for it to finish to proceed to step 2. +- The following errors can be ignored during step 2: + +``` +ERRO[0007] gcloud binary not found +... +ERRO[0003] gcloud binary not found +... +ERRO[0187] walk.go:74: found symbolic link in path: /workspaces/karpenter/charts/karpenter/crds resolves to /workspaces/karpenter/pkg/apis/crds. Contents of linked file included and used  subtask=0 task=Render +``` +- If you see platform architecture error during `skaffold debug`, adjust (or comment out) `--platform` argument. +- If you are not able to set/hit breakpoints, it could be an issue with source paths mapping; see comments in debug launch configuration (`launch.json`) + +#### FAQs + +Q: I was able to trigger Karpenter to execute scaling up nodes as expected, using my own customized deployment of pods. However, scaling down was not handled automatically when I removed the deployment. The two new nodes created by Karpenter were left around. What is going on? + +A: Additional system workloads (such as metrics server) can get scheduled on the new nodes, preventing Karpenter from removing them. Note that you can always use `kubectl delete node `, which will have Karpenter drain the node and terminate the instance from cloud provider. + +Q: When running some of the tests locally, the environment failed to start. How can I resolve this? + +A: Oftentimes, especially for pre-existing tests, running `make toolchain` will fix this. This target will ensure that you have the correct versions of binaries installed. + +--- + +Karpenter is an open-source node provisioning project built for Kubernetes. +Karpenter improves the efficiency and cost of running workloads on Kubernetes clusters by: + +* **Watching** for pods that the Kubernetes scheduler has marked as unschedulable +* **Evaluating** scheduling constraints (resource requests, nodeselectors, affinities, tolerations, and topology spread constraints) requested by the pods +* **Provisioning** nodes that meet the requirements of the pods +* **Removing** the nodes when the nodes are no longer needed + +### Source Attribution + +Notice: Files in this source code originated from a fork of https://github.com/aws/karpenter +which is under an Apache 2.0 license. Those files have been modified to reflect environmental requirements in AKS and Azure. + +Many thanks to @ellistarn, @jonathan-innis, @tzneal, @bwagner5, @njtran, and many other developers active in the Karpenter community for laying the foundations of a Karpenter provider ecosystem! + +Many thanks to @Bryce-Soghigian, @rakechill, @charliedmcb, @jackfrancis, @comtalyst, @aagusuab, @matthchr, @gandhipr, @dtzar for contributing to AKS Karpenter Provider! + +--- + +This project has adopted the [Microsoft Open Source Code of Conduct](https://opensource.microsoft.com/codeofconduct/). +For more information see the [Code of Conduct FAQ](https://opensource.microsoft.com/codeofconduct/faq/) +or contact [opencode@microsoft.com](mailto:opencode@microsoft.com) with any additional questions or comments. + +Come discuss Karpenter in the [#karpenter](https://kubernetes.slack.com/archives/C02SFFZSA2K) channel in the [Kubernetes slack](https://slack.k8s.io/)! + +Check out the [Docs](https://karpenter.sh/) to learn more. diff --git a/SECURITY.md b/SECURITY.md new file mode 100644 index 000000000..78a37736b --- /dev/null +++ b/SECURITY.md @@ -0,0 +1,38 @@ + +## Security + +Microsoft takes the security of our software products and services seriously, which includes all source code repositories managed through our GitHub organizations, which include [Microsoft](https://github.com/Microsoft), [Azure](https://github.com/Azure), [DotNet](https://github.com/dotnet), [AspNet](https://github.com/aspnet) and [Xamarin](https://github.com/xamarin). + +If you believe you have found a security vulnerability in any Microsoft-owned repository that meets [Microsoft's definition of a security vulnerability](https://aka.ms/security.md/definition), please report it to us as described below. + +## Reporting Security Issues + +**Please do not report security vulnerabilities through public GitHub issues.** + +Instead, please report them to the Microsoft Security Response Center (MSRC) at [https://msrc.microsoft.com/create-report](https://aka.ms/security.md/msrc/create-report). + +If you prefer to submit without logging in, send email to [secure@microsoft.com](mailto:secure@microsoft.com). If possible, encrypt your message with our PGP key; please download it from the [Microsoft Security Response Center PGP Key page](https://aka.ms/security.md/msrc/pgp). + +You should receive a response within 24 hours. If for some reason you do not, please follow up via email to ensure we received your original message. Additional information can be found at [microsoft.com/msrc](https://www.microsoft.com/msrc). + +Please include the requested information listed below (as much as you can provide) to help us better understand the nature and scope of the possible issue: + + * Type of issue (e.g. buffer overflow, SQL injection, cross-site scripting, etc.) + * Full paths of source file(s) related to the manifestation of the issue + * The location of the affected source code (tag/branch/commit or direct URL) + * Any special configuration required to reproduce the issue + * Step-by-step instructions to reproduce the issue + * Proof-of-concept or exploit code (if possible) + * Impact of the issue, including how an attacker might exploit the issue + +This information will help us triage your report more quickly. + +If you are reporting for a bug bounty, more complete reports can contribute to a higher bounty award. Please visit our [Microsoft Bug Bounty Program](https://aka.ms/security.md/msrc/bounty) page for more details about our active programs. + +## Preferred Languages + +We prefer all communications to be in English. + +## Policy + +Microsoft follows the principle of [Coordinated Vulnerability Disclosure](https://aka.ms/security.md/cvd). \ No newline at end of file diff --git a/SUPPORT.md b/SUPPORT.md new file mode 100644 index 000000000..1261983fa --- /dev/null +++ b/SUPPORT.md @@ -0,0 +1,10 @@ +# Support +## How to file issues and get help +This project uses GitHub Issues to track bugs and feature requests. Please search the existing +issues before filing new issues to avoid duplicates. For new issues, file your bug or +feature request as a new Issue. + +For help and questions about using this project, please **Raise An issue with your question in the issues of this project!** + +## Microsoft Support Policy +Support for this **PROJECT or PRODUCT** is limited to the resources listed above. diff --git a/charts/karpenter-crd/.helmignore b/charts/karpenter-crd/.helmignore new file mode 100644 index 000000000..0e8a0eb36 --- /dev/null +++ b/charts/karpenter-crd/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/charts/karpenter-crd/Chart.yaml b/charts/karpenter-crd/Chart.yaml new file mode 100644 index 000000000..dcef507eb --- /dev/null +++ b/charts/karpenter-crd/Chart.yaml @@ -0,0 +1,16 @@ +apiVersion: v2 +name: karpenter-crd +description: A Helm chart for AKS Karpenter provider Custom Resource Definitions (CRDs) +type: application +version: 0.1.0 +appVersion: 0.1.0 +keywords: + - cluster + - node + - scheduler + - autoscaling + - lifecycle +home: https://github.com/Azure/karpenter/ +icon: https://repository-images.githubusercontent.com/278480393/dab059c8-caa1-4b55-aaa7-3d30e47a5616 +sources: + - https://github.com/Azure/karpenter/ diff --git a/charts/karpenter-crd/templates/karpenter.azure.com_aksnodeclasses.yaml b/charts/karpenter-crd/templates/karpenter.azure.com_aksnodeclasses.yaml new file mode 120000 index 000000000..4917e92a9 --- /dev/null +++ b/charts/karpenter-crd/templates/karpenter.azure.com_aksnodeclasses.yaml @@ -0,0 +1 @@ +../../../pkg/apis/crds/karpenter.azure.com_aksnodeclasses.yaml \ No newline at end of file diff --git a/charts/karpenter-crd/templates/karpenter.sh_machines.yaml b/charts/karpenter-crd/templates/karpenter.sh_machines.yaml new file mode 120000 index 000000000..f08680c94 --- /dev/null +++ b/charts/karpenter-crd/templates/karpenter.sh_machines.yaml @@ -0,0 +1 @@ +../../../pkg/apis/crds/karpenter.sh_machines.yaml \ No newline at end of file diff --git a/charts/karpenter-crd/templates/karpenter.sh_nodeclaims.yaml b/charts/karpenter-crd/templates/karpenter.sh_nodeclaims.yaml new file mode 120000 index 000000000..3f572b575 --- /dev/null +++ b/charts/karpenter-crd/templates/karpenter.sh_nodeclaims.yaml @@ -0,0 +1 @@ +../../../pkg/apis/crds/karpenter.sh_nodeclaims.yaml \ No newline at end of file diff --git a/charts/karpenter-crd/templates/karpenter.sh_nodepools.yaml b/charts/karpenter-crd/templates/karpenter.sh_nodepools.yaml new file mode 120000 index 000000000..36d2d1dd9 --- /dev/null +++ b/charts/karpenter-crd/templates/karpenter.sh_nodepools.yaml @@ -0,0 +1 @@ +../../../pkg/apis/crds/karpenter.sh_nodepools.yaml \ No newline at end of file diff --git a/charts/karpenter-crd/templates/karpenter.sh_provisioners.yaml b/charts/karpenter-crd/templates/karpenter.sh_provisioners.yaml new file mode 120000 index 000000000..921d1456e --- /dev/null +++ b/charts/karpenter-crd/templates/karpenter.sh_provisioners.yaml @@ -0,0 +1 @@ +../../../pkg/apis/crds/karpenter.sh_provisioners.yaml \ No newline at end of file diff --git a/charts/karpenter/.helmignore b/charts/karpenter/.helmignore new file mode 100644 index 000000000..0e8a0eb36 --- /dev/null +++ b/charts/karpenter/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/charts/karpenter/Chart.yaml b/charts/karpenter/Chart.yaml new file mode 100644 index 000000000..e1202578d --- /dev/null +++ b/charts/karpenter/Chart.yaml @@ -0,0 +1,16 @@ +apiVersion: v2 +name: karpenter +description: A Helm chart for AKS Karpenter provider. +type: application +version: 0.1.0 +appVersion: 0.29.2 +keywords: + - cluster + - node + - scheduler + - autoscaling + - lifecycle +home: https://github.com/Azure/karpenter/ +icon: https://repository-images.githubusercontent.com/278480393/dab059c8-caa1-4b55-aaa7-3d30e47a5616 +sources: + - https://github.com/Azure/karpenter/ diff --git a/charts/karpenter/README.md b/charts/karpenter/README.md new file mode 100644 index 000000000..f4afa3ea6 --- /dev/null +++ b/charts/karpenter/README.md @@ -0,0 +1,7 @@ +# AKS Karpenter provider + +A Helm chart for AKS Karpenter provider + +## Documentation + +For full Karpenter documentation please checkout [https://karpenter.sh](https://karpenter.sh/v0.32.1/). diff --git a/charts/karpenter/crds b/charts/karpenter/crds new file mode 120000 index 000000000..e33998437 --- /dev/null +++ b/charts/karpenter/crds @@ -0,0 +1 @@ +../../pkg/apis/crds \ No newline at end of file diff --git a/charts/karpenter/templates/_helpers.tpl b/charts/karpenter/templates/_helpers.tpl new file mode 100644 index 000000000..90415751c --- /dev/null +++ b/charts/karpenter/templates/_helpers.tpl @@ -0,0 +1,205 @@ +{{/* +Expand the name of the chart. +*/}} +{{- define "karpenter.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "karpenter.fullname" -}} +{{- if .Values.fullnameOverride }} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- $name := default .Chart.Name .Values.nameOverride }} +{{- if contains $name .Release.Name }} +{{- .Release.Name | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} +{{- end }} +{{- end }} +{{- end }} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "karpenter.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Common labels +*/}} +{{- define "karpenter.labels" -}} +helm.sh/chart: {{ include "karpenter.chart" . }} +{{ include "karpenter.selectorLabels" . }} +{{- if .Chart.AppVersion }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +{{- end }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- with .Values.additionalLabels }} +{{ toYaml . }} +{{- end }} +{{- end }} + +{{/* +Selector labels +*/}} +{{- define "karpenter.selectorLabels" -}} +app.kubernetes.io/name: {{ include "karpenter.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end }} + +{{/* +Create the name of the service account to use +*/}} +{{- define "karpenter.serviceAccountName" -}} +{{- if .Values.serviceAccount.create }} +{{- default (include "karpenter.fullname" .) .Values.serviceAccount.name }} +{{- else }} +{{- default "default" .Values.serviceAccount.name }} +{{- end }} +{{- end }} + +{{/* +Karpenter image to use +*/}} +{{- define "karpenter.controller.image" -}} +{{- if .Values.controller.image.digest }} +{{- printf "%s:%s@%s" .Values.controller.image.repository (default (printf "v%s" .Chart.AppVersion) .Values.controller.image.tag) .Values.controller.image.digest }} +{{- else }} +{{- printf "%s:%s" .Values.controller.image.repository (default (printf "v%s" .Chart.AppVersion) .Values.controller.image.tag) }} +{{- end }} +{{- end }} + + +{{/* Get PodDisruptionBudget API Version */}} +{{- define "karpenter.pdb.apiVersion" -}} +{{- if and (.Capabilities.APIVersions.Has "policy/v1") (semverCompare ">= 1.21-0" .Capabilities.KubeVersion.Version) -}} +{{- print "policy/v1" -}} +{{- else -}} +{{- print "policy/v1beta1" -}} +{{- end -}} +{{- end -}} + +{{/* +Patch the label selector on an object +This template will add a labelSelector using matchLabels to the object referenced at _target if there is no labelSelector specified. +The matchLabels are created with the selectorLabels template. +This works because Helm treats dictionaries as mutable objects and allows passing them by reference. +*/}} +{{- define "karpenter.patchLabelSelector" -}} +{{- if not (hasKey ._target "labelSelector") }} +{{- $selectorLabels := (include "karpenter.selectorLabels" .) | fromYaml }} +{{- $_ := set ._target "labelSelector" (dict "matchLabels" $selectorLabels) }} +{{- end }} +{{- end }} + +{{/* +Patch pod affinity +This template uses the patchLabelSelector template to add a labelSelector to pod affinity objects if there is no labelSelector specified. +This works because Helm treats dictionaries as mutable objects and allows passing them by reference. +*/}} +{{- define "karpenter.patchPodAffinity" -}} +{{- if (hasKey ._podAffinity "requiredDuringSchedulingIgnoredDuringExecution") }} +{{- range $term := ._podAffinity.requiredDuringSchedulingIgnoredDuringExecution }} +{{- include "karpenter.patchLabelSelector" (merge (dict "_target" $term) $) }} +{{- end }} +{{- end }} +{{- if (hasKey ._podAffinity "preferredDuringSchedulingIgnoredDuringExecution") }} +{{- range $weightedTerm := ._podAffinity.preferredDuringSchedulingIgnoredDuringExecution }} +{{- include "karpenter.patchLabelSelector" (merge (dict "_target" $weightedTerm.podAffinityTerm) $) }} +{{- end }} +{{- end }} +{{- end }} + +{{/* +Patch affinity +This template uses patchPodAffinity template to add a labelSelector to podAffinity & podAntiAffinity if one isn't specified. +This works because Helm treats dictionaries as mutable objects and allows passing them by reference. +*/}} +{{- define "karpenter.patchAffinity" -}} +{{- if (hasKey .Values.affinity "podAffinity") }} +{{- include "karpenter.patchPodAffinity" (merge (dict "_podAffinity" .Values.affinity.podAffinity) .) }} +{{- end }} +{{- if (hasKey .Values.affinity "podAntiAffinity") }} +{{- include "karpenter.patchPodAffinity" (merge (dict "_podAffinity" .Values.affinity.podAntiAffinity) .) }} +{{- end }} +{{- end }} + +{{/* +Patch topology spread constraints +This template uses the patchLabelSelector template to add a labelSelector to topologySpreadConstraints if one isn't specified. +This works because Helm treats dictionaries as mutable objects and allows passing them by reference. +*/}} +{{- define "karpenter.patchTopologySpreadConstraints" -}} +{{- range $constraint := .Values.topologySpreadConstraints }} +{{- include "karpenter.patchLabelSelector" (merge (dict "_target" $constraint) $) }} +{{- end }} +{{- end }} + +{{/* +Flatten Settings Map using "." syntax +*/}} +{{- define "flattenSettings" -}} +{{- $map := first . -}} +{{- $label := last . -}} +{{- range $key := (keys $map | uniq | sortAlpha) }} + {{- $sublabel := $key -}} + {{- $val := (get $map $key) -}} + {{- if $label -}} + {{- $sublabel = list $label $key | join "." -}} + {{- end -}} + {{/* Special-case "tags" since we want this to be a JSON object */}} + {{- if eq $key "tags" -}} + {{- if not (kindIs "invalid" $val) -}} + {{- $sublabel | quote | nindent 2 }}: {{ $val | toJson | quote }} + {{- end -}} + {{- else if kindOf $val | eq "map" -}} + {{- list $val $sublabel | include "flattenSettings" -}} + {{- else -}} + {{- if not (kindIs "invalid" $val) -}} + {{- $sublabel | quote | nindent 2 -}}: {{ $val | quote }} + {{- end -}} +{{- end -}} +{{- end -}} +{{- end -}} + +{{/* +Flatten the stdout logging outputs from args provided +*/}} +{{- define "karpenter.outputPathsList" -}} +{{ $paths := list -}} +{{- range .Values.controller.outputPaths -}} + {{- if not (has (printf "%s" . | quote) $paths) -}} + {{- $paths = printf "%s" . | quote | append $paths -}} + {{- end -}} +{{- end -}} +{{- range .Values.logConfig.outputPaths -}} + {{- if not (has (printf "%s" . | quote) $paths) -}} + {{- $paths = printf "%s" . | quote | append $paths -}} + {{- end -}} +{{- end -}} +{{ $paths | join ", " }} +{{- end -}} + +{{/* +Flatten the stderr logging outputs from args provided +*/}} +{{- define "karpenter.errorOutputPathsList" -}} +{{ $paths := list -}} +{{- range .Values.controller.errorOutputPaths -}} + {{- if not (has (printf "%s" . | quote) $paths) -}} + {{- $paths = printf "%s" . | quote | append $paths -}} + {{- end -}} +{{- end -}} +{{- range .Values.logConfig.errorOutputPaths -}} + {{- if not (has (printf "%s" . | quote) $paths) -}} + {{- $paths = printf "%s" . | quote | append $paths -}} + {{- end -}} +{{- end -}} +{{ $paths | join ", " }} +{{- end -}} \ No newline at end of file diff --git a/charts/karpenter/templates/aggregate-clusterrole.yaml b/charts/karpenter/templates/aggregate-clusterrole.yaml new file mode 100644 index 000000000..d84d11e75 --- /dev/null +++ b/charts/karpenter/templates/aggregate-clusterrole.yaml @@ -0,0 +1,23 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ include "karpenter.fullname" . }}-admin + labels: + rbac.authorization.k8s.io/aggregate-to-admin: "true" + {{- include "karpenter.labels" . | nindent 4 }} + {{- with .Values.additionalAnnotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +rules: + # at least listing machines appears to be required for now + - apiGroups: ["karpenter.sh"] + resources: ["provisioners", "provisioners/status", "machines", "machines/status"] + verbs: ["get", "list", "watch", "create", "delete", "patch"] + - apiGroups: ["karpenter.sh"] + resources: ["nodepools", "nodepools/status", "nodeclaims", "nodeclaims/status"] + verbs: ["get", "list", "watch", "create", "delete", "patch"] + - apiGroups: ["karpenter.azure.com"] + resources: ["aksnodeclasses"] + verbs: ["get", "list", "watch", "create", "delete", "patch"] + \ No newline at end of file diff --git a/charts/karpenter/templates/clusterrole-core.yaml b/charts/karpenter/templates/clusterrole-core.yaml new file mode 100644 index 000000000..df9c61ff1 --- /dev/null +++ b/charts/karpenter/templates/clusterrole-core.yaml @@ -0,0 +1,79 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ include "karpenter.fullname" . }}-core + labels: + {{- include "karpenter.labels" . | nindent 4 }} + {{- with .Values.additionalAnnotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ include "karpenter.fullname" . }}-core +subjects: + - kind: ServiceAccount + name: {{ template "karpenter.serviceAccountName" . }} + namespace: {{ .Release.Namespace }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ include "karpenter.fullname" . }}-core + labels: + {{- include "karpenter.labels" . | nindent 4 }} + {{- with .Values.additionalAnnotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +rules: + # Read + - apiGroups: ["karpenter.sh"] + resources: ["provisioners", "provisioners/status", "machines", "machines/status"] + verbs: ["get", "list", "watch"] + - apiGroups: ["karpenter.sh"] + resources: ["nodepools", "nodepools/status", "nodeclaims", "nodeclaims/status"] + verbs: ["get", "list", "watch"] + - apiGroups: [""] + resources: ["pods", "nodes", "persistentvolumes", "persistentvolumeclaims", "replicationcontrollers", "namespaces"] + verbs: ["get", "list", "watch"] + - apiGroups: ["storage.k8s.io"] + resources: ["storageclasses", "csinodes"] + verbs: ["get", "watch", "list"] + - apiGroups: ["apps"] + resources: ["daemonsets", "deployments", "replicasets", "statefulsets"] + verbs: ["list", "watch"] +{{- if .Values.webhook.enabled }} + - apiGroups: ["admissionregistration.k8s.io"] + resources: ["validatingwebhookconfigurations", "mutatingwebhookconfigurations"] + verbs: ["get", "watch", "list"] +{{- end }} + - apiGroups: ["policy"] + resources: ["poddisruptionbudgets"] + verbs: ["get", "list", "watch"] + # Write + - apiGroups: ["karpenter.sh"] + resources: ["nodeclaims", "nodeclaims/status"] + verbs: ["create", "delete", "update", "patch"] + - apiGroups: ["karpenter.sh"] + resources: ["nodepools", "nodepools/status"] + verbs: ["update", "patch"] + - apiGroups: [""] + resources: ["events"] + verbs: ["create", "patch"] + - apiGroups: [""] + resources: ["nodes"] + verbs: ["create", "patch", "delete"] + - apiGroups: [""] + resources: ["pods/eviction"] + verbs: ["create"] +{{- if .Values.webhook.enabled }} + - apiGroups: ["admissionregistration.k8s.io"] + resources: ["validatingwebhookconfigurations"] + verbs: ["update"] + resourceNames: ["validation.webhook.karpenter.sh", "validation.webhook.config.karpenter.sh"] +{{- end }} + {{- with .Values.additionalClusterRoleRules -}} + {{ toYaml . | nindent 2 }} + {{- end -}} diff --git a/charts/karpenter/templates/clusterrole.yaml b/charts/karpenter/templates/clusterrole.yaml new file mode 100644 index 000000000..b981c1bc5 --- /dev/null +++ b/charts/karpenter/templates/clusterrole.yaml @@ -0,0 +1,48 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ include "karpenter.fullname" . }} + labels: + {{- include "karpenter.labels" . | nindent 4 }} + {{- with .Values.additionalAnnotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ include "karpenter.fullname" . }} +subjects: + - kind: ServiceAccount + name: {{ template "karpenter.serviceAccountName" . }} + namespace: {{ .Release.Namespace }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ include "karpenter.fullname" . }} + labels: + {{- include "karpenter.labels" . | nindent 4 }} + {{- with .Values.additionalAnnotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +rules: + # Read + - apiGroups: ["karpenter.azure.com"] + resources: ["aksnodeclasses"] + verbs: ["get", "list", "watch"] + # Write + - apiGroups: ["karpenter.kubernetes.com"] + resources: ["aksnodeclasses", "aksnodeclasses/status"] + verbs: ["patch", "update"] +{{- if .Values.webhook.enabled }} + - apiGroups: ["admissionregistration.k8s.io"] + resources: ["validatingwebhookconfigurations"] + verbs: ["update"] + resourceNames: ["validation.webhook.karpenter.azure.com"] + - apiGroups: ["admissionregistration.k8s.io"] + resources: ["mutatingwebhookconfigurations"] + verbs: ["update"] + resourceNames: ["defaulting.webhook.karpenter.azure.com"] +{{- end }} diff --git a/charts/karpenter/templates/configmap-logging.yaml b/charts/karpenter/templates/configmap-logging.yaml new file mode 100644 index 000000000..0de88c183 --- /dev/null +++ b/charts/karpenter/templates/configmap-logging.yaml @@ -0,0 +1,41 @@ +{{- if .Values.logConfig.enabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + name: config-logging + namespace: {{ .Release.Namespace }} + labels: + {{- include "karpenter.labels" . | nindent 4 }} + {{- with .Values.additionalAnnotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +data: + # https://github.com/uber-go/zap/blob/aa3e73ec0896f8b066ddf668597a02f89628ee50/config.go + zap-logger-config: | + { + "level": "{{ .Values.logConfig.logLevel.global }}", + "development": false, + "disableStacktrace": true, + "disableCaller": true, + "sampling": { + "initial": 100, + "thereafter": 100 + }, + "outputPaths": [{{ include "karpenter.outputPathsList" . }}], + "errorOutputPaths": [{{ include "karpenter.errorOutputPathsList" . }}], + "encoding": "{{ or .Values.logEncoding .Values.logConfig.logEncoding }}", + "encoderConfig": { + "timeKey": "time", + "levelKey": "level", + "nameKey": "logger", + "callerKey": "caller", + "messageKey": "message", + "stacktraceKey": "stacktrace", + "levelEncoder": "capital", + "timeEncoder": "iso8601" + } + } + loglevel.controller: {{ or .Values.controller.logLevel .Values.logConfig.logLevel.controller }} + loglevel.webhook: {{ or .Values.webhook.logLevel .Values.logConfig.logLevel.webhook }} +{{- end }} diff --git a/charts/karpenter/templates/configmap.yaml b/charts/karpenter/templates/configmap.yaml new file mode 100644 index 000000000..d52c7b6fe --- /dev/null +++ b/charts/karpenter/templates/configmap.yaml @@ -0,0 +1,13 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: karpenter-global-settings + namespace: {{ .Release.Namespace }} + labels: + {{- include "karpenter.labels" . | nindent 4 }} + {{- with .Values.additionalAnnotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +data: + {{- list .Values.settings "" | include "flattenSettings" | indent 2 }} diff --git a/charts/karpenter/templates/deployment.yaml b/charts/karpenter/templates/deployment.yaml new file mode 100644 index 000000000..d86cf7cc3 --- /dev/null +++ b/charts/karpenter/templates/deployment.yaml @@ -0,0 +1,216 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "karpenter.fullname" . }} + namespace: {{ .Release.Namespace }} + labels: + {{- include "karpenter.labels" . | nindent 4 }} + {{- with .Values.additionalAnnotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + replicas: {{ .Values.replicas }} + revisionHistoryLimit: {{ .Values.revisionHistoryLimit }} + {{- with .Values.strategy }} + strategy: + {{- toYaml . | nindent 4 }} + {{- end }} + selector: + matchLabels: + {{- include "karpenter.selectorLabels" . | nindent 6 }} + template: + metadata: + labels: + {{- include "karpenter.selectorLabels" . | nindent 8 }} + {{- with .Values.podLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + annotations: + {{- with .Values.podAnnotations }} + {{- toYaml . | nindent 8 }} + {{- end }} + checksum/settings: {{ include (print $.Template.BasePath "/configmap.yaml") . | sha256sum }} + spec: + {{- with .Values.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} + serviceAccountName: {{ include "karpenter.serviceAccountName" . }} + {{- with .Values.priorityClassName }} + priorityClassName: {{ . | quote }} + {{- end }} + {{- with .Values.terminationGracePeriodSeconds }} + terminationGracePeriodSeconds: {{ . }} + {{- end }} + {{- with .Values.dnsPolicy }} + dnsPolicy: {{ . }} + {{- end }} + {{- with .Values.dnsConfig }} + dnsConfig: + {{- toYaml . | nindent 8}} + {{- end }} + {{- if .Values.hostNetwork }} + hostNetwork: true + {{- end }} + containers: + - name: controller + securityContext: + runAsUser: 65536 + runAsGroup: 65536 + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + readOnlyRootFilesystem: true + image: {{ include "karpenter.controller.image" . }} + imagePullPolicy: {{ .Values.imagePullPolicy }} + env: + - name: KUBERNETES_MIN_VERSION + value: "1.19.0-0" + - name: KARPENTER_SERVICE + value: {{ include "karpenter.fullname" . }} + {{- if .Values.webhook.enabled }} + - name: WEBHOOK_PORT + value: "{{ .Values.webhook.port }}" + - name: WEBHOOK_METRICS_PORT + value: "{{ .Values.webhook.metrics.port }}" + {{- else }} + - name: DISABLE_WEBHOOK + value: "true" + {{- end }} + {{- with .Values.logLevel }} + - name: LOG_LEVEL + value: "{{ . }}" + {{- end }} + - name: METRICS_PORT + value: "{{ .Values.controller.metrics.port }}" + - name: HEALTH_PROBE_PORT + value: "{{ .Values.controller.healthProbe.port }}" + - name: SYSTEM_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: MEMORY_LIMIT + valueFrom: + resourceFieldRef: + containerName: controller + divisor: "0" + resource: limits.memory + - name: FEATURE_GATES + value: "Drift={{ or .Values.settings.featureGates.driftEnabled .Values.settings.featureGates.drift }}" + {{- with .Values.settings.batchMaxDuration }} + - name: BATCH_MAX_DURATION + value: "{{ . }}" + {{- end }} + {{- with .Values.settings.batchIdleDuration }} + - name: BATCH_IDLE_DURATION + value: "{{ . }}" + {{- end }} + {{- with or .Values.settings.azure.clusterName .Values.settings.clusterName }} + - name: CLUSTER_NAME + value: "{{ . }}" + {{- end }} + {{- with or .Values.settings.azure.clusterEndpoint .Values.settings.clusterEndpoint }} + - name: CLUSTER_ENDPOINT + value: "{{ . }}" + {{- end }} + {{- with or .Values.settings.azure.vmMemoryOverheadPercent .Values.settings.vmMemoryOverheadPercent }} + - name: VM_MEMORY_OVERHEAD_PERCENT + value: "{{ . }}" + {{- end }} + {{- with .Values.controller.env }} + {{- toYaml . | nindent 12 }} + {{- end }} + {{- with .Values.controller.envFrom }} + envFrom: + {{- toYaml . | nindent 12 }} + {{- end }} + ports: + - name: http-metrics + containerPort: {{ .Values.controller.metrics.port }} + protocol: TCP + {{- if .Values.webhook.enabled }} + - name: webhook-metrics + containerPort: {{ .Values.webhook.metrics.port }} + {{- end }} + protocol: TCP + - name: http + containerPort: {{ .Values.controller.healthProbe.port }} + protocol: TCP + - name: https-webhook + containerPort: {{ .Values.webhook.port }} + protocol: TCP + livenessProbe: + initialDelaySeconds: 30 + timeoutSeconds: 30 + httpGet: + path: /healthz + port: http + readinessProbe: + initialDelaySeconds: 5 + timeoutSeconds: 30 + httpGet: + path: /readyz + port: http + {{- with .Values.controller.resources }} + resources: + {{- toYaml . | nindent 12 }} + {{- end }} + {{- if or (.Values.logConfig.enabled) (.Values.controller.extraVolumeMounts) }} + volumeMounts: + {{- if .Values.logConfig.enabled }} + - name: config-logging + mountPath: /etc/karpenter/logging + {{- end }} + {{- with .Values.controller.extraVolumeMounts }} + {{- toYaml . | nindent 12 }} + {{- end }} + {{- end }} + {{- with .Values.controller.sidecarContainer }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- if and (.Values.controller.sidecarContainer) (or .Values.controller.extraVolumeMounts .Values.controller.sidecarVolumeMounts) }} + volumeMounts: + {{- with .Values.controller.extraVolumeMounts }} + {{- toYaml . | nindent 12 }} + {{- end }} + {{- with .Values.controller.sidecarVolumeMounts }} + {{- toYaml . | nindent 12 }} + {{- end }} + {{- end }} + {{- with .Values.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.affinity }} + # The template below patches the .Values.affinity to add a default label selector where not specificed + {{- $_ := include "karpenter.patchAffinity" $ }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.topologySpreadConstraints }} + # The template below patches the .Values.topologySpreadConstraints to add a default label selector where not specificed + {{- $_ := include "karpenter.patchTopologySpreadConstraints" $ }} + topologySpreadConstraints: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- if or (.Values.logConfig.enabled) (.Values.extraVolumes) }} + volumes: + {{- if .Values.logConfig.enabled }} + - name: config-logging + configMap: + name: config-logging + {{- end }} + {{- with .Values.extraVolumes }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + \ No newline at end of file diff --git a/charts/karpenter/templates/poddisruptionbudget.yaml b/charts/karpenter/templates/poddisruptionbudget.yaml new file mode 100644 index 000000000..a5a55cadf --- /dev/null +++ b/charts/karpenter/templates/poddisruptionbudget.yaml @@ -0,0 +1,16 @@ +apiVersion: {{ include "karpenter.pdb.apiVersion" . }} +kind: PodDisruptionBudget +metadata: + name: {{ .Values.podDisruptionBudget.name }} + namespace: {{ .Release.Namespace }} + labels: + {{- include "karpenter.labels" . | nindent 4 }} + {{- with .Values.additionalAnnotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + maxUnavailable: {{ .Values.podDisruptionBudget.maxUnavailable }} + selector: + matchLabels: + {{- include "karpenter.selectorLabels" . | nindent 6 }} diff --git a/charts/karpenter/templates/role.yaml b/charts/karpenter/templates/role.yaml new file mode 100644 index 000000000..4bacb3862 --- /dev/null +++ b/charts/karpenter/templates/role.yaml @@ -0,0 +1,87 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: {{ include "karpenter.fullname" . }} + namespace: {{ .Release.Namespace }} + labels: + {{- include "karpenter.labels" . | nindent 4 }} + {{- with .Values.additionalAnnotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +rules: + # Read + - apiGroups: ["coordination.k8s.io"] + resources: ["leases"] + verbs: ["get", "watch"] + - apiGroups: [""] + resources: ["configmaps", "namespaces", "secrets"] + verbs: ["get", "list", "watch"] + # Write + - apiGroups: [""] + resources: ["secrets"] + verbs: ["update"] + resourceNames: ["{{ include "karpenter.fullname" . }}-cert"] + - apiGroups: [""] + resources: ["configmaps"] + verbs: ["update", "patch", "delete"] + resourceNames: + - karpenter-global-settings + - config-logging + - apiGroups: ["coordination.k8s.io"] + resources: ["leases"] + verbs: ["patch", "update"] + resourceNames: + - "karpenter-leader-election" + - "webhook.configmapwebhook.00-of-01" + - "webhook.defaultingwebhook.00-of-01" + - "webhook.validationwebhook.00-of-01" + - "webhook.webhookcertificates.00-of-01" + # Cannot specify resourceNames on create + # https://kubernetes.io/docs/reference/access-authn-authz/rbac/#referring-to-resources + - apiGroups: ["coordination.k8s.io"] + resources: ["leases"] + verbs: ["create"] + - apiGroups: [""] + resources: ["configmaps"] + verbs: ["create"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: {{ include "karpenter.fullname" . }}-dns + namespace: kube-system + labels: + {{- include "karpenter.labels" . | nindent 4 }} + {{- with .Values.additionalAnnotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +rules: + # Read + - apiGroups: [""] + resources: ["services"] + resourceNames: ["kube-dns"] + verbs: ["get"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: {{ include "karpenter.fullname" . }}-lease + namespace: kube-node-lease + labels: + {{- include "karpenter.labels" . | nindent 4 }} + {{- with .Values.additionalAnnotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +rules: + # Read + - apiGroups: ["coordination.k8s.io"] + resources: ["leases"] + verbs: ["get", "list", "watch"] + # Write + - apiGroups: ["coordination.k8s.io"] + resources: ["leases"] + verbs: ["delete"] + diff --git a/charts/karpenter/templates/rolebinding.yaml b/charts/karpenter/templates/rolebinding.yaml new file mode 100644 index 000000000..8606a4902 --- /dev/null +++ b/charts/karpenter/templates/rolebinding.yaml @@ -0,0 +1,59 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: {{ include "karpenter.fullname" . }} + namespace: {{ .Release.Namespace }} + labels: + {{- include "karpenter.labels" . | nindent 4 }} + {{- with .Values.additionalAnnotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: {{ include "karpenter.fullname" . }} +subjects: + - kind: ServiceAccount + name: {{ template "karpenter.serviceAccountName" . }} + namespace: {{ .Release.Namespace }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: {{ include "karpenter.fullname" . }}-dns + namespace: kube-system + labels: + {{- include "karpenter.labels" . | nindent 4 }} + {{- with .Values.additionalAnnotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: {{ include "karpenter.fullname" . }}-dns +subjects: + - kind: ServiceAccount + name: {{ template "karpenter.serviceAccountName" . }} + namespace: {{ .Release.Namespace }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: {{ include "karpenter.fullname" . }}-lease + namespace: kube-node-lease + labels: + {{- include "karpenter.labels" . | nindent 4 }} + {{- with .Values.additionalAnnotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: {{ include "karpenter.fullname" . }}-lease +subjects: + - kind: ServiceAccount + name: {{ template "karpenter.serviceAccountName" . }} + namespace: {{ .Release.Namespace }} \ No newline at end of file diff --git a/charts/karpenter/templates/secret-webhook-cert.yaml b/charts/karpenter/templates/secret-webhook-cert.yaml new file mode 100644 index 000000000..a364f9b71 --- /dev/null +++ b/charts/karpenter/templates/secret-webhook-cert.yaml @@ -0,0 +1,12 @@ +apiVersion: v1 +kind: Secret +metadata: + name: {{ include "karpenter.fullname" . }}-cert + namespace: {{ .Release.Namespace }} + labels: + {{- include "karpenter.labels" . | nindent 4 }} + {{- with .Values.additionalAnnotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +# data: {} # Injected by karpenter-webhook diff --git a/charts/karpenter/templates/service.yaml b/charts/karpenter/templates/service.yaml new file mode 100644 index 000000000..5236489ae --- /dev/null +++ b/charts/karpenter/templates/service.yaml @@ -0,0 +1,24 @@ +apiVersion: v1 +kind: Service +metadata: + name: {{ include "karpenter.fullname" . }} + namespace: {{ .Release.Namespace }} + labels: + {{- include "karpenter.labels" . | nindent 4 }} + {{- with .Values.additionalAnnotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + type: ClusterIP + ports: + - name: http-metrics + port: {{ .Values.controller.metrics.port }} + targetPort: http-metrics + protocol: TCP + - name: https-webhook + port: {{ .Values.webhook.port }} + targetPort: https-webhook + protocol: TCP + selector: + {{- include "karpenter.selectorLabels" . | nindent 4 }} diff --git a/charts/karpenter/templates/serviceaccount.yaml b/charts/karpenter/templates/serviceaccount.yaml new file mode 100644 index 000000000..0141afc29 --- /dev/null +++ b/charts/karpenter/templates/serviceaccount.yaml @@ -0,0 +1,18 @@ +{{- if .Values.serviceAccount.create -}} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "karpenter.serviceAccountName" . }} + namespace: {{ .Release.Namespace }} + labels: + {{- include "karpenter.labels" . | nindent 4 }} + {{- if or .Values.additionalAnnotations .Values.serviceAccount.annotations }} + annotations: + {{- with .Values.additionalAnnotations }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- with .Values.serviceAccount.annotations }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- end }} +{{- end -}} diff --git a/charts/karpenter/templates/servicemonitor.yaml b/charts/karpenter/templates/servicemonitor.yaml new file mode 100644 index 000000000..70f07608f --- /dev/null +++ b/charts/karpenter/templates/servicemonitor.yaml @@ -0,0 +1,29 @@ +{{- if.Values.serviceMonitor.enabled -}} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ include "karpenter.fullname" . }} + namespace: {{ .Release.Namespace }} + labels: + {{- include "karpenter.labels" . | nindent 4 }} + {{- with .Values.serviceMonitor.additionalLabels }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- with .Values.additionalAnnotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + namespaceSelector: + matchNames: + - {{ .Release.Namespace }} + selector: + matchLabels: + {{- include "karpenter.selectorLabels" . | nindent 6 }} + endpoints: + - port: http-metrics + path: /metrics + {{- with .Values.serviceMonitor.endpointConfig }} + {{- toYaml . | nindent 6 }} + {{- end }} +{{- end -}} diff --git a/charts/karpenter/templates/webhooks-core.yaml b/charts/karpenter/templates/webhooks-core.yaml new file mode 100644 index 000000000..17104e104 --- /dev/null +++ b/charts/karpenter/templates/webhooks-core.yaml @@ -0,0 +1,81 @@ +{{- if .Values.webhook.enabled }} +apiVersion: admissionregistration.k8s.io/v1 +kind: ValidatingWebhookConfiguration +metadata: + name: validation.webhook.karpenter.sh + labels: + {{- include "karpenter.labels" . | nindent 4 }} + {{- with .Values.additionalAnnotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +webhooks: + - name: validation.webhook.karpenter.sh + admissionReviewVersions: ["v1"] + clientConfig: + service: + name: {{ include "karpenter.fullname" . }} + namespace: {{ .Release.Namespace }} + port: {{ .Values.webhook.port }} + failurePolicy: Fail + sideEffects: None + rules: + - apiGroups: + - karpenter.sh + apiVersions: + - v1alpha5 + operations: + - CREATE + - UPDATE + resources: + - provisioners + - provisioners/status + scope: '*' + - apiGroups: + - karpenter.sh + apiVersions: + - v1beta1 + operations: + - CREATE + - UPDATE + - DELETE + resources: + - nodeclaims + - nodeclaims/status + scope: '*' + - apiGroups: + - karpenter.sh + apiVersions: + - v1beta1 + operations: + - CREATE + - UPDATE + resources: + - nodepools + - nodepools/status + scope: '*' +--- +apiVersion: admissionregistration.k8s.io/v1 +kind: ValidatingWebhookConfiguration +metadata: + name: validation.webhook.config.karpenter.sh + labels: + {{- include "karpenter.labels" . | nindent 4 }} + {{- with .Values.additionalAnnotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +webhooks: + - name: validation.webhook.config.karpenter.sh + admissionReviewVersions: ["v1"] + clientConfig: + service: + name: {{ include "karpenter.fullname" . }} + namespace: {{ .Release.Namespace }} + port: {{ .Values.webhook.port }} + failurePolicy: Fail + sideEffects: None + objectSelector: + matchLabels: + app.kubernetes.io/part-of: {{ template "karpenter.name" . }} +{{- end }} diff --git a/charts/karpenter/templates/webhooks.yaml b/charts/karpenter/templates/webhooks.yaml new file mode 100644 index 000000000..6af7df8bf --- /dev/null +++ b/charts/karpenter/templates/webhooks.yaml @@ -0,0 +1,67 @@ +{{- if .Values.webhook.enabled }} +apiVersion: admissionregistration.k8s.io/v1 +kind: MutatingWebhookConfiguration +metadata: + name: defaulting.webhook.karpenter.azure.com + labels: + {{- include "karpenter.labels" . | nindent 4 }} + {{- with .Values.additionalAnnotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +webhooks: + - name: defaulting.webhook.karpenter.azure.com + admissionReviewVersions: ["v1"] + clientConfig: + service: + name: {{ include "karpenter.fullname" . }} + namespace: {{ .Release.Namespace }} + port: {{ .Values.webhook.port }} + failurePolicy: Fail + sideEffects: None + rules: + - apiGroups: + - karpenter.azure.com + apiVersions: + - v1alpha2 + operations: + - CREATE + - UPDATE + resources: + - aksnodeclasses + - aksnodeclasses/status + scope: '*' +--- +apiVersion: admissionregistration.k8s.io/v1 +kind: ValidatingWebhookConfiguration +metadata: + name: validation.webhook.karpenter.azure.com + labels: + {{- include "karpenter.labels" . | nindent 4 }} + {{- with .Values.additionalAnnotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +webhooks: + - name: validation.webhook.karpenter.azure.com + admissionReviewVersions: ["v1"] + clientConfig: + service: + name: {{ include "karpenter.fullname" . }} + namespace: {{ .Release.Namespace }} + port: {{ .Values.webhook.port }} + failurePolicy: Fail + sideEffects: None + rules: + - apiGroups: + - karpenter.azure.com + apiVersions: + - v1alpha2 + operations: + - CREATE + - UPDATE + resources: + - aksnodeclasses + - aksnodeclasses/status + scope: '*' +{{- end }} diff --git a/charts/karpenter/values.yaml b/charts/karpenter/values.yaml new file mode 100644 index 000000000..1660d875b --- /dev/null +++ b/charts/karpenter/values.yaml @@ -0,0 +1,191 @@ +# -- Overrides the chart's name. +nameOverride: "" +# -- Overrides the chart's computed fullname. +fullnameOverride: "" +# -- Additional labels to add into metadata. +additionalLabels: {} +# app: karpenter + +# -- Additional annotations to add into metadata. +additionalAnnotations: {} +# -- Image pull policy for Docker images. +imagePullPolicy: IfNotPresent +# -- Image pull secrets for Docker images. +imagePullSecrets: [] +serviceAccount: + # -- Specifies if a ServiceAccount should be created. + create: true + # -- The name of the ServiceAccount to use. + # If not set and create is true, a name is generated using the fullname template. + name: "" + # -- Additional annotations for the ServiceAccount. + annotations: {} +# -- Specifies additional rules for the core ClusterRole. +additionalClusterRoleRules: [] +serviceMonitor: + # -- Specifies whether a ServiceMonitor should be created. + enabled: false + # -- Additional labels for the ServiceMonitor. + additionalLabels: {} + # -- Endpoint configuration for the ServiceMonitor. + endpointConfig: {} +# -- Number of replicas. +replicas: 2 +# -- The number of old ReplicaSets to retain to allow rollback. +revisionHistoryLimit: 10 +# -- Strategy for updating the pod. +strategy: + rollingUpdate: + maxUnavailable: 1 +# -- Additional labels for the pod. +podLabels: {} +# -- Additional annotations for the pod. +podAnnotations: {} +podDisruptionBudget: + name: karpenter + maxUnavailable: 1 +# -- PriorityClass name for the pod. +priorityClassName: system-cluster-critical +# -- Override the default termination grace period for the pod. +terminationGracePeriodSeconds: +# -- Bind the pod to the host network. +# This is required when using a custom CNI. +hostNetwork: false +# -- Configure the DNS Policy for the pod +dnsPolicy: Default +# -- Configure DNS Config for the pod +dnsConfig: {} +# options: +# - name: ndots +# value: "1" +# -- Node selectors to schedule the pod to nodes with labels. +nodeSelector: + kubernetes.io/os: linux +# -- Affinity rules for scheduling the pod. If an explicit label selector is not provided for pod affinity or pod anti-affinity one will be created from the pod selector labels. +affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: karpenter.sh/nodepool + operator: DoesNotExist + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - topologyKey: "kubernetes.io/hostname" +# -- Topology spread constraints to increase the controller resilience by distributing pods across the cluster zones. If an explicit label selector is not provided one will be created from the pod selector labels. +topologySpreadConstraints: + - maxSkew: 1 + topologyKey: topology.kubernetes.io/zone + whenUnsatisfiable: ScheduleAnyway +# -- Tolerations to allow the pod to be scheduled to nodes with taints. +tolerations: + - key: CriticalAddonsOnly + operator: Exists +# -- Additional volumes for the pod. +extraVolumes: [] +# - name: +# projected: +# defaultMode: 420 +# sources: +# - serviceAccountToken: +# audience: +# expirationSeconds: 86400 +# path: token +controller: + image: + # -- Repository path to the controller image. + repository: public.ecr.aws/karpenter/controller + # -- Tag of the controller image. + tag: v0.32.1 + # -- SHA256 digest of the controller image. + # digest: + # -- Additional environment variables for the controller pod. + env: [] + # - name: + # value: + envFrom: [] + # -- Resources for the controller pod. + resources: {} + # We usually recommend not to specify default resources and to leave this as a conscious + # choice for the user. This also increases chances charts run on environments with little + # resources, such as Minikube. If you do want to specify resources, uncomment the following + # lines, adjust them as necessary, and remove the curly braces after 'resources:'. + # requests: + # cpu: 1 + # memory: 1Gi + # limits: + # cpu: 1 + # memory: 1Gi + + # -- Additional volumeMounts for the controller pod. + extraVolumeMounts: [] + # - name: + # mountPath: + # readOnly: + # -- Additional sidecarContainer config + sidecarContainer: [] + # -- Additional volumeMounts for the sidecar - this will be added to the volume mounts on top of extraVolumeMounts + sidecarVolumeMounts: [] + metrics: + # -- The container port to use for metrics. + port: 8000 + healthProbe: + # -- The container port to use for http health probe. + port: 8081 +webhook: + # -- Whether to enable the webhooks and webhook permissions. + enabled: true + # -- The container port to use for the webhook. + port: 8443 + metrics: + # -- The container port to use for webhook metrics. + port: 8001 +# -- Global log level +logLevel: debug +# -- Log configuration (Deprecated: Logging configuration will be dropped by v1, use logLevel instead) +logConfig: + # -- Whether to enable provisioning and mounting the log ConfigMap + enabled: true + # -- Log outputPaths - defaults to stdout only + outputPaths: + - stdout + # -- Log errorOutputPaths - defaults to stderr only + errorOutputPaths: + - stderr + # -- Log encoding - defaults to json - must be one of 'json', 'console' + logEncoding: json + # -- Component-based log configuration + logLevel: + # -- Global log level, defaults to 'debug' + global: debug + # -- Controller log level, defaults to 'debug' + controller: debug + # -- Error log level, defaults to 'error' + webhook: error +# -- Global Settings to configure Karpenter +settings: + # -- The maximum length of a batch window. The longer this is, the more pods we can consider for provisioning at one + # time which usually results in fewer but larger nodes. + batchMaxDuration: 10s + # -- The maximum amount of time with no new ending pods that if exceeded ends the current batching window. If pods arrive + # faster than this time, the batching window will be extended up to the maxDuration. If they arrive slower, the pods + # will be batched separately. + batchIdleDuration: 1s + # -- Azure-specific configuration values + azure: + # -- Cluster name. + clusterName: "" + # -- Cluster endpoint. + clusterEndpoint: "" + # -- The VM memory overhead as a percent that will be subtracted from the total memory for all instance types + vmMemoryOverheadPercent: 0.075 + # -- The global tags to use on all Azure infrastructure resources (VMs, etc.) + # TODO: not propagated yet ... + tags: + # -- Feature Gate configuration values. Feature Gates will follow the same graduation process and requirements as feature gates + # in Kubernetes. More information here https://kubernetes.io/docs/reference/command-line-tools-reference/feature-gates/#feature-gates-for-alpha-or-beta-features + featureGates: + # -- drift is in ALPHA and is disabled by default. + # Setting drift to true enables the drift disruption method to watch for drift between currently deployed nodes + # and the desired state of nodes set in provisioners and node templates + drift: false diff --git a/cmd/controller/kodata/HEAD b/cmd/controller/kodata/HEAD new file mode 120000 index 000000000..8f63681d3 --- /dev/null +++ b/cmd/controller/kodata/HEAD @@ -0,0 +1 @@ +../../../.git/HEAD \ No newline at end of file diff --git a/cmd/controller/kodata/refs b/cmd/controller/kodata/refs new file mode 120000 index 000000000..739d35bf9 --- /dev/null +++ b/cmd/controller/kodata/refs @@ -0,0 +1 @@ +../../../.git/refs \ No newline at end of file diff --git a/cmd/controller/main.go b/cmd/controller/main.go new file mode 100644 index 000000000..1d31fa701 --- /dev/null +++ b/cmd/controller/main.go @@ -0,0 +1,67 @@ +//go:build !ccp + +/* +Portions Copyright (c) Microsoft Corporation. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "github.com/samber/lo" + + "github.com/Azure/karpenter/pkg/cloudprovider" + "github.com/Azure/karpenter/pkg/operator" + "github.com/Azure/karpenter/pkg/webhooks" + + controllers "github.com/Azure/karpenter/pkg/controllers" + "github.com/aws/karpenter-core/pkg/cloudprovider/metrics" + corecontrollers "github.com/aws/karpenter-core/pkg/controllers" + "github.com/aws/karpenter-core/pkg/controllers/state" + coreoperator "github.com/aws/karpenter-core/pkg/operator" + corewebhooks "github.com/aws/karpenter-core/pkg/webhooks" +) + +func main() { + ctx, op := operator.NewOperator(coreoperator.NewOperator()) + aksCloudProvider := cloudprovider.New( + op.InstanceTypesProvider, + op.InstanceProvider, + op.EventRecorder, + op.GetClient(), + op.ImageProvider, + ) + + lo.Must0(op.AddHealthzCheck("cloud-provider", aksCloudProvider.LivenessProbe)) + cloudProvider := metrics.Decorate(aksCloudProvider) + + op. + WithControllers(ctx, corecontrollers.NewControllers( + op.Clock, + op.GetClient(), + op.KubernetesInterface, + state.NewCluster(op.Clock, op.GetClient(), cloudProvider), + op.EventRecorder, + cloudProvider, + )...). + WithWebhooks(ctx, corewebhooks.NewWebhooks()...). + WithControllers(ctx, controllers.NewControllers( + ctx, + op.GetClient(), + aksCloudProvider, + op.InstanceProvider, + )...). + WithWebhooks(ctx, webhooks.NewWebhooks()...). + Start(ctx) +} diff --git a/cmd/controller/main_ccp.go b/cmd/controller/main_ccp.go new file mode 100644 index 000000000..e04f1df9d --- /dev/null +++ b/cmd/controller/main_ccp.go @@ -0,0 +1,71 @@ +//go:build ccp + +/* +Portions Copyright (c) Microsoft Corporation. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "github.com/samber/lo" + + "github.com/Azure/karpenter/pkg/cloudprovider" + "github.com/Azure/karpenter/pkg/operator" + + altOperator "github.com/Azure/karpenter/pkg/alt/karpenter-core/pkg/operator" + controllers "github.com/Azure/karpenter/pkg/controllers" + "github.com/aws/karpenter-core/pkg/cloudprovider/metrics" + corecontrollers "github.com/aws/karpenter-core/pkg/controllers" + + // Note the absence of corewebhooks: these pull in knative webhook-related packages and informers in init() + // We don't give cluster-level roles when running in AKS managed mode, so their informers will produce errors and halt all other operations + // corewebhooks "github.com/aws/karpenter-core/pkg/webhooks" + + "github.com/aws/karpenter-core/pkg/controllers/state" +) + +func main() { + //ctx, op := operator.NewOperator(coreoperator.NewOperator()) + ctx, op := operator.NewOperator(altOperator.NewOperator()) + aksCloudProvider := cloudprovider.New( + op.InstanceTypesProvider, + op.InstanceProvider, + op.EventRecorder, + op.GetClient(), + op.ImageProvider, + ) + + lo.Must0(op.AddHealthzCheck("cloud-provider", aksCloudProvider.LivenessProbe)) + cloudProvider := metrics.Decorate(aksCloudProvider) + + op. + WithControllers(ctx, corecontrollers.NewControllers( + op.Clock, + op.GetClient(), + op.KubernetesInterface, + state.NewCluster(op.Clock, op.GetClient(), cloudProvider), + op.EventRecorder, + cloudProvider, + )...). + // WithWebhooks(ctx, corewebhooks.NewWebhooks()...). + WithControllers(ctx, controllers.NewControllers( + ctx, + op.GetClient(), + aksCloudProvider, + op.InstanceProvider, + )...). + // WithWebhooks(ctx, corewebhooks.NewWebhooks()...). + Start(ctx) +} diff --git a/designs/README.md b/designs/README.md new file mode 100644 index 000000000..e6ba35e53 --- /dev/null +++ b/designs/README.md @@ -0,0 +1,7 @@ +# Overview + +Designs in this section are specific to Azure. + +- [AKS node bootstrap](./aks-node-bootstrap.md) +- [GPU selection and bootstrap](./gpu-selection-and-bootstrap.md) +- [K8s node image upgrade](./k8s-node-image-upgrade.md) diff --git a/designs/aks-node-bootstrap.md b/designs/aks-node-bootstrap.md new file mode 100644 index 000000000..ae5c3c6ed --- /dev/null +++ b/designs/aks-node-bootstrap.md @@ -0,0 +1,159 @@ + +# AKS Node Bootstrapping + +## Table of Contents + +- [Overview](#overview) +- [Node bootstrapping](#node-bootstrapping) + - [Node Bootstrapping Variables](#node-bootstrapping-variables) +- [Karpenter configuration sources](#karpenter-configuration-sources) + - [Hardcoded values](#hardcoded-values) + - [Environment variables](#environment-variables) + - [Karpenter global settings](#karpenter-global-settings) + - [Provisioner CR spec](#provisioner-cr-spec) + - [NodeTemplate CR spec](#nodetemplate-cr-spec) + - [Auto-detected values](#auto-detected-values) + - [Computed values](#computed-values) +- [AKS cluster configuration compatibility and drift](#aks-cluster-configuration-compatibility-and-drift) +- [Relevant interfaces and wiring](#relevant-interfaces-and-wiring) + - [Launch template provider](#launch-template-provider) + - [Image provider](#image-provider) + - [Image family](#image-family) + - [Bootstrapper interface](#bootstrapper-interface) + +## Overview + +Azure/AKS provider for Karpenter needs to be able to create standalone VMs that join AKS clusters. This requires both configuring Azure resources (e.g. network interface, VM, etc.) and bootstrapping the VM so that it connects to the cluster. The set of input parameters required for bootstrap is currently quite large, though there is an ongoing effort to reduce it. There are multiple sources for these parameters - from user input, to data from the cluster, to internal defaults. + +The goal of this document is to describe the relevant configuration data flows and their implementation. It starts with AKS VM bootstrapping needs, works its way up to Karpenter configuration mechanisms and sources, touches on AKS cluster configuration compatibility and drift, and then describes how everything is wired together. It stays very close to the source code, using it as the primary reference. The document is primarily focused on node bootstrapping, but it also touches on the configuration of Azure resources. + +Some of the mechanisms described below are specific to AKS. However, the overall configuration flow and wiring should be flexible enough to accommodate other flavors of Kubernetes on Azure in the future. + +> Note: The document uses "Karpenter" for brevity, though in some places "Azure/AKS Cloud Provider for Karpenter" might be more accurate. The distinction is not critical, especially since karpenter-core is used as library, and the build of Azure/AKS Cloud Provider for Karpenter represents Azure/AKS version of Karpenter. + +## Node bootstrapping + +The most common current way of bootstrapping an AKS node it by providing a highly structured [NodeBootstrappingConfiguration](https://github.com/Azure/AgentBaker/blob/3a5c5f2f2c3acd7ebcb82d73352ad6119e1522d6/pkg/agent/datamodel/types.go#L1480) to AgentBaker library to generate [Custom Script Extension](https://learn.microsoft.com/en-us/azure/virtual-machines/extensions/custom-script-linux) (CSE) and Azure [Custom Data](https://learn.microsoft.com/en-us/azure/virtual-machines/custom-data) for the VM. + +A newer, emerging, approach - possible with the latest AKS VM images - is to populate Custom Data with a more streamlined set of parameters, in a well-defined format, without using CSE (one less call, and faster) and without having to use AgentBaker library. This both simplifies the bootstrapping contract, and speeds up the VM bootstrap process. Note that the set of fields and the bootstrapping contract are evolving together with corresponding support in AKS node images. + +This new bootstrapping approach is used by Karpenter. In this context Karpenter is helping to drive and validate the evolution of the bootstrapping contract. Until there is a better name, this document refers to this approach as "Node Bootstrapping Variables". + +### Node Bootstrapping Variables + +When Karpenter creates a VM, Custom Data (often referred to as User Data) is populated with the result of rendering the template in [bootstrap/cse_cmd.sh.gtpl](/pkg/providers/imagefamily/bootstrap/cse_cmd.sh.gtpl). The template is rendered using `NodeBootstrapVariables` structure, defined in [bootstrap/aksbootstrap.go](/pkg/providers/imagefamily/bootstrap/aksbootstrap.go). The structures, variables and helper functions in that file are the primary reference for what ultimately ends up in Custom Data. `(AKS).Script` function - implementing the [Bootstrapper interface](/pkg/providers/imagefamily/bootstrap/bootstrap.go) - is the entry point, and the only inputs flowing in are the `AKS` structure fields. + +The structure fields are populated from a combination of internal defaults, user input, data from the cluster, and computed values. The comments in code categorize `NodeBootstrapVariables` fields into several groups, primarily based on the source of the values, and assign them to one or more configuration sources (described below). This categorization is for informational purposes only, and is not used directly by the code - though it usually dictates the corresponding data flow. + +The number of fields is expected to go down; the goal is to have a minimal set that is required and sufficient to bootstrap a VM and have it join the cluster, and the expectation is that this set will end up being quite small. (Some of the current fields may already be unnecessary, to be verified.) The format of the Custom Data may change as well, with evolution of the contract. + +## Karpenter configuration sources + +The following sources of bootstrap configuration are available to Karpenter: + +* Hardcoded values +* Environment variables +* Karpenter global configuration / ConfigMap (API) +* Provisioner CR spec (API) +* NodeTemplate CR spec (API) +* Auto-detected values +* Computed values + +The source of each parameter is documented in comments to `NodeBootstrapVariables` fields in [aksbootstrap.go](/pkg/providers/imagefamily/bootstrap/aksbootstrap.go). Note that some source assignments are still subject to change, and some are not implemented yet. + +Of theses sources, Karpenter global configuration, Provisioner CR and NodeTemplate CR represent part of the external configuration surface / API, and should be treated accordingly. + +The following sections describe each category in more detail. + + + +### Hardcoded values + +Hardcoded values are used for parameters in one of the following categories: +* unused (but required) +* unsupported by Karpenter (set to some sane default) +* static values that are either not expected to change, or are changing very slowly +* selected defaults + +### Environment variables + +Environment variables are used for global parameters that are needed for bootstrap and already required to be set for other reasons, such as Subscription ID - needed for Azure SDK. + +### Karpenter global settings + +Karpenter uses a ConfigMap with flexible structure for global settings. Part of this configuration is generic, and part is provider specific. See [Concepts/Settings/ConfigMap](https://karpenter.sh/preview/concepts/settings/#configmap) in Karpenter documentation for an overview. Note that this represents part of the external configuration surface / API, and should be treated accordingly. + +For implementation see [pkg/apis/settings/settings.go](/pkg/apis/settings/settings.go). Note that these settings are made available anywhere in provider code via context. + +### Provisioner CR spec + +Some of the bootstrap-relevant configuration comes from the standard [Provisioner](https://karpenter.sh/preview/concepts/provisioners/) CR spec. The relevant fields are: + +* `spec.taints` +* `spec.startupTaints` +* `spec.labels` +* `spec.annotations` +* `spec.requirements` +* `spec.kubeletConfiguration` + +### NodeTemplate CR spec + +Karpenter also supports provider-specific configuration via `NodeTemplate` custom resource. (See public docs for [`AWSNodeTemplate`](https://karpenter.sh/preview/concepts/nodetemplates/).) + +Note that this represents part of the external configuration surface / API, and should be treated as such. + + + + +### Auto-detected values + +A small set of values that are (currently) required for bootstrap are auto-detected or computed: +* Kubernetes version is auto-detected from the cluster (though we will likely add a way to override it) +* CA bundle is obtained from TLS connection to kube-apiserver +* ClusterID is computed from the cluster endpoint FQDN (currently also customizable) + +### Computed values + +A small set of values is fully defined by, and is computed from, other variables. One example is `EnsureNoDupePromiscuousBridge`. (These are good candidates for node image to compute internally in the future.) + +## AKS cluster configuration compatibility and drift + +A subset of the bootstrap configuration is specific to a given AKS cluster. Cluster endpoint is an obvious input. Other examples include CNI plugin used in the cluster, bootstrap token (currently required to join the node to the cluster; expected to go away in the future), Kubernetes version, etc. Some of these values (ideally all but a handful) could be detected automatically, by interrogating the AKS or Kubernetes API. A subset could also change over the lifetime of the cluster, and ideally Karpenter should be able to detect and accommodate these changes. + +For the sake of simplicity, current design does not fully address this. Instead it opts for sane defaults for some values, auto-detection for a few others, and expecting the remaining cluster-specific values to be explicitly specified in Karpenter global settings, without drift detection. This may be addressed in the future, by adding support for cluster configuration detection, refresh and propagation. + +Note that nothing prevents external tooling from automatically configuring Karpenter global settings based on cluster configuration, prior to deployment. The provided `az-patch-skaffold` make target is an example of this. (Skaffold overrides propagate to Helm chart values which populate the global settings ConfigMap.) + + + + + +## Relevant interfaces and wiring + +This section describes some of the concepts and interfaces used in code that help abstract parts of the bootstrap process and wire everything together. + +### Launch template provider + +Launch Template is a useful concept that is used here to represent selected shared parameters needed to create VMs and associated resources. Launch template structure currently carries Custom Data, VM image ID, and desired Azure resource tags. Launch template is obtained and applied by instance provider at the time of VM creation. + +Launch template provider is responsible for the generation of launch templates, using `Machine`, `NodeTemplate` and `InstanceType` as input. It delegates the generation of Custom Data to a particular image family, and the generation of VM image ID to an image provider. A component called Resolver helps pick the right image family and image provider, based on machine, node template and instance type. + +### Image provider + +Image provider is responsible for deciding on the right VM image to use, based on node template, instance type and image family. + +### Image family + +Image family abstracts everything related to a particular image family - such as Ubuntu, which is the only one currently supported. Its primary responsibility is the generation of Custom Data via selected implementation of the Bootstrapper interface. + +### Bootstrapper interface + +`Bootstrapper` interface defined in [providers/imagefamily/bootstrap/bootstrap.go](/pkg/providers/imagefamily/bootstrap/bootstrap.go) abstracts the process of generating bootstrap script (user data / custom data). It is implemented by the provider-specific bootstrapper, currently [AKS bootstrapper](/pkg/providers/imagefamily/bootstrap/aksbootstrap.go) is the only implementation. The interface is ultimately used by launch template to render the bootstrap script for passing into VM's custom data. + +AKS bootstrapper is the one implementing the generation of Custom Data for the new "Node Bootstrap Variables" bootstrap process, described in the beginning of this document. + + + \ No newline at end of file diff --git a/designs/gpu-selection-and-bootstrap.md b/designs/gpu-selection-and-bootstrap.md new file mode 100644 index 000000000..c0db1080f --- /dev/null +++ b/designs/gpu-selection-and-bootstrap.md @@ -0,0 +1,196 @@ +# Overview +This document outlines all of the functional requirements for the preview AKS API for karpenter GPU Support. + +## GPU Provisioning Requirements for Preview + +### 1. Bootstrapping GPU Nodes + +Nodes should successfully bootstrap GPU Nodes for both Generation 1 and Generation 2 Virtual Machine Generations using the Ubuntu2204 Image and the nvidia device plugin. The long term direction of AKS Is to leverage [GPU Operator](https://github.com/NVIDIA/gpu-operator#audience-and-use-cases) for installation of the required software to operate with GPUS in the AKS Cluster. +Karpenter should populate the following fields in the bootstrapping contract for preview: + +- **1a.** GPUNode +- **1b.** GPUDriverVersion +- **1c.** ConfigGPUDriverIfNeeded + +**Note:** The following are explicitly ignored for the preview: + +- **1d.** SGXNode +- **1e.** MIGNode +- **1f.** EnableGPUDevicePluginIfNeeded + +### 2. User-Level Control + +The user, through the requirements API in the provisioner, should be able to limit or exclude GPU instance types using the following dimensions: +- **2a.** vCPU +- **2b.** GPUCount + +### 3. Provisioner Limits and Workload + +Provisioner limits should respect GPU resource types. Workloads should be able to request GPUs via the `nvidia.com/gpu` resource requests. + +**Provisioner Example:** + +```yaml +apiVersion: karpenter.sh/v1alpha5 +kind: Provisioner +metadata: + name: default +spec: + limits: + resources: + cpu: 100 + nvidia.com/gpu: 2 +``` +**Workload Example** +``` +apiVersion: apps/v1 +kind: Deployment +metadata: + name: gpu-nvidia +spec: + containers: + - image: public.ecr.aws/eks-distro/kubernetes/pause:3.2 + name: gpu-nvidia + resources: + limits: + nvidia.com/gpu: "1" + requests: + cpu: "1" + memory: 256M +``` + + +### 4. GPU Driver Testing + +Bootstrapping should work for both graphics and compute workloads. Note that NV Series GPUs target graphics workloads while NC targets compute. The different series use different drivers (GRID for NV and CUDA for NC). We need to: + +- **4a.** Test support for Grid drivers +- **4b.** Test support for CUDA drivers +- **4c.** Test support for Converged drivers + +The way we determine these drivers is via trial and error, and there is not a great way to check other than manually provisioning if a specific sku with a driver, and seeing that it works. + +For Converged drivers they are a mix of multiple drivers installing vanilla cuda drivers will fail to install with opaque errors. +nvidia-bug-report.sh may be helpful, but usually it tells you the pci card id is incompatible. + +So manual trial and error, or leveraging other peoples manual trial and error, and published gpu drivers seems to be the prefered method for approaching this. +see https://github.com/Azure/azhpc-extensions/blob/daaefd78df6f27012caf30f3b54c3bd6dc437652/NvidiaGPU/resources.json for the HPC list of skus and converged drivers, and the driver matrix used by HPC + +**Ownership:** Node SIG is responsible for ensuring successful and functional installation. Our goal is to share a bootstrap contract, and the oblication of a functional successfully bootstrapped vhd relies on the node sig. +Sharing between agentbaker and karpenter is an additional goal, as to which drivers are supported, and for which skus. As well as extending skewer for better GPU Support is also a goal. + +#### 4a. GPU Driver installation +The bootstrap parameter `DRIVER_VERSION` will select the driver version and `ConfigGPUDriverIfNeeded` serves the purpose of telling the VHD to install all gpu helpers such as nvidia-smi etc. +### 5. containerd configuration +GPU's also use a different container runtime, so we need to be able to dynamically modify the containerd configuration we pass into karpenter's bootstrapping contract to include the proper configuration of the gpu settings + +[plugins."io.containerd.grpc.v1.cri".containerd] +Needs to be able to alternate between normal containerd configuration of "io.containerd.runc" and "nvidia-container-runtime" in order to properly connect with device plugin and add the capacity for "nvidia.com/gpu" to our nodes to allow for scheduling of GPU Nodes. + +**Note** We also need to ensure that containerd, and kubelet are using the same cgroup driver. So when leveraging cgroupsv2, we need to make sure we are setting the kubelet flag, we need to update the containerd con + +### 6. Nvidia Device Plugin +The NVIDIA device plugin for Kubernetes is designed to enable GPU support within Kubernetes clusters. Kubernetes, by default, doesn’t recognize GPUs as a native resource. However, using device plugins, one can extend Kubernetes to manage other types of hardware resources, such as GPUs from NVIDIA. + +We will require the customer to install the nvidia device plugin daemonset to enable GPU support through karpenter. + +When a node with Nvidia GPUS joins the cluster, the device plugin detects available gpus and notifies the k8s scheduler that we have a new Allocatable Resource type of `nvidia.com/gpu` along with a resource quanity that can be considered for scheduling. + +Note the device plugin is also reponsible for the allocation of that resource and reporting that other pods can not use that resource and marking it as used by changing the allocatable capacity on the node. + +## Changes to Requirements API + +The Requirements API needs to provide an interface for selecting GPU SKUs, sizes, and preferences. + +## Karpenter Azure Provisioner: Available Labels + +Here are some relevant labels: + +| Label | Description | +|---------------------------------------|-------------------------------------------| +| `karpenter.k8s.azure/sku-family` | Family of the SKU (e.g., standardAv2Family)| +| `karpenter.k8s.azure/sku-subfamily` | Subfamily of the SKU | +| `karpenter.k8s.azure/sku-cpu` | Number of virtual CPUs | +| `karpenter.k8s.azure/sku-accelerator` | Type of accelerator (e.g., Nvidia) | + +**Note:** GPU SKUs usually support only a single hypervisor generation. Explicit image selection is moot in most cases, though there are some exceptions. + +## Karpenter AWS Provisioner: Existing GPU Labels + +| Label | Example Value | Description | +|-----------------------------------------------|---------------|-----------------------------------------------------| +| `karpenter.k8s.aws/instance-gpu-name` | `t4` | Name of the GPU on the instance, if available | +| `karpenter.k8s.aws/instance-gpu-manufacturer` | `nvidia` | Name of the GPU manufacturer | +| `karpenter.k8s.aws/instance-gpu-count` | `1` | Number of GPUs on the instance | +| `karpenter.k8s.aws/instance-gpu-memory` | `16384` | Number of mebibytes of memory on the GPU | + +**Note:** The AKS VHD Validator currently only supports Nvidia GPUs, which will be the focus for the preview. Our apis don't natively support gpu memory, so we will need to add this at a later date for proper support. + +## Proposed New Labels for Azure + +| Selector Label | Values | Description | Where to get the value | +|-------------------------------------------|---------|----------------------------------------------------|----------------------------------------| +| `karpenter.k8s.azure/instance-gpu-name` | `t4` | Name of the GPU on the instance, if available | vmSizeAcceleratorType | +| `karpenter.k8s.azure/instance-gpu-manufacturer` | `nvidia` | GPU Manufacturer | Can be inferred, all Nvidia for preview | +| `karpenter.k8s.azure/instance-gpu-count` | `1` | Number of GPUs on the instance | sku.capabilities["GPU"] | + +## Supported GPU SKUs and Expected Drivers + +This table will outline for each SKU, the supported OS and the driver we commit to passing into the contract. This serves as documentation for which SKUs will be supported in Karpenter preview and should be actively expanded. + +**Note:** For bootstrapping with Nvidia drivers, we need to ensure we are passing the correct driver and OS into the contract. + + +| SKU | Supported OS | GPU Driver | +|-------------------------|--------------|-----------------------------| +| standard_nc6 | Ubuntu | Nvidia470CudaDriver | +| standard_nc12 | Ubuntu | Nvidia470CudaDriver | +| standard_nc24 | Ubuntu | Nvidia470CudaDriver | +| standard_nc24r | Ubuntu | Nvidia470CudaDriver | +| standard_nv6 | Ubuntu | Nvidia510GridDriver | +| standard_nv12 | Ubuntu | Nvidia510GridDriver | +| standard_nv12s_v3 | Ubuntu | Nvidia510GridDriver | +| standard_nv24 | Ubuntu | Nvidia510GridDriver | +| standard_nv24s_v3 | Ubuntu | Nvidia510GridDriver | +| standard_nv24r | Ubuntu | Nvidia510GridDriver | +| standard_nv48s_v3 | Ubuntu | Nvidia510GridDriver | +| standard_nd6s | Ubuntu | Nvidia525CudaDriver | +| standard_nd12s | Ubuntu | Nvidia525CudaDriver | +| standard_nd24s | Ubuntu | Nvidia525CudaDriver | +| standard_nd24rs | Ubuntu | Nvidia525CudaDriver | +| standard_nc6s_v2 | Ubuntu | Nvidia525CudaDriver | +| standard_nc12s_v2 | Ubuntu | Nvidia525CudaDriver | +| standard_nc24s_v2 | Ubuntu | Nvidia525CudaDriver | +| standard_nc24rs_v2 | Ubuntu | Nvidia525CudaDriver | +| standard_nc6s_v3 | Mariner, Ubuntu | Nvidia525CudaDriver | +| standard_nc12s_v3 | Mariner, Ubuntu | Nvidia525CudaDriver | +| standard_nc24s_v3 | Mariner, Ubuntu | Nvidia525CudaDriver | +| standard_nc24rs_v3 | Mariner, Ubuntu | Nvidia525CudaDriver | +| standard_nd40s_v3 | Mariner, Ubuntu | Nvidia525CudaDriver | +| standard_nd40rs_v2 | Mariner, Ubuntu | Nvidia525CudaDriver | +| standard_nc4as_t4_v3 | Mariner, Ubuntu | Nvidia525CudaDriver | +| standard_nc8as_t4_v3 | Mariner, Ubuntu | Nvidia525CudaDriver | +| standard_nc16as_t4_v3 | Mariner, Ubuntu | Nvidia525CudaDriver | +| standard_nc64as_t4_v3 | Mariner, Ubuntu | Nvidia525CudaDriver | +| standard_nd96asr_v4 | Ubuntu | Nvidia525CudaDriver | +| standard_nd112asr_a100_v4| Ubuntu | Nvidia525CudaDriver | +| standard_nd120asr_a100_v4| Ubuntu | Nvidia525CudaDriver | +| standard_nd96amsr_a100_v4| Ubuntu | Nvidia525CudaDriver | +| standard_nd112amsr_a100_v4| Ubuntu | Nvidia525CudaDriver | +| standard_nd120amsr_a100_v4| Ubuntu | Nvidia525CudaDriver | +| standard_nc24ads_a100_v4| Ubuntu | Nvidia525CudaDriver | +| standard_nc48ads_a100_v4| Ubuntu | Nvidia525CudaDriver | +| standard_nc96ads_a100_v4| Ubuntu | Nvidia525CudaDriver | +| standard_ncads_a100_v4 | Ubuntu | Nvidia525CudaDriver | +| standard_nc8ads_a10_v4 | Ubuntu | Nvidia510GridDriver | +| standard_nc16ads_a10_v4 | Ubuntu | Nvidia510GridDriver | +| standard_nc32ads_a10_v4 | Ubuntu | Nvidia510GridDriver | +| standard_nv6ads_a10_v5 | Ubuntu | Nvidia510GridDriver | +| standard_nv12ads_a10_v5 | Ubuntu | Nvidia510GridDriver | +| standard_nv18ads_a10_v5 | Ubuntu | Nvidia510GridDriver | +| standard_nv36ads_a10_v5 | Ubuntu | Nvidia510GridDriver | +| standard_nv36adms_a10_v5| Ubuntu | Nvidia510GridDriver | +| standard_nv72ads_a10_v5 | Ubuntu | Nvidia510GridDriver | +| standard_nd96ams_v4 | Ubuntu | Nvidia525CudaDriver | +| standard_nd96ams_a100_v4| Ubuntu | Nvidia525CudaDriver | + diff --git a/designs/k8s-node-image-upgrade.md b/designs/k8s-node-image-upgrade.md new file mode 100644 index 000000000..019a652ac --- /dev/null +++ b/designs/k8s-node-image-upgrade.md @@ -0,0 +1,317 @@ +# Overview + +## Motivation + +There is no current support for upgrading the k8s version, and/or node image version for karpenter managed nodes. + +Current behavior: +- K8s version is simply pulled from the API server at node creation and if an API server version upgrade happens any older nodes on previous k8s versions are left running on their given k8s version +- If the imageID is specified for a NodeTemplate that will be used. Otherwise, the newest node image version is pulled on node creation, and is then cached for 3 days. There is no upgrade process for node image versions once the node is created + +## Goals + +The goal is to allow for us to control both k8s, and node image upgrade of karpenter controlled nodes, with an automated upgrade option. + +## Non-goals + +- Any other reimage scenario + +## Metrics for Success + +- Karpenter adoption, with high QoS, and low number of customer CRIs + +# Solution Overview + +When considering options, there have been multiple factors important and considered to us: + +- Requirements:
+ + - Committed: + - automated upgrading of k8s [AKS autoupgader], and node image version [Karpenter drift] + - Maintenance windows support for k8s autoupgrading + - Resource Thresholds [Exists] + + - Deferred: + - Maintenance windows support for node image version upgrade + - Deferred as its a risk to preview + - Requires a whole new design space for addressing this + - Changed to post preview support/stretch goal, as its a far easier design space once we have an agentpool representation + - Max Surge [Deferring as it'd require a `karpenter-core` fork, and they already have a design on the way for delivering this within the main line] + - Max surge doesn't currently exist as a settable condition within karpetner core for drift. + - Drift's current batching logic is to deprovision and remove all drifted nodes that don't have pods running on them in one big batch. After that it goes through one by one deprovisioning one candidate at a time, where it will spin up any needed replacement nodes before removing the older one. + - If we wanted to implement it for preview, I don't think it would be a hard section of code to add. There is actually a design in the works for `kaprenter-core`to add batching by node counts, but my concern is that it won't be available before our preview timeline. + - I think if we do implement max surge we should have it only be pod based, as its a better proxy for resources than node count. Note: it may have to round up to the nearest integer to have a minimum of 1 node [0 special cased to 0], which doesn't feel great as it'd be violating the actual setting as a threshold. + - I think the following design within `karpenter-core` will be the best working solution for us here:
+ https://github.com/aws/karpenter-core/pull/516#pullrequestreview-1636663551
+ Although, it's node based which I don't like that much. That said if node based becomes an issue for other customers I expect `karpenter-core` to provide other options for configurability. For preview though, the timeline of it is unknown, and expect it may not be available in time. + - Soak [Requires a more complicated work around, or forking `karpenter-core`] + - Soak currently doesn't exist for karpenter + - Currently after a successful drift it will requeue after 1 millisecond + - I don't think implementing this would be too hard, if added within `karpenter-core`, as we could simply add a wait after the drifting of a batch is complete if its not he last batch. However, it would require forking `karpenter-core`, so plan to defer as we'd like to avoid that. + - The plan is to defer and push for/contribute to `karpenter-core` design/implementation to get this behavioral control within `karpenter-core` directly. + + - Removed: + - Timeout [Removed as a requirement since karpenter retries forever, and so this doesn’t make much sense] + - Note: we should ensure we have good backoffs though for failure. + +The solutions are based on the following assumptions: +1. No new AKS API representation of provisioners or their nodes + - AKS API representation will possibly come down the line. +2. ~~We want k8s, and node image upgrade to follow the same API flow~~ +3. We are using karpenter drift to do the actual upgrades +4. Customers need to have an override to disable automated node image upgrades if desired + - There is one new field on the NodeTemplate to support this: + - Node Image: nodetemplates.imageVersion + - Note: imageVersion is new to allow the customer to lock all nodes associated with a NodeTemplate to an associated imageVersion, and prevent automated upgrading. We don't want to use the current imageId, as we don't want to force them to all be on the same imageId. + +## Option A: Managed + +Upgrade Flow: +- node image: + - Defaults: to node image automated upgrade behavior [Not using AKS autoupgrader. Karpenter detects the new versions itself] + - Allows overriding to a manual upgrade process by setting the imageVersion on the nodetemplates, and locking associated nodes to the given imageVersion, until either the customer manually upgrades by setting a new imageVersion, or removed to migrate back to autoupgrade +- k8s: + - locked to always update and upgrade with the MC/Control plane version, with no override option + +Validation: +- node image: + - no committed validation for customer specified node image + - stretch goal: + - op1: via a webhook + - op2 [preferred]: accepting invalid input, but displaying within status the invalid and last valid version +- k8s: + - validated via AKS RP, since its triggered based on AKS API call to upgrade MC k8s version + +Defaulting: +- karpenter is currently already setup to handle the defaulting of both k8s/node image as presented in this design option + +Pros/Cons: + +- Pros + - Follows the flow of karpenter quite well + - Customers don't have to worry about upgrades of k8s, or node image unless they want to. + - Enabling autoupgrader on a k8s version setting will make the cluster fully managed for upgrades. + - No need for an agentpool representation, or any new AKS API representation + - Get to use the AKS RP validation on k8s version + - Defaults customers to automated upgrades of node image versions which will help them get security updates. + - Allows downgrading node image version? + +- Cons + - Customer has to specify versions themselves when overriding. i.e. finding the node image version they want + - Allows downgrading of node image version? + - k8s, and node image have different upgrade flows + +## Option B: Manual [Upgrade via CRs] + +Upgrade Flow: +- node image: + - Default: CR gets locked to the latest node image version at the time of creation. + - Upgrade: Customers can manually trigger upgrades by specifying a node image version, and/or triggering an empty field to retrigger the default fill the upgrade to latest +- k8s: + - Default: CR gets locked to the on create MC/API server version. + - Upgrade: Customers can manually trigger upgrades by specifying a new k8s version. + +Validation/Defaulting: +- Validating webhook: handles k8s, and node image version validation + - Requires a supported k8s/node image version, and k8s version to be <= MC/Control Plane version + - Note: we are missing validation for the MC/Control Plane not getting too far ahead of the Provisioner, which we have similar code in RP for with agentpools. + - Also, missing other potentially re-usable validation/restrictions like jumping too many k8s versions at once? +- Defaulting webhook: handles defaulting k8s version to API-server version, and node image to latest. + - Note: this would be a path for customers to upgrade k8s to MC/Control Plane version, and node image to latest, if they sent an empty field that would be defaulted. +- The other option for validation, and defaulting is skipping it entirely for preview + - Note: We might end up not having a choice within this with how the CCP webhooks work as we might not be able to inject this validation logic we want. + - If that is the case we still could add a reconciler for defaulting the k8s/node image versions though. Alex Leites' idea + +Pros/Cons: + +- Pros + - All done directly within karpenter. + - k8s and node image version upgrade both follow the same API flow through CRs + - Allows downgrading of k8s, and node image version? + +- Cons + - harder, and/or less validation + - No automated upgrades, until we have agentpool representation + - Customer has to specify versions themselves. i.e. finding the node image version, k8s version they want, etc, unless defaulting is supported. + - Allows downgrading of k8s, and node image version? + +## Choice + +We are going with Option A (Managed) for a couple reasons: +- NAP clusters are meant to be a more managed/automated offering. Due to this it makes sense for the clusters to have automated security node image upgrades by default. + - Supporting automated upgrading is a committed requirement +- AKS RP is already going to default create clusters with node image autoupgrade equivalent on as well, so this is aligning with the existing AKS RP design as a whole. +- This option still allows customers to override the higher frequency node image upgrades if needed, but makes it as managed by default + - The plan is to allow for more configurability if/once we get to a "nodepool" concept within the AKS API + +# High Level Design + +Drift: +1. Karpenter will detect associated nodes to the NodeTemplate have drifted on k8s, or node image version and an upgrade is required +2. The nodes will be replaced, until all nodes associated with the NodeTemplate are on the correct k8s, and/or node image versions + +node image: +- Automated: + 1. A new node image version is released + 2. karpenter will detect the new node image version within ~3 days [caching length for latest node image version] + 3. `Drift` +- Overridden Manual: + 1. customer sets a new node image version within the NodeTemplate CR + 2. `Drift` + +k8s: +1. Customer triggers a k8s upgrade of the MC +2. API server is upgraded + - Note: should there be a delay here? +3. `Drift` + +# Detailed Design + +Modifying NodeTemplate [API]: +- **NEW**: nodetemplates.imageVersion + +Features: +- **NEW**: Drift enabled for all karpenter clusters + +## Api and Data Model Change + +Modifying NodeTemplate [API]: +- **NEW**: nodetemplates.imageVersion + +**Open Question**: + +1. Q: Do we fork `karpenter-core`? + - Is `Max Surge` a hard requirement for preview, and will there be a satisfying `karpenter-core` option by preview? + - A: No, `Max Surge`, and `Soak` have been deferred as to not require a fork +2. Q: Do we do validation on node image version for preview? + - A: Moved to a stretch goal +3. Will enabling drift for all clusters get as any behavior we don't want? +4. How do we do monitoring? I believe Bryce was setting up monitoring for karpenter? +5. What solution will we have for node bootstrapping breaking on MC k8s version upgrade? + - Node bootstrapping breaks on k8s version upgrades of the API server. This might get fixed by secure bootstrapping if available before preview, but if its not than we need some other sort of work around. We have options that should be doable, but need to keep an eye on the solution space here. + +### **Testing Plan** + +- E2E tests: + - Drift feature works in general + - Should not drift if feature is disabled + - Drift on k8s + - Drift on node image +- Unit tests: + - Any new core functionally should have supporting tests + +### **Perf & Scale Consideration** + +If Max Surge remains at 1, this will be an issue for scale, as cluster upgrades will then take a long time to complete. + +I would like if Max Surge is controllable, and has a reasonable baseline default of 10%, or 30% of whatever resources its based on (pods, nodes, etc) [10-30% are magic numbers based on experience with RP upgrades], but this is dependent upon if we fork `karpenter-core`, and/or what design options are available to us. + +### **Monitoring** + +TBD - Need to talk with Bryce + +### **Deployment** + +General deployment will be bundled with the karpenter deployment. + +If a validation webhook is done, my understanding the CCP webhooks run as a sidecar for the API server. Unsure how that deployment might be different + +## Security and Privacy + +### Authentication and Authorization + +Are there any concerns around customers having an access to an upgrade path through the kubernetes API as opposed to the AKS API? + +Would customers ability to define permissions for their teams be restricted in ways they might not be satisfied with? + +### Auditing + +Unsure. Does anything have to be done here? + +### Privacy + +Don't think there's anything special that has to be done here. + +## Observability Considerations + +Dependent upon the open question on monitoring, and how we can do about monitoring upgrades, and their failures. + +From that we should define metrics, accepted thresholds and alerts based on the monitoring data we have. + +## Compatibility Consideration + +Main potential issue with compatibility would be if we add a validation webhook and if they are non-functional with karpenter deployed within the CCP + +## COGS Impacts + +I don't think so. The only real change here would be using up some of our quota for API calls as far as I can tell. + +If a validation webhook is added, its good to consider the resources it'd be allotted and using + +## Rollout Plan + +This will rollout with the karpenter preview launch. + +## Migration + +This is more a question of the migration plan for old clusters to karpenter in general. I don't think there's anything specific for upgrade here, unless we'd need to ensure they're on the MC/API server's k8s version, and possibly fill in the node image version on the CR if we weren't wanting to put them on autoupgrade for node image versions. + +## Supportability Considerations + +I think this will mostly just be whatever monitoring, alerting, and TSGs we end up getting out of the open question around monitoring. + +# Meeting Notes[^6] + +**9-22:**
+Updated design heavily from Option B to Option A. + +The original design had gone under the assumed requirement that we weren't providing any autoupgrade solution by default, and so something along those lines would come around when the AKS API was introduced. This assumption had been a misunderstanding, which made Option A available and the new preferable option. + +This doc has been adjusted accordingly + +**Offline chats:**
+Long discussions around the potential support for maintenance windows for node image version automated upgrading. There were a few different control patterns: +1. autoupgrader handles the detection/triggering + - Already had the handling for respecting maintenance windows + - Would need kubeclient access, or new API access to detect/trigger upgrades on karpenter CRs +2. Karpenter controlled + - Would need to have a way of either fetching the maintenance windows, or having them pushed to it + - Would block customer’s non-automated upgrades as well. + +# References + +- Mega Issue: Deprovisioning Controls
+https://github.com/aws/karpenter/issues/1738 + +- Drift Max Surge design within `karpenter-core`:
+https://github.com/aws/karpenter-core/pull/516#pullrequestreview-1636663551 + +- POC PR for drift supporting k8s version upgrade:
+https://github.com/Azure/karpenter/pull/262 + +--- + +From template: + +[^1]: Coordinator is in charge of helping the author to find right + person to answer technical questions (e.g if I do this, is there any + big concern from RP's perspective?), as well as finding the right + person to approve the design from the point of their domain. The + coordinator is also in charge of setting up review meeting and + making sure all the right stakeholders are involved in the design + discussion and review. + +[^3]: This list is dynamic. The Sponsor, Airlock, Security rows should + always be there, but the remaining rows are as needed. The + Coordinator's job is to make sure all required rows are filled. + +[^4]: Sponsor is the EM who allocates resources to accomplish + engineering work. For cross team collaboration, this needs to be + approved by all EM's of involved team. + +[^5]: For each SIG, the SIG lead is in charge of either approving the + design doc by carefully reviewing it or assigning a tech leads that + are domain expert in that SIG to review and approve this doc + +[^6]: Q&A style meeting notes from desgin review meeting to capture + todos diff --git a/examples/README.md b/examples/README.md new file mode 100644 index 000000000..7ba0ffd95 --- /dev/null +++ b/examples/README.md @@ -0,0 +1,11 @@ +## Examples + +Checkout example Provisioner and workload specs to demo with Karpenter. + +## Usage: + +Provisioner specs expect a `CLUSTER_NAME` environment variable to be set to your cluster name. You can use the following command to substitute the environment variable and `kubectl apply` to your cluster: + +``` +CLUSTER_NAME= envsubst < provisioner/general-purpose-azure.yaml | kubectl apply -f - +``` diff --git a/examples/grafana-values.yaml b/examples/grafana-values.yaml new file mode 100644 index 000000000..acbfb818d --- /dev/null +++ b/examples/grafana-values.yaml @@ -0,0 +1,27 @@ +datasources: + datasources.yaml: + apiVersion: 1 + datasources: + - name: Prometheus + type: prometheus + version: 1 + url: http://prometheus-server:80 + access: proxy +dashboardProviders: + dashboardproviders.yaml: + apiVersion: 1 + providers: + - name: 'default' + orgId: 1 + folder: '' + type: file + disableDeletion: false + editable: true + options: + path: /var/lib/grafana/dashboards/default +dashboards: + default: + capacity-dashboard: + url: https://karpenter.sh/preview/getting-started/getting-started-with-karpenter/karpenter-capacity-dashboard.json + performance-dashboard: + url: https://karpenter.sh/preview/getting-started/getting-started-with-karpenter/karpenter-performance-dashboard.json diff --git a/examples/prometheus-values.yaml b/examples/prometheus-values.yaml new file mode 100644 index 000000000..4cd784953 --- /dev/null +++ b/examples/prometheus-values.yaml @@ -0,0 +1,20 @@ +alertmanager: + persistentVolume: + enabled: false + +server: + fullnameOverride: prometheus-server + persistentVolume: + enabled: false + +extraScrapeConfigs: | + - job_name: karpenter + kubernetes_sd_configs: + - role: endpoints + namespaces: + names: + - karpenter + relabel_configs: + - source_labels: [__meta_kubernetes_endpoint_port_name] + regex: http-metrics + action: keep diff --git a/examples/v1beta1/general-purpose.yaml b/examples/v1beta1/general-purpose.yaml new file mode 100644 index 000000000..b7a4e72d9 --- /dev/null +++ b/examples/v1beta1/general-purpose.yaml @@ -0,0 +1,40 @@ +# This example NodePool will provision general purpose instances +--- +apiVersion: karpenter.sh/v1beta1 +kind: NodePool +metadata: + name: general-purpose + annotations: + kubernetes.io/description: "General purpose NodePool for generic workloads" +spec: + disruption: + expireAfter: Never + template: + spec: + requirements: + - key: kubernetes.io/arch + operator: In + values: ["amd64"] + - key: kubernetes.io/os + operator: In + values: ["linux"] + - key: karpenter.azure.com/sku-hyperv-generation + operator: In + values: ["2"] + - key: karpenter.sh/capacity-type + operator: In + values: ["on-demand"] + - key: karpenter.azure.com/sku-family + operator: In + values: [D] + nodeClassRef: + name: default +--- +apiVersion: karpenter.azure.com/v1alpha2 +kind: AKSNodeClass +metadata: + name: default + annotations: + kubernetes.io/description: "General purpose AKSNodeClass for running Ubuntu2204 nodes" +spec: + imageFamily: Ubuntu2204 diff --git a/examples/v1beta1/multi-arch.yaml b/examples/v1beta1/multi-arch.yaml new file mode 100644 index 000000000..24a99127b --- /dev/null +++ b/examples/v1beta1/multi-arch.yaml @@ -0,0 +1,70 @@ +# This example NodePool will provision general purpose instances for either amd64 or arm64 workloads +--- +apiVersion: karpenter.sh/v1beta1 +kind: NodePool +metadata: + name: arm64 + annotations: + kubernetes.io/description: "Nodepool for arm64 workloads" +spec: + disruption: + expireAfter: Never + template: + spec: + requirements: + - key: kubernetes.io/arch + operator: In + values: ["arm64"] + - key: karpenter.azure.com/sku-hyperv-generation + operator: In + values: ["2"] + - key: kubernetes.io/os + operator: In + values: ["linux"] + - key: karpenter.sh/capacity-type + operator: In + values: ["on-demand"] + - key: karpenter.azure.com/sku-family + operator: In + values: [D] + nodeClassRef: + name: default +--- +apiVersion: karpenter.sh/v1beta1 +kind: NodePool +metadata: + name: amd64 + annotations: + kubernetes.io/description: "Nodepool for amd64 workloads" +spec: + disruption: + expireAfter: Never + template: + spec: + requirements: + - key: kubernetes.io/arch + operator: In + values: ["amd64"] + - key: karpenter.azure.com/sku-hyperv-generation + operator: In + values: ["2"] + - key: kubernetes.io/os + operator: In + values: ["linux"] + - key: karpenter.sh/capacity-type + operator: In + values: ["on-demand"] + - key: karpenter.azure.com/sku-family + operator: In + values: [D] + nodeClassRef: + name: default +--- +apiVersion: karpenter.azure.com/v1alpha2 +kind: AKSNodeClass +metadata: + name: default + annotations: + kubernetes.io/description: "General purpose AKSNodeClass for running Ubuntu2204 nodes" +spec: + imageFamily: Ubuntu2204 diff --git a/examples/v1beta1/system-surge.yaml b/examples/v1beta1/system-surge.yaml new file mode 100644 index 000000000..f9f3ce06c --- /dev/null +++ b/examples/v1beta1/system-surge.yaml @@ -0,0 +1,47 @@ +# This example NodePool will provision general purpose instances +--- +apiVersion: karpenter.sh/v1beta1 +kind: NodePool +metadata: + name: system-surge + annotations: + kubernetes.io/description: "Surge capacity pool for system pod pressure" +spec: + disruption: + expireAfter: Never + template: + metadata: + labels: + kubernetes.azure.com/mode: "system" + spec: + taints: + - key: "CriticalAddonsOnly" + value: "true" + effect: "NoSchedule" + requirements: + - key: kubernetes.io/arch + operator: In + values: ["amd64"] + - key: karpenter.azure.com/sku-hyperv-generation + operator: In + values: ["2"] + - key: kubernetes.io/os + operator: In + values: ["linux"] + - key: karpenter.sh/capacity-type + operator: In + values: ["on-demand"] + - key: karpenter.azure.com/sku-family + operator: In + values: [D] + nodeClassRef: + name: system-surge +--- +apiVersion: karpenter.azure.com/v1alpha2 +kind: AKSNodeClass +metadata: + name: system-surge + annotations: + kubernetes.io/description: "General purpose AKSNodeClass for running Ubuntu2204 nodes" +spec: + imageFamily: Ubuntu2204 diff --git a/examples/workloads/device-plugin.yaml b/examples/workloads/device-plugin.yaml new file mode 100644 index 000000000..9db79ae28 --- /dev/null +++ b/examples/workloads/device-plugin.yaml @@ -0,0 +1,42 @@ +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: nvidia-device-plugin-daemonset + namespace: kube-system +spec: + selector: + matchLabels: + name: nvidia-device-plugin-ds + updateStrategy: + type: RollingUpdate + template: + metadata: + labels: + name: nvidia-device-plugin-ds + spec: + tolerations: + - key: nvidia.com/gpu + operator: Exists + effect: NoSchedule + # Mark this pod as a critical add-on; when enabled, the critical add-on + # scheduler reserves resources for critical add-on pods so that they can + # be rescheduled after a failure. + # See https://kubernetes.io/docs/tasks/administer-cluster/guaranteed-scheduling-critical-addon-pods/ + priorityClassName: "system-node-critical" + containers: + - image: nvcr.io/nvidia/k8s-device-plugin:v0.14.1 + name: nvidia-device-plugin-ctr + env: + - name: FAIL_ON_INIT_ERROR + value: "false" + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: ["ALL"] + volumeMounts: + - name: device-plugin + mountPath: /var/lib/kubelet/device-plugins + volumes: + - name: device-plugin + hostPath: + path: /var/lib/kubelet/device-plugins diff --git a/examples/workloads/inflate.yaml b/examples/workloads/inflate.yaml new file mode 100644 index 000000000..9bf75d133 --- /dev/null +++ b/examples/workloads/inflate.yaml @@ -0,0 +1,21 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: inflate +spec: + replicas: 0 + selector: + matchLabels: + app: inflate + template: + metadata: + labels: + app: inflate + spec: + containers: + - image: mcr.microsoft.com/oss/kubernetes/pause:3.6 + name: inflate + resources: + requests: + cpu: "1" + memory: 256M diff --git a/examples/workloads/samples-mnist.yaml b/examples/workloads/samples-mnist.yaml new file mode 100644 index 000000000..0e263fff2 --- /dev/null +++ b/examples/workloads/samples-mnist.yaml @@ -0,0 +1,26 @@ +apiVersion: batch/v1 +kind: Job +metadata: + labels: + app: samples-tf-mnist-demo + name: samples-tf-mnist-demo +spec: + template: + metadata: + labels: + app: samples-tf-mnist-demo + spec: + containers: + - name: samples-tf-mnist-demo + image: mcr.microsoft.com/azuredocs/samples-tf-mnist-demo:gpu + args: ["--max_steps", "500"] + imagePullPolicy: IfNotPresent + resources: + limits: + nvidia.com/gpu: "1" + restartPolicy: OnFailure + tolerations: + - key: "sku" + operator: "Equal" + value: "gpu" + effect: "NoSchedule" diff --git a/go.mod b/go.mod new file mode 100644 index 000000000..7dc3ef35b --- /dev/null +++ b/go.mod @@ -0,0 +1,143 @@ +module github.com/Azure/karpenter + +go 1.21 + +require ( + github.com/Azure/azure-kusto-go v0.14.0 + github.com/Azure/azure-sdk-for-go v68.0.0+incompatible + github.com/Azure/azure-sdk-for-go-extensions v0.1.2 + github.com/Azure/azure-sdk-for-go/sdk/azcore v1.8.0-beta.1 + github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.3.1 + github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute v1.0.0 + github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute/v5 v5.1.0 + github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/network/armnetwork v1.1.0 + github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resourcegraph/armresourcegraph v0.8.1 + github.com/Azure/go-armbalancer v0.0.2 + github.com/Azure/go-autorest/autorest v0.11.29 + github.com/Azure/go-autorest/autorest/adal v0.9.23 + github.com/Azure/go-autorest/autorest/to v0.4.0 + github.com/Azure/skewer v0.0.19 + github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137 + github.com/aws/karpenter-core v0.32.1 + github.com/go-logr/logr v1.2.4 + github.com/go-logr/zapr v1.2.4 + github.com/go-playground/validator/v10 v10.13.0 + github.com/imdario/mergo v0.3.16 + github.com/mitchellh/hashstructure/v2 v2.0.2 + github.com/onsi/ginkgo/v2 v2.13.0 + github.com/onsi/gomega v1.29.0 + github.com/patrickmn/go-cache v2.1.0+incompatible + github.com/prometheus/client_golang v1.17.0 + github.com/samber/lo v1.38.1 + github.com/stretchr/testify v1.8.4 + go.uber.org/multierr v1.11.0 + go.uber.org/zap v1.26.0 + golang.org/x/crypto v0.14.0 + k8s.io/api v0.28.3 + k8s.io/apiextensions-apiserver v0.28.3 + k8s.io/apimachinery v0.28.3 + k8s.io/client-go v0.28.3 + k8s.io/klog/v2 v2.100.1 + k8s.io/utils v0.0.0-20230726121419-3b25d923346b + knative.dev/pkg v0.0.0-20231010144348-ca8c009405dd + sigs.k8s.io/cloud-provider-azure v1.28.2 + sigs.k8s.io/controller-runtime v0.16.3 +) + +require ( + contrib.go.opencensus.io/exporter/ocagent v0.7.1-0.20200907061046-05415f1de66d // indirect + contrib.go.opencensus.io/exporter/prometheus v0.4.2 // indirect + github.com/Azure/azure-sdk-for-go/sdk/internal v1.3.0 // indirect + github.com/Azure/go-autorest v14.2.0+incompatible // indirect + github.com/Azure/go-autorest/autorest/date v0.3.0 // indirect + github.com/Azure/go-autorest/autorest/mocks v0.4.2 // indirect + github.com/Azure/go-autorest/autorest/validation v0.3.1 // indirect + github.com/Azure/go-autorest/logger v0.2.1 // indirect + github.com/Azure/go-autorest/tracing v0.6.0 // indirect + github.com/AzureAD/microsoft-authentication-library-for-go v1.1.1 // indirect + github.com/Pallinder/go-randomdata v1.2.0 // indirect + github.com/avast/retry-go v3.0.0+incompatible // indirect + github.com/beorn7/perks v1.0.1 // indirect + github.com/blang/semver/v4 v4.0.0 // indirect + github.com/blendle/zapdriver v1.3.1 // indirect + github.com/census-instrumentation/opencensus-proto v0.4.1 // indirect + github.com/cespare/xxhash/v2 v2.2.0 // indirect + github.com/davecgh/go-spew v1.1.1 // indirect + github.com/deckarep/golang-set v1.8.0 // indirect + github.com/emicklei/go-restful/v3 v3.11.0 // indirect + github.com/evanphx/json-patch/v5 v5.7.0 // indirect + github.com/fsnotify/fsnotify v1.6.0 // indirect + github.com/go-kit/log v0.2.1 // indirect + github.com/go-logfmt/logfmt v0.6.0 // indirect + github.com/go-openapi/jsonpointer v0.20.0 // indirect + github.com/go-openapi/jsonreference v0.20.2 // indirect + github.com/go-openapi/swag v0.22.4 // indirect + github.com/go-playground/locales v0.14.1 // indirect + github.com/go-playground/universal-translator v0.18.1 // indirect + github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 // indirect + github.com/gobuffalo/flect v1.0.2 // indirect + github.com/gogo/protobuf v1.3.2 // indirect + github.com/golang-jwt/jwt/v4 v4.5.0 // indirect + github.com/golang-jwt/jwt/v5 v5.0.0 // indirect + github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect + github.com/golang/mock v1.6.0 // indirect + github.com/golang/protobuf v1.5.3 // indirect + github.com/google/gnostic-models v0.6.8 // indirect + github.com/google/go-cmp v0.6.0 // indirect + github.com/google/gofuzz v1.2.0 // indirect + github.com/google/pprof v0.0.0-20230926050212-f7f687d19a98 // indirect + github.com/google/uuid v1.3.1 // indirect + github.com/grpc-ecosystem/grpc-gateway/v2 v2.18.0 // indirect + github.com/hashicorp/golang-lru v1.0.2 // indirect + github.com/inconshreveable/mousetrap v1.1.0 // indirect + github.com/josharian/intern v1.0.0 // indirect + github.com/json-iterator/go v1.1.12 // indirect + github.com/kelseyhightower/envconfig v1.4.0 // indirect + github.com/kylelemons/godebug v1.1.0 // indirect + github.com/leodido/go-urn v1.2.3 // indirect + github.com/mailru/easyjson v0.7.7 // indirect + github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect + github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect + github.com/modern-go/reflect2 v1.0.2 // indirect + github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect + github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8 // indirect + github.com/pkg/errors v0.9.1 // indirect + github.com/pmezard/go-difflib v1.0.0 // indirect + github.com/prometheus/client_model v0.5.0 // indirect + github.com/prometheus/common v0.44.0 // indirect + github.com/prometheus/procfs v0.12.0 // indirect + github.com/prometheus/statsd_exporter v0.24.0 // indirect + github.com/shopspring/decimal v1.3.1 // indirect + github.com/spf13/cobra v1.7.0 // indirect + github.com/spf13/pflag v1.0.5 // indirect + go.opencensus.io v0.24.0 // indirect + go.uber.org/automaxprocs v1.5.3 // indirect + golang.org/x/exp v0.0.0-20231006140011-7918f672742d // indirect + golang.org/x/net v0.17.0 // indirect + golang.org/x/oauth2 v0.13.0 // indirect + golang.org/x/sync v0.4.0 // indirect + golang.org/x/sys v0.13.0 // indirect + golang.org/x/term v0.13.0 // indirect + golang.org/x/text v0.13.0 // indirect + golang.org/x/time v0.3.0 // indirect + golang.org/x/tools v0.14.0 // indirect + gomodules.xyz/jsonpatch/v2 v2.4.0 // indirect + google.golang.org/api v0.146.0 // indirect + google.golang.org/appengine v1.6.8 // indirect + google.golang.org/genproto v0.0.0-20231009173412-8bfb1ae86b6c // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20231009173412-8bfb1ae86b6c // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20231009173412-8bfb1ae86b6c // indirect + google.golang.org/grpc v1.58.3 // indirect + google.golang.org/protobuf v1.31.0 // indirect + gopkg.in/inf.v0 v0.9.1 // indirect + gopkg.in/yaml.v2 v2.4.0 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect + k8s.io/cloud-provider v0.28.3 // indirect + k8s.io/component-base v0.28.3 // indirect + k8s.io/component-helpers v0.28.3 // indirect + k8s.io/csi-translation-lib v0.28.3 // indirect + k8s.io/kube-openapi v0.0.0-20231010175941-2dd684a91f00 // indirect + sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect + sigs.k8s.io/structured-merge-diff/v4 v4.3.0 // indirect + sigs.k8s.io/yaml v1.3.0 // indirect +) diff --git a/go.sum b/go.sum new file mode 100644 index 000000000..b10ea7ff2 --- /dev/null +++ b/go.sum @@ -0,0 +1,835 @@ +cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU= +cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6AU= +cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY= +cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTjc= +cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0= +cloud.google.com/go v0.50.0/go.mod h1:r9sluTvynVuxRIOHXQEHMFffphuXHOMZMycpNR5e6To= +cloud.google.com/go v0.52.0/go.mod h1:pXajvRH/6o3+F9jDHZWQ5PbGhn+o8w9qiu/CffaVdO4= +cloud.google.com/go v0.53.0/go.mod h1:fp/UouUEsRkN6ryDKNW/Upv/JBKnv6WDthjR6+vze6M= +cloud.google.com/go v0.54.0/go.mod h1:1rq2OEkV3YMf6n/9ZvGWI3GWw0VoqH/1x2nd8Is/bPc= +cloud.google.com/go v0.56.0/go.mod h1:jr7tqZxxKOVYizybht9+26Z/gUq7tiRzu+ACVAMbKVk= +cloud.google.com/go v0.57.0/go.mod h1:oXiQ6Rzq3RAkkY7N6t3TcE6jE+CIBBbA36lwQ1JyzZs= +cloud.google.com/go v0.62.0/go.mod h1:jmCYTdRCQuc1PHIIJ/maLInMho30T/Y0M4hTdTShOYc= +cloud.google.com/go v0.65.0/go.mod h1:O5N8zS7uWy9vkA9vayVHs65eM1ubvY4h553ofrNHObY= +cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= +cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE= +cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc= +cloud.google.com/go/bigquery v1.5.0/go.mod h1:snEHRnqQbz117VIFhE8bmtwIDY80NLUZUMb4Nv6dBIg= +cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4gLoIoXIAPc= +cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ= +cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= +cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk= +cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= +cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw= +cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA= +cloud.google.com/go/pubsub v1.3.1/go.mod h1:i+ucay31+CNRpDW4Lu78I4xXG+O1r/MAHgjpRVR+TSU= +cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw= +cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos= +cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk= +cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs= +cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= +contrib.go.opencensus.io/exporter/ocagent v0.7.1-0.20200907061046-05415f1de66d h1:LblfooH1lKOpp1hIhukktmSAxFkqMPFk9KR6iZ0MJNI= +contrib.go.opencensus.io/exporter/ocagent v0.7.1-0.20200907061046-05415f1de66d/go.mod h1:IshRmMJBhDfFj5Y67nVhMYTTIze91RUeT73ipWKs/GY= +contrib.go.opencensus.io/exporter/prometheus v0.4.2 h1:sqfsYl5GIY/L570iT+l93ehxaWJs2/OwXtiWwew3oAg= +contrib.go.opencensus.io/exporter/prometheus v0.4.2/go.mod h1:dvEHbiKmgvbr5pjaF9fpw1KeYcjrnC1J8B+JKjsZyRQ= +dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= +github.com/Azure/azure-kusto-go v0.14.0 h1:5XVmjh5kVgsm2scpsWisJ6Q1ZgWHJcIOPCZC1gatD4I= +github.com/Azure/azure-kusto-go v0.14.0/go.mod h1:wSmXIsQwBVPHDNsSQsX98nuc12VyvxoNHQa2q9t1Ce0= +github.com/Azure/azure-sdk-for-go v68.0.0+incompatible h1:fcYLmCpyNYRnvJbPerq7U0hS+6+I79yEDJBqVNcqUzU= +github.com/Azure/azure-sdk-for-go v68.0.0+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc= +github.com/Azure/azure-sdk-for-go-extensions v0.1.2 h1:Wu5hT6KxmcwarIeam3dP+aTZmCpqbZBC9Zzr2xJlFt8= +github.com/Azure/azure-sdk-for-go-extensions v0.1.2/go.mod h1:ysQIgudolR9hbYTQ08G/PTeK/ukOHkDDJnpS9Xrk3ns= +github.com/Azure/azure-sdk-for-go/sdk/azcore v1.8.0-beta.1 h1:8t6ZZtkOCl+rx7uBn40Nj62ABVGkXK69U/En44wJIlE= +github.com/Azure/azure-sdk-for-go/sdk/azcore v1.8.0-beta.1/go.mod h1:bjGvMhVMb+EEm3VRNQawDMUyMMjo+S5ewNjflkep/0Q= +github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.3.1 h1:LNHhpdK7hzUcx/k1LIcuh5k7k1LGIWLQfCjaneSj7Fc= +github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.3.1/go.mod h1:uE9zaUfEQT/nbQjVi2IblCG9iaLtZsuYZ8ne+PuQ02M= +github.com/Azure/azure-sdk-for-go/sdk/internal v1.3.0 h1:sXr+ck84g/ZlZUOZiNELInmMgOsuGwdjjVkEIde0OtY= +github.com/Azure/azure-sdk-for-go/sdk/internal v1.3.0/go.mod h1:okt5dMMTOFjX/aovMlrjvvXoPMBVSPzk9185BT0+eZM= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute v1.0.0 h1:/Di3vB4sNeQ+7A8efjUVENvyB945Wruvstucqp7ZArg= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute v1.0.0/go.mod h1:gM3K25LQlsET3QR+4V74zxCsFAy0r6xMNN9n80SZn+4= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute/v5 v5.1.0 h1:Sg/D8VuUQ+bw+FOYJF+xRKcwizCOP13HL0Se8pWNBzE= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute/v5 v5.1.0/go.mod h1:Kyqzdqq0XDoCm+o9aZ25wZBmBUBzPBzPAj1R5rYsT6I= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/internal v1.1.2 h1:mLY+pNLjCUeKhgnAJWAKhEUQM+RJQo2H1fuGSw1Ky1E= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/internal v1.1.2/go.mod h1:FbdwsQ2EzwvXxOPcMFYO8ogEc9uMMIj3YkmCdXdAFmk= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/network/armnetwork v1.1.0 h1:QM6sE5k2ZT/vI5BEe0r7mqjsUSnhVBFbOsVkEuaEfiA= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/network/armnetwork v1.1.0/go.mod h1:243D9iHbcQXoFUtgHJwL7gl2zx1aDuDMjvBZVGr2uW0= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resourcegraph/armresourcegraph v0.8.1 h1:nGiU2ovpbtkcC3x+g/wNHV4S9TOIYe2/yOVAj3wiGHI= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resourcegraph/armresourcegraph v0.8.1/go.mod h1:T3ZgvD1aRKu12mEA0fU3PPvI7V0Nh0wzIdK0QMBhf0Y= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resources/armresources v1.1.1 h1:7CBQ+Ei8SP2c6ydQTGCCrS35bDxgTMfoP2miAwK++OU= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resources/armresources v1.1.1/go.mod h1:c/wcGeGx5FUPbM/JltUYHZcKmigwyVLJlDq+4HdtXaw= +github.com/Azure/go-armbalancer v0.0.2 h1:NVnxsTWHI5/fEzL6k6TjxPUfcB/3Si3+HFOZXOu0QtA= +github.com/Azure/go-armbalancer v0.0.2/go.mod h1:yTg7MA/8YnfKQc9o97tzAJ7fbdVkod1xGsIvKmhYPRE= +github.com/Azure/go-autorest v14.2.0+incompatible h1:V5VMDjClD3GiElqLWO7mz2MxNAK/vTfRHdAubSIPRgs= +github.com/Azure/go-autorest v14.2.0+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24= +github.com/Azure/go-autorest/autorest v0.11.29 h1:I4+HL/JDvErx2LjyzaVxllw2lRDB5/BT2Bm4g20iqYw= +github.com/Azure/go-autorest/autorest v0.11.29/go.mod h1:ZtEzC4Jy2JDrZLxvWs8LrBWEBycl1hbT1eknI8MtfAs= +github.com/Azure/go-autorest/autorest/adal v0.9.22/go.mod h1:XuAbAEUv2Tta//+voMI038TrJBqjKam0me7qR+L8Cmk= +github.com/Azure/go-autorest/autorest/adal v0.9.23 h1:Yepx8CvFxwNKpH6ja7RZ+sKX+DWYNldbLiALMC3BTz8= +github.com/Azure/go-autorest/autorest/adal v0.9.23/go.mod h1:5pcMqFkdPhviJdlEy3kC/v1ZLnQl0MH6XA5YCcMhy4c= +github.com/Azure/go-autorest/autorest/date v0.3.0 h1:7gUk1U5M/CQbp9WoqinNzJar+8KY+LPI6wiWrP/myHw= +github.com/Azure/go-autorest/autorest/date v0.3.0/go.mod h1:BI0uouVdmngYNUzGWeSYnokU+TrmwEsOqdt8Y6sso74= +github.com/Azure/go-autorest/autorest/mocks v0.4.1/go.mod h1:LTp+uSrOhSkaKrUy935gNZuuIPPVsHlr9DSOxSayd+k= +github.com/Azure/go-autorest/autorest/mocks v0.4.2 h1:PGN4EDXnuQbojHbU0UWoNvmu9AGVwYHG9/fkDYhtAfw= +github.com/Azure/go-autorest/autorest/mocks v0.4.2/go.mod h1:Vy7OitM9Kei0i1Oj+LvyAWMXJHeKH1MVlzFugfVrmyU= +github.com/Azure/go-autorest/autorest/to v0.4.0 h1:oXVqrxakqqV1UZdSazDOPOLvOIz+XA683u8EctwboHk= +github.com/Azure/go-autorest/autorest/to v0.4.0/go.mod h1:fE8iZBn7LQR7zH/9XU2NcPR4o9jEImooCeWJcYV/zLE= +github.com/Azure/go-autorest/autorest/validation v0.3.1 h1:AgyqjAd94fwNAoTjl/WQXg4VvFeRFpO+UhNyRXqF1ac= +github.com/Azure/go-autorest/autorest/validation v0.3.1/go.mod h1:yhLgjC0Wda5DYXl6JAsWyUe4KVNffhoDhG0zVzUMo3E= +github.com/Azure/go-autorest/logger v0.2.1 h1:IG7i4p/mDa2Ce4TRyAO8IHnVhAVF3RFU+ZtXWSmf4Tg= +github.com/Azure/go-autorest/logger v0.2.1/go.mod h1:T9E3cAhj2VqvPOtCYAvby9aBXkZmbF5NWuPV8+WeEW8= +github.com/Azure/go-autorest/tracing v0.6.0 h1:TYi4+3m5t6K48TGI9AUdb+IzbnSxvnvUMfuitfgcfuo= +github.com/Azure/go-autorest/tracing v0.6.0/go.mod h1:+vhtPC754Xsa23ID7GlGsrdKBpUA79WCAKPPZVC2DeU= +github.com/Azure/skewer v0.0.19 h1:+qA1z8isKmlNkhAwZErNS2wD2jaemSk9NszYKr8dddU= +github.com/Azure/skewer v0.0.19/go.mod h1:LVH7jmduRKmPj8YcIz7V4f53xJEntjweL4aoLyChkwk= +github.com/AzureAD/microsoft-authentication-library-for-go v1.1.1 h1:WpB/QDNLpMw72xHJc34BNNykqSOeEJDAWkhf0u12/Jk= +github.com/AzureAD/microsoft-authentication-library-for-go v1.1.1/go.mod h1:wP83P5OoQ5p6ip3ScPr0BAq0BvuPAvacpEuSzyouqAI= +github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= +github.com/Pallinder/go-randomdata v1.2.0 h1:DZ41wBchNRb/0GfsePLiSwb0PHZmT67XY00lCDlaYPg= +github.com/Pallinder/go-randomdata v1.2.0/go.mod h1:yHmJgulpD2Nfrm0cR9tI/+oAgRqCQQixsA8HyRZfV9Y= +github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= +github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= +github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= +github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= +github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho= +github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137 h1:s6gZFSlWYmbqAuRjVTiNNhvNRfY2Wxp9nhfyel4rklc= +github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137/go.mod h1:OMCwj8VM1Kc9e19TLln2VL61YJF0x1XFtfdL4JdbSyE= +github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= +github.com/avast/retry-go v3.0.0+incompatible h1:4SOWQ7Qs+oroOTQOYnAHqelpCO0biHSxpiH9JdtuBj0= +github.com/avast/retry-go v3.0.0+incompatible/go.mod h1:XtSnn+n/sHqQIpZ10K1qAevBhOOCWBLXXy3hyiqqBrY= +github.com/aws/karpenter-core v0.32.1 h1:H5XBLZvhAEj8na30IIF1g9N0vj16OruHZiHWGJZuzGk= +github.com/aws/karpenter-core v0.32.1/go.mod h1:RNih2g6qCiah8rFaZ7HkmClIK66Hjj38z3DbWnWGM2w= +github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= +github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= +github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= +github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= +github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= +github.com/blang/semver/v4 v4.0.0 h1:1PFHFE6yCCTv8C1TeyNNarDzntLi7wMI5i/pzqYIsAM= +github.com/blang/semver/v4 v4.0.0/go.mod h1:IbckMUScFkM3pff0VJDNKRiT6TG/YpiHIM2yvyW5YoQ= +github.com/blendle/zapdriver v1.3.1 h1:C3dydBOWYRiOk+B8X9IVZ5IOe+7cl+tGOexN4QqHfpE= +github.com/blendle/zapdriver v1.3.1/go.mod h1:mdXfREi6u5MArG4j9fewC+FGnXaBR+T4Ox4J2u4eHCc= +github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= +github.com/census-instrumentation/opencensus-proto v0.4.1 h1:iKLQ0xPNFxR/2hzXZMrBo8f1j86j5WHzznCCQxV/b8g= +github.com/census-instrumentation/opencensus-proto v0.4.1/go.mod h1:4T9NM4+4Vw91VeyqjLS6ao50K5bOcLKN6Q42XnYaRYw= +github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/cespare/xxhash/v2 v2.1.2/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= +github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= +github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= +github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= +github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= +github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= +github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= +github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/deckarep/golang-set v1.8.0 h1:sk9/l/KqpunDwP7pSjUg0keiOOLEnOBHzykLrsPppp4= +github.com/deckarep/golang-set v1.8.0/go.mod h1:5nI87KwE7wgsBU1F4GKAw2Qod7p5kyS383rP6+o6qqo= +github.com/dnaeon/go-vcr v1.2.0 h1:zHCHvJYTMh1N7xnV7zf1m1GPBF9Ad0Jk/whtQ1663qI= +github.com/dnaeon/go-vcr v1.2.0/go.mod h1:R4UdLID7HZT3taECzJs4YgbbH6PIGXB6W/sc5OLb6RQ= +github.com/emicklei/go-restful/v3 v3.11.0 h1:rAQeMHw1c7zTmncogyy8VvRZwtkmkZ4FxERmMY4rD+g= +github.com/emicklei/go-restful/v3 v3.11.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= +github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= +github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= +github.com/evanphx/json-patch v5.7.0+incompatible h1:vgGkfT/9f8zE6tvSCe74nfpAVDQ2tG6yudJd8LBksgI= +github.com/evanphx/json-patch v5.7.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= +github.com/evanphx/json-patch/v5 v5.7.0 h1:nJqP7uwL84RJInrohHfW0Fx3awjbm8qZeFv0nW9SYGc= +github.com/evanphx/json-patch/v5 v5.7.0/go.mod h1:VNkHZ/282BpEyt/tObQO8s5CMPmYYq14uClGH4abBuQ= +github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY= +github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbSClcnxKAGw= +github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= +github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= +github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= +github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= +github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= +github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= +github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vbaY= +github.com/go-kit/log v0.2.0/go.mod h1:NwTd00d/i8cPZ3xOwwiv2PO5MOcx78fFErGNcVmBjv0= +github.com/go-kit/log v0.2.1 h1:MRVx0/zhvdseW+Gza6N9rVzU/IVzaeE1SFI4raAhmBU= +github.com/go-kit/log v0.2.1/go.mod h1:NwTd00d/i8cPZ3xOwwiv2PO5MOcx78fFErGNcVmBjv0= +github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= +github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= +github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= +github.com/go-logfmt/logfmt v0.5.1/go.mod h1:WYhtIu8zTZfxdn5+rREduYbwxfcBr/Vr6KEVveWlfTs= +github.com/go-logfmt/logfmt v0.6.0 h1:wGYYu3uicYdqXVgoYbvnkrPVXkuLM1p1ifugDMEdRi4= +github.com/go-logfmt/logfmt v0.6.0/go.mod h1:WYhtIu8zTZfxdn5+rREduYbwxfcBr/Vr6KEVveWlfTs= +github.com/go-logr/logr v1.2.0/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.2.4 h1:g01GSCwiDw2xSZfjJ2/T9M+S6pFdcNtFYsp+Y43HYDQ= +github.com/go-logr/logr v1.2.4/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/zapr v1.2.4 h1:QHVo+6stLbfJmYGkQ7uGHUCu5hnAFAj6mDe6Ea0SeOo= +github.com/go-logr/zapr v1.2.4/go.mod h1:FyHWQIzQORZ0QVE1BtVHv3cKtNLuXsbNLtpuhNapBOA= +github.com/go-openapi/jsonpointer v0.19.6/go.mod h1:osyAmYz/mB/C3I+WsTTSgw1ONzaLJoLCyoi6/zppojs= +github.com/go-openapi/jsonpointer v0.20.0 h1:ESKJdU9ASRfaPNOPRx12IUyA1vn3R9GiE3KYD14BXdQ= +github.com/go-openapi/jsonpointer v0.20.0/go.mod h1:6PGzBjjIIumbLYysB73Klnms1mwnU4G3YHOECG3CedA= +github.com/go-openapi/jsonreference v0.20.2 h1:3sVjiK66+uXK/6oQ8xgcRKcFgQ5KXa2KvnJRumpMGbE= +github.com/go-openapi/jsonreference v0.20.2/go.mod h1:Bl1zwGIM8/wsvqjsOQLJ/SH+En5Ap4rVB5KVcIDZG2k= +github.com/go-openapi/swag v0.22.3/go.mod h1:UzaqsxGiab7freDnrUUra0MwWfN/q7tE4j+VcZ0yl14= +github.com/go-openapi/swag v0.22.4 h1:QLMzNJnMGPRNDCbySlcj1x01tzU8/9LTTL9hZZZogBU= +github.com/go-openapi/swag v0.22.4/go.mod h1:UzaqsxGiab7freDnrUUra0MwWfN/q7tE4j+VcZ0yl14= +github.com/go-playground/assert/v2 v2.2.0 h1:JvknZsQTYeFEAhQwI4qEt9cyV5ONwRHC+lYKSsYSR8s= +github.com/go-playground/assert/v2 v2.2.0/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4= +github.com/go-playground/locales v0.14.1 h1:EWaQ/wswjilfKLTECiXz7Rh+3BjFhfDFKv/oXslEjJA= +github.com/go-playground/locales v0.14.1/go.mod h1:hxrqLVvrK65+Rwrd5Fc6F2O76J/NuW9t0sjnWqG1slY= +github.com/go-playground/universal-translator v0.18.1 h1:Bcnm0ZwsGyWbCzImXv+pAJnYK9S473LQFuzCbDbfSFY= +github.com/go-playground/universal-translator v0.18.1/go.mod h1:xekY+UJKNuX9WP91TpwSH2VMlDf28Uj24BCp08ZFTUY= +github.com/go-playground/validator/v10 v10.13.0 h1:cFRQdfaSMCOSfGCCLB20MHvuoHb/s5G8L5pu2ppK5AQ= +github.com/go-playground/validator/v10 v10.13.0/go.mod h1:dwu7+CG8/CtBiJFZDz4e+5Upb6OLw04gtBYw0mcG/z4= +github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= +github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 h1:tfuBGBXKqDEevZMzYi5KSi8KkcZtzBcTgAUUtapy0OI= +github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572/go.mod h1:9Pwr4B2jHnOSGXyyzV8ROjYa2ojvAY6HCGYYfMoC3Ls= +github.com/gobuffalo/flect v1.0.2 h1:eqjPGSo2WmjgY2XlpGwo2NXgL3RucAKo4k4qQMNA5sA= +github.com/gobuffalo/flect v1.0.2/go.mod h1:A5msMlrHtLqh9umBSnvabjsMrCcCpAyzglnDvkbYKHs= +github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= +github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= +github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= +github.com/golang-jwt/jwt/v4 v4.0.0/go.mod h1:/xlHOz8bRuivTWchD4jCa+NbatV+wEUSzwAxVc6locg= +github.com/golang-jwt/jwt/v4 v4.5.0 h1:7cYmW1XlMY7h7ii7UhUyChSgS5wUJEnm9uZVTGqOWzg= +github.com/golang-jwt/jwt/v4 v4.5.0/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0= +github.com/golang-jwt/jwt/v5 v5.0.0 h1:1n1XNM9hk7O9mnQoNBGolZvzebBQ7p93ULHRc28XJUE= +github.com/golang-jwt/jwt/v5 v5.0.0/go.mod h1:pqrtFR0X4osieyHYxtmOUWsAWrfe1Q5UVIyoH402zdk= +github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= +github.com/golang/glog v1.1.0 h1:/d3pCKDPWNnvIWe0vVUpNP32qc8U3PDVxySP/y360qE= +github.com/golang/glog v1.1.0/go.mod h1:pfYeQZ3JWZoXTV5sFc986z3HTpwQs9At6P4ImfuP3NQ= +github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE= +github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y= +github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= +github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= +github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= +github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4= +github.com/golang/mock v1.6.0 h1:ErTB+efbowRARo13NNdxyJji2egdxLGQhRaY+DUumQc= +github.com/golang/mock v1.6.0/go.mod h1:p6yTPP+5HYm5mzsMV8JkE6ZKdX+/wYM6Hr+LicevLPs= +github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= +github.com/golang/protobuf v1.3.4/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= +github.com/golang/protobuf v1.3.5/go.mod h1:6O5/vntMXwX2lRkT1hjjk0nAC1IDOTvTlVgjlRvqsdk= +github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= +github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= +github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= +github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= +github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= +github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= +github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= +github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= +github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= +github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= +github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= +github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= +github.com/google/gnostic-models v0.6.8 h1:yo/ABAfM5IMRsS1VnXjTBvUb61tFIHozhlYvRgGre9I= +github.com/google/gnostic-models v0.6.8/go.mod h1:5n7qKqH0f5wFt+aWF8CW6pZLLNOfYuF5OpfBSENuI8U= +github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= +github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.4.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0= +github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= +github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= +github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= +github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= +github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20230926050212-f7f687d19a98 h1:pUa4ghanp6q4IJHwE9RwLgmVFfReJN+KbQ8ExNEUUoQ= +github.com/google/pprof v0.0.0-20230926050212-f7f687d19a98/go.mod h1:czg5+yv1E0ZGTi6S6vVK1mke0fV+FaUhNGcd6VRS9Ik= +github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= +github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.3.1 h1:KjJaJ9iWZ3jOFZIf1Lqf4laDRCasjl0BCmnEGxkdLb4= +github.com/google/uuid v1.3.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= +github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= +github.com/grpc-ecosystem/grpc-gateway v1.14.6/go.mod h1:zdiPV4Yse/1gnckTHtghG4GkDEdKCRJduHpTxT3/jcw= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.18.0 h1:RtRsiaGvWxcwd8y3BiRZxsylPT8hLWZ5SPcfI+3IDNk= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.18.0/go.mod h1:TzP6duP4Py2pHLVPPQp42aoYI92+PCrVotyR5e8Vqlk= +github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= +github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= +github.com/hashicorp/golang-lru v1.0.2 h1:dV3g9Z/unq5DpblPpw+Oqcv4dU/1omnb4Ok8iPY6p1c= +github.com/hashicorp/golang-lru v1.0.2/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= +github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= +github.com/imdario/mergo v0.3.16 h1:wwQJbIsHYGMUyLSPrEq1CT16AhnhNJQ51+4fdHUnCl4= +github.com/imdario/mergo v0.3.16/go.mod h1:WBLT9ZmE3lPoWsEzCh9LPo3TiwVN+ZKEjmz+hD27ysY= +github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= +github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= +github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= +github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= +github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4= +github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= +github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= +github.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= +github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= +github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= +github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= +github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= +github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= +github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= +github.com/kelseyhightower/envconfig v1.4.0 h1:Im6hONhd3pLkfDFsbRgu68RDNkGF1r3dvMUtDTo2cv8= +github.com/kelseyhightower/envconfig v1.4.0/go.mod h1:cccZRl6mQpaq41TPp5QxidR+Sa3axMbJDNb//FQX6Gg= +github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= +github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= +github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= +github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= +github.com/leodido/go-urn v1.2.3 h1:6BE2vPT0lqoz3fmOesHZiaiFh7889ssCo2GMvLCfiuA= +github.com/leodido/go-urn v1.2.3/go.mod h1:7ZrI8mTSeBSHl/UaRyKQW1qZeMgak41ANeCNaVckg+4= +github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= +github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= +github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= +github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zkfA9PSy5pEvNWRP0ET0TIVo= +github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= +github.com/mitchellh/hashstructure/v2 v2.0.2 h1:vGKWl0YJqUNxE8d+h8f6NJLcCJrgbhC4NcD46KavDd4= +github.com/mitchellh/hashstructure/v2 v2.0.2/go.mod h1:MG3aRVU/N29oo/V/IhBX8GR/zz4kQkprJgF2EVszyDE= +github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= +github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= +github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= +github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= +github.com/onsi/ginkgo/v2 v2.13.0 h1:0jY9lJquiL8fcf3M4LAXN5aMlS/b2BV86HFFPCPMgE4= +github.com/onsi/ginkgo/v2 v2.13.0/go.mod h1:TE309ZR8s5FsKKpuB1YAQYBzCaAfUgatB/xlT/ETL/o= +github.com/onsi/gomega v1.29.0 h1:KIA/t2t5UBzoirT4H9tsML45GEbo3ouUnBHsCfD2tVg= +github.com/onsi/gomega v1.29.0/go.mod h1:9sxs+SwGrKI0+PWe4Fxa9tFQQBG5xSsSbMXOI8PPpoQ= +github.com/patrickmn/go-cache v2.1.0+incompatible h1:HRMgzkcYKYpi3C8ajMPV8OFXaaRUnok+kx1WdO15EQc= +github.com/patrickmn/go-cache v2.1.0+incompatible/go.mod h1:3Qf8kWWT7OJRJbdiICTKqZju1ZixQ/KpMGzzAfe6+WQ= +github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8 h1:KoWmjvw+nsYOo29YJK9vDA65RGE3NrOnUtO7a+RF9HU= +github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8/go.mod h1:HKlIX3XHQyzLZPlr7++PzdhaXEj94dEiJgZDTsxEqUI= +github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/prashantv/gostub v1.1.0 h1:BTyx3RfQjRHnUWaGF9oQos79AlQ5k8WNktv7VGvVH4g= +github.com/prashantv/gostub v1.1.0/go.mod h1:A5zLQHz7ieHGG7is6LLXLz7I8+3LZzsrV0P1IAHhP5U= +github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= +github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= +github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M= +github.com/prometheus/client_golang v1.11.0/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0= +github.com/prometheus/client_golang v1.12.1/go.mod h1:3Z9XVyYiZYEO+YQWt3RD2R3jrbd179Rt297l4aS6nDY= +github.com/prometheus/client_golang v1.12.2/go.mod h1:3Z9XVyYiZYEO+YQWt3RD2R3jrbd179Rt297l4aS6nDY= +github.com/prometheus/client_golang v1.13.0/go.mod h1:vTeo+zgvILHsnnj/39Ou/1fPN5nJFOEMgftOUOmlvYQ= +github.com/prometheus/client_golang v1.17.0 h1:rl2sfwZMtSthVU752MqfjQozy7blglC+1SOtjMAMh+Q= +github.com/prometheus/client_golang v1.17.0/go.mod h1:VeL+gMmOAxkS2IqfCq0ZmHSL+LjWfWDUmp1mBz9JgUY= +github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= +github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/client_model v0.5.0 h1:VQw1hfvPvk3Uv6Qf29VrPF32JB6rtbgI6cYPYQjL0Qw= +github.com/prometheus/client_model v0.5.0/go.mod h1:dTiFglRmd66nLR9Pv9f0mZi7B7fk5Pm3gvsjB5tr+kI= +github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= +github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo= +github.com/prometheus/common v0.26.0/go.mod h1:M7rCNAaPfAosfx8veZJCuw84e35h3Cfd9VFqTh1DIvc= +github.com/prometheus/common v0.32.1/go.mod h1:vu+V0TpY+O6vW9J44gczi3Ap/oXXR10b+M/gUGO4Hls= +github.com/prometheus/common v0.35.0/go.mod h1:phzohg0JFMnBEFGxTDbfu3QyL5GI8gTQJFhYO5B3mfA= +github.com/prometheus/common v0.37.0/go.mod h1:phzohg0JFMnBEFGxTDbfu3QyL5GI8gTQJFhYO5B3mfA= +github.com/prometheus/common v0.44.0 h1:+5BrQJwiBB9xsMygAB3TNvpQKOwlkc25LbISbrdOOfY= +github.com/prometheus/common v0.44.0/go.mod h1:ofAIvZbQ1e/nugmZGz4/qCb9Ap1VoSTIO7x0VV9VvuY= +github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= +github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= +github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= +github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= +github.com/prometheus/procfs v0.7.3/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= +github.com/prometheus/procfs v0.8.0/go.mod h1:z7EfXMXOkbkqb9IINtpCn86r/to3BnA0uaxHdg830/4= +github.com/prometheus/procfs v0.12.0 h1:jluTpSng7V9hY0O2R9DzzJHYb2xULk9VTR1V1R/k6Bo= +github.com/prometheus/procfs v0.12.0/go.mod h1:pcuDEFsWDnvcgNzo4EEweacyhjeA9Zk3cnaOZAZEfOo= +github.com/prometheus/statsd_exporter v0.22.7/go.mod h1:N/TevpjkIh9ccs6nuzY3jQn9dFqnUakOjnEuMPJJJnI= +github.com/prometheus/statsd_exporter v0.24.0 h1:aZmN6CzS2H1Non1JKZdjkQlAkDtGoQBYIESk2SlU1OI= +github.com/prometheus/statsd_exporter v0.24.0/go.mod h1:+dQiRTqn9DnPmN5mI5Xond+k8nuRKzdgh1omxh9OgFY= +github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= +github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= +github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= +github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog= +github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/samber/lo v1.38.1 h1:j2XEAqXKb09Am4ebOg31SpvzUTTs6EN3VfgeLUhPdXM= +github.com/samber/lo v1.38.1/go.mod h1:+m/ZKRl6ClXCE2Lgf3MsQlWfh4bn1bz6CXEOxnEXnEA= +github.com/shopspring/decimal v1.3.1 h1:2Usl1nmF/WZucqkFZhnfFYxxxu8LG21F6nPQBE5gKV8= +github.com/shopspring/decimal v1.3.1/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o= +github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= +github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= +github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= +github.com/spf13/cobra v1.7.0 h1:hyqWnYt1ZQShIddO5kBpj3vu05/++x6tJ6dg8EC572I= +github.com/spf13/cobra v1.7.0/go.mod h1:uLxZILRyS/50WlhOIKD7W6V5bgeIt+4sICxh6uRMrb0= +github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= +github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= +github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= +github.com/stvp/go-udp-testing v0.0.0-20201019212854-469649b16807/go.mod h1:7jxmlfBCDBXRzr0eAQJ48XC1hBu1np4CS5+cHEYfwpc= +github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= +github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= +go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= +go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= +go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= +go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= +go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= +go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E= +go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0= +go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo= +go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= +go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= +go.uber.org/automaxprocs v1.5.3 h1:kWazyxZUrS3Gs4qUpbwo5kEIMGe/DAvi5Z4tl2NW4j8= +go.uber.org/automaxprocs v1.5.3/go.mod h1:eRbA25aqJrxAbsLO0xy5jVwPt7FQnRgjW+efnwa1WM0= +go.uber.org/goleak v1.1.11/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ= +go.uber.org/goleak v1.2.1 h1:NBol2c7O1ZokfZ0LEU9K6Whx/KnwvepVetCUhtKja4A= +go.uber.org/goleak v1.2.1/go.mod h1:qlT2yGI9QafXHhZZLxlSuNsMw3FFLxBr+tBRlmO1xH4= +go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= +go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= +go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= +go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= +go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= +go.uber.org/zap v1.24.0/go.mod h1:2kMP+WWQ8aoFoedH3T2sq6iJ2yDWpHbP0f6MQbS9Gkg= +go.uber.org/zap v1.26.0 h1:sI7k6L95XOKS281NhVKOFCUNIvv9e0w4BF8N3u+tCRo= +go.uber.org/zap v1.26.0/go.mod h1:dtElttAiwGvoJ/vj4IwHBS/gXsEu/pZ50mUIRWuG0so= +golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= +golang.org/x/crypto v0.6.0/go.mod h1:OFC/31mSvZgRz0V1QTNCzfAI1aIRzbiufJtkMIlEp58= +golang.org/x/crypto v0.14.0 h1:wBqGXzWJW6m1XrIKlAH0Hs1JJ7+9KBwnIO8v66Q9cHc= +golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4= +golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= +golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek= +golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY= +golang.org/x/exp v0.0.0-20191129062945-2f5052295587/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= +golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= +golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= +golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= +golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= +golang.org/x/exp v0.0.0-20231006140011-7918f672742d h1:jtJma62tbqLibJ5sFQz8bKtEM8rJBtfilJ2qTU199MI= +golang.org/x/exp v0.0.0-20231006140011-7918f672742d/go.mod h1:ldy0pHrwJyGW56pPQzzkH36rKxoZW1tw7ZJpeKx+hdo= +golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= +golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= +golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= +golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= +golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= +golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs= +golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= +golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= +golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= +golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY= +golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= +golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= +golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= +golang.org/x/mod v0.13.0 h1:I/DsJXRlw/8l/0c24sM9yb0T4z9liZTduXvdAWYiysY= +golang.org/x/mod v0.13.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= +golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= +golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20191002035440-2ec189313ef0/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200222125558-5a598a2470a0/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200501053045-e0ff5e5a1de5/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200506145744-7e3656a0809f/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200513185701-a91f0712d120/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200520182314-0ba52f642ac2/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= +golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= +golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= +golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= +golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= +golang.org/x/net v0.17.0 h1:pVaXccu2ozPjCXewfr1S7xza/zcXTity9cCdXQYSjIM= +golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE= +golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= +golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20210514164344-f6687ab2804c/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20220223155221-ee480838109b/go.mod h1:DAh4E804XQdzx2j+YRIaUnCqCV2RuMz24cGBJ5QYIrc= +golang.org/x/oauth2 v0.13.0 h1:jDDenyj+WgFtmV3zYVoi8aE2BwtXFLWOA67ZfNWftiY= +golang.org/x/oauth2 v0.13.0/go.mod h1:/JMhi4ZRXAf4HG9LiNmxvk+45+96RUlVThiH8FzNBn0= +golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20220601150217-0de741cfad7f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.4.0 h1:zxkM55ReGkDlKSM+Fu41A+zmbZuaPVbGMzvvdUPznYQ= +golang.org/x/sync v0.4.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= +golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200331124033-c3d80250170d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200501052902-10377860bb8e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200511232937-7e40ca221e25/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200515095857-1151b9dac4a9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210616045830-e2b7044e8c71/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220708085239-5a0f0661e09d/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.13.0 h1:Af8nKPmuFypiUBjVoU9V20FiaFXOcuZI21p0ycVYYGE= +golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= +golang.org/x/term v0.13.0 h1:bb+I9cTfFazGW51MZqBVmZy7+JEJMouUHTUSKVQLBek= +golang.org/x/term v0.13.0/go.mod h1:LTmsnFJwVN6bCy1rVCoS+qHT1HhALEFxKncY3WNNh4U= +golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= +golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/text v0.13.0 h1:ablQoSUd0tRdKxZewP80B+BaqeKJuVhuRxj/dkrun3k= +golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= +golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.3.0 h1:rg5rLMjNzMS1RkNLzCG38eapWhnYLFYXDXj2gOlr8j4= +golang.org/x/time v0.3.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= +golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191130070609-6e064ea0cf2d/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191216173652-a0e659d51361/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200117161641-43d50277825c/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200122220014-bf1340f18c4a/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200204074204-1cc6d1ef6c74/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200207183749-b753a1ba74fa/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200212150539-ea181f53ac56/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200224181240-023911ca70b2/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200227222343-706bc42d1f0d/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200304193943-95d2e580d8eb/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= +golang.org/x/tools v0.0.0-20200312045724-11d5b4c81c7d/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= +golang.org/x/tools v0.0.0-20200331025713-a30bf2db82d4/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8= +golang.org/x/tools v0.0.0-20200501065659-ab2804fb9c9d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200512131952-2bc93b1c0c88/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200515010526-7d3b6ebf133d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200618134242-20370b0cb4b2/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200729194436-6467de6f59a7/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= +golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= +golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= +golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= +golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= +golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= +golang.org/x/tools v0.14.0 h1:jvNa2pY0M4r62jkRQ6RwEZZyPcymeL9XZMLBbV7U2nc= +golang.org/x/tools v0.14.0/go.mod h1:uYBEerGOWcJyEORxN+Ek8+TT266gXkNlHdJBwexUsBg= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +gomodules.xyz/jsonpatch/v2 v2.4.0 h1:Ci3iUJyx9UeRx7CeFN8ARgGbkESwJK+KB9lLcWxY/Zw= +gomodules.xyz/jsonpatch/v2 v2.4.0/go.mod h1:AH3dM2RI6uoBZxn3LVrfvJ3E0/9dG4cSrbuBJT4moAY= +google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= +google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= +google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= +google.golang.org/api v0.9.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= +google.golang.org/api v0.13.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= +google.golang.org/api v0.14.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= +google.golang.org/api v0.15.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= +google.golang.org/api v0.17.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.18.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.19.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.20.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.22.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.24.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= +google.golang.org/api v0.25.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= +google.golang.org/api v0.28.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= +google.golang.org/api v0.29.0/go.mod h1:Lcubydp8VUV7KeIHD9z2Bys/sm/vGKnG1UHuDBSrHWM= +google.golang.org/api v0.30.0/go.mod h1:QGmEvQ87FHZNiUVJkT14jQNYJ4ZJjdRF23ZXz5138Fc= +google.golang.org/api v0.146.0 h1:9aBYT4vQXt9dhCuLNfwfd3zpwu8atg0yPkjBymwSrOM= +google.golang.org/api v0.146.0/go.mod h1:OARJqIfoYjXJj4C1AiBSXYZt03qsoz8FQYU6fBEfrHM= +google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= +google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= +google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/appengine v1.6.8 h1:IhEN5q69dyKagZPYMSdIjS2HqprW324FRQZJcGqPAsM= +google.golang.org/appengine v1.6.8/go.mod h1:1jJ3jBArFh5pcgW8gCtRJnepW8FzD1V44FJffLiz/Ds= +google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= +google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= +google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= +google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8= +google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20191115194625-c23dd37a84c9/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20191216164720-4f79533eabd1/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20191230161307-f3c370f40bfb/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20200115191322-ca5a22157cba/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20200122232147-0452cf42e150/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20200204135345-fa8e72b47b90/go.mod h1:GmwEX6Z4W5gMy59cAlVYjN9JhxgbQH6Gn+gFDQe2lzA= +google.golang.org/genproto v0.0.0-20200212174721-66ed5ce911ce/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200224152610-e50cd9704f63/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200228133532-8c2c7df3a383/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200305110556-506484158171/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200312145019-da6875a35672/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200331122359-1ee6d9798940/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200430143042-b979b6f78d84/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200511104702-f5ebc3bea380/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200513103714-09dca8ec2884/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200515170657-fc4c6c6a6587/go.mod h1:YsZOwe1myG/8QRHRsmBRE1LrgQY60beZKjly0O1fX9U= +google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= +google.golang.org/genproto v0.0.0-20200527145253-8367513e4ece/go.mod h1:jDfRM7FcilCzHH/e9qn6dsT145K34l5v+OpcnNgKAAA= +google.golang.org/genproto v0.0.0-20200618031413-b414f8b61790/go.mod h1:jDfRM7FcilCzHH/e9qn6dsT145K34l5v+OpcnNgKAAA= +google.golang.org/genproto v0.0.0-20200729003335-053ba62fc06f/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20231009173412-8bfb1ae86b6c h1:ml3TAUoIIzQUtX88s/icpXCFW9lV5VwsuIuS1htNjKY= +google.golang.org/genproto v0.0.0-20231009173412-8bfb1ae86b6c/go.mod h1:MugzuwC+GYOxyF0XUGQvsT97bOgWCV7MM1XMc5FZv8E= +google.golang.org/genproto/googleapis/api v0.0.0-20231009173412-8bfb1ae86b6c h1:0RtEmmHjemvUXloH7+RuBSIw7n+GEHMOMY1CkGYnWq4= +google.golang.org/genproto/googleapis/api v0.0.0-20231009173412-8bfb1ae86b6c/go.mod h1:Wth13BrWMRN/G+guBLupKa6fslcWZv14R0ZKDRkNfY8= +google.golang.org/genproto/googleapis/rpc v0.0.0-20231009173412-8bfb1ae86b6c h1:jHkCUWkseRf+W+edG5hMzr/Uh1xkDREY4caybAq4dpY= +google.golang.org/genproto/googleapis/rpc v0.0.0-20231009173412-8bfb1ae86b6c/go.mod h1:4cYg8o5yUbm77w8ZX00LhMVNl/YVBFJRYWDc0uYWMs0= +google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= +google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= +google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= +google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= +google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= +google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.27.1/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.28.0/go.mod h1:rpkK4SK4GF4Ach/+MFLZUBavHOvF2JJB5uozKKal+60= +google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk= +google.golang.org/grpc v1.30.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= +google.golang.org/grpc v1.31.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= +google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc= +google.golang.org/grpc v1.58.3 h1:BjnpXut1btbtgN/6sp+brB2Kbm2LjNXnidYujAVbSoQ= +google.golang.org/grpc v1.58.3/go.mod h1:tgX3ZQDlNJGU96V6yHh1T/JeoBQ2TXdr43YbYSsCJk0= +google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= +google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= +google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= +google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= +google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= +google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4= +google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= +google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= +google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= +google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +google.golang.org/protobuf v1.28.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs8= +google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= +gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= +gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= +gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= +gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= +honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= +honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= +k8s.io/api v0.28.3 h1:Gj1HtbSdB4P08C8rs9AR94MfSGpRhJgsS+GF9V26xMM= +k8s.io/api v0.28.3/go.mod h1:MRCV/jr1dW87/qJnZ57U5Pak65LGmQVkKTzf3AtKFHc= +k8s.io/apiextensions-apiserver v0.28.3 h1:Od7DEnhXHnHPZG+W9I97/fSQkVpVPQx2diy+2EtmY08= +k8s.io/apiextensions-apiserver v0.28.3/go.mod h1:NE1XJZ4On0hS11aWWJUTNkmVB03j9LM7gJSisbRt8Lc= +k8s.io/apimachinery v0.28.3 h1:B1wYx8txOaCQG0HmYF6nbpU8dg6HvA06x5tEffvOe7A= +k8s.io/apimachinery v0.28.3/go.mod h1:uQTKmIqs+rAYaq+DFaoD2X7pcjLOqbQX2AOiO0nIpb8= +k8s.io/client-go v0.28.3 h1:2OqNb72ZuTZPKCl+4gTKvqao0AMOl9f3o2ijbAj3LI4= +k8s.io/client-go v0.28.3/go.mod h1:LTykbBp9gsA7SwqirlCXBWtK0guzfhpoW4qSm7i9dxo= +k8s.io/cloud-provider v0.28.3 h1:9u+JjA3zIn0nqLOOa8tWnprFkffguSAhfBvo8p7LhBQ= +k8s.io/cloud-provider v0.28.3/go.mod h1:shAJxdrKu+SwwGUhkodxByPjaH8KBFZqXo6jU1F0ehI= +k8s.io/component-base v0.28.3 h1:rDy68eHKxq/80RiMb2Ld/tbH8uAE75JdCqJyi6lXMzI= +k8s.io/component-base v0.28.3/go.mod h1:fDJ6vpVNSk6cRo5wmDa6eKIG7UlIQkaFmZN2fYgIUD8= +k8s.io/component-helpers v0.28.3 h1:te9ieTGzcztVktUs92X53P6BamAoP73MK0qQP0WmDqc= +k8s.io/component-helpers v0.28.3/go.mod h1:oJR7I9ist5UAQ3y/CTdbw6CXxdMZ1Lw2Ua/EZEwnVLs= +k8s.io/csi-translation-lib v0.28.3 h1:7deV+HZjV418AGikSDPW8dyzTpm4K3tNbQUp3KmR7cs= +k8s.io/csi-translation-lib v0.28.3/go.mod h1:zlrYwakCz2yji9/8EaJk+afIKPrYXPNXXLDO8DVuuTk= +k8s.io/klog/v2 v2.100.1 h1:7WCHKK6K8fNhTqfBhISHQ97KrnJNFZMcQvKp7gP/tmg= +k8s.io/klog/v2 v2.100.1/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0= +k8s.io/kube-openapi v0.0.0-20231010175941-2dd684a91f00 h1:aVUu9fTY98ivBPKR9Y5w/AuzbMm96cd3YHRTU83I780= +k8s.io/kube-openapi v0.0.0-20231010175941-2dd684a91f00/go.mod h1:AsvuZPBlUDVuCdzJ87iajxtXuR9oktsTctW/R9wwouA= +k8s.io/utils v0.0.0-20230726121419-3b25d923346b h1:sgn3ZU783SCgtaSJjpcVVlRqd6GSnlTLKgpAAttJvpI= +k8s.io/utils v0.0.0-20230726121419-3b25d923346b/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= +knative.dev/pkg v0.0.0-20231010144348-ca8c009405dd h1:KJXBX9dOmRTUWduHg1gnWtPGIEl+GMh8UHdrBEZgOXE= +knative.dev/pkg v0.0.0-20231010144348-ca8c009405dd/go.mod h1:36cYnaOVHkzmhgybmYX6zDaTl3PakFeJQJl7wi6/RLE= +rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= +rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= +rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= +sigs.k8s.io/cloud-provider-azure v1.28.2 h1:KKrWdC1+p2xXdT1VRmSkT57MhKNzPXk3yPcrwUDIr5I= +sigs.k8s.io/cloud-provider-azure v1.28.2/go.mod h1:vDsaFOrvDDEUg0mLF2eoUeneCK+ROlRf4zACA91iwHs= +sigs.k8s.io/controller-runtime v0.16.3 h1:2TuvuokmfXvDUamSx1SuAOO3eTyye+47mJCigwG62c4= +sigs.k8s.io/controller-runtime v0.16.3/go.mod h1:j7bialYoSn142nv9sCOJmQgDXQXxnroFU4VnX/brVJ0= +sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd h1:EDPBXCAspyGV4jQlpZSudPeMmr1bNJefnuqLsRAsHZo= +sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd/go.mod h1:B8JuhiUyNFVKdsE8h686QcCxMaH6HrOAZj4vswFpcB0= +sigs.k8s.io/structured-merge-diff/v4 v4.3.0 h1:UZbZAZfX0wV2zr7YZorDz6GXROfDFj6LvqCRm4VUVKk= +sigs.k8s.io/structured-merge-diff/v4 v4.3.0/go.mod h1:N8hJocpFajUSSeSJ9bOZ77VzejKZaXsTtZo4/u7Io08= +sigs.k8s.io/yaml v1.3.0 h1:a2VclLzOGrwOHDiV8EfBGhvjHvP46CtW5j6POvhYGGo= +sigs.k8s.io/yaml v1.3.0/go.mod h1:GeOyir5tyXNByN85N/dRIT9es5UQNerPYEKK56eTBm8= diff --git a/grafana-values.yaml b/grafana-values.yaml new file mode 100644 index 000000000..831d55573 --- /dev/null +++ b/grafana-values.yaml @@ -0,0 +1,27 @@ +datasources: + datasources.yaml: + apiVersion: 1 + datasources: + - name: Prometheus + type: prometheus + version: 1 + url: http://prometheus-server:80 + access: proxy +dashboardProviders: + dashboardproviders.yaml: + apiVersion: 1 + providers: + - name: 'default' + orgId: 1 + folder: '' + type: file + disableDeletion: false + editable: true + options: + path: /var/lib/grafana/dashboards/default +dashboards: + default: + capacity-dashboard: + url: https://karpenter.sh/v0.16.1/getting-started/getting-started-with-eksctl/karpenter-capacity-dashboard.json + performance-dashboard: + url: https://karpenter.sh/v0.16.1/getting-started/getting-started-with-eksctl/karpenter-performance-dashboard.json diff --git a/hack/azure/aks-savm.bicep b/hack/azure/aks-savm.bicep new file mode 100644 index 000000000..6ad992d2a --- /dev/null +++ b/hack/azure/aks-savm.bicep @@ -0,0 +1,93 @@ +// AKS (Azure CNI & StandaloneVirtualMachines) & ACR + +param aksname string = 'karpenter' // also used for vnet +param acrname string = 'karpenter' +param location string = resourceGroup().location +param dnsPrefix string = aksname + +resource vnet 'Microsoft.Network/virtualNetworks@2022-05-01' = { + location: location + name: aksname + properties: { + addressSpace: { addressPrefixes: ['10.0.0.0/8'] } + subnets: [ + { + name: 'nodesubnet' + properties: { addressPrefix: '10.240.0.0/16' } + } +/* + { + name: 'podsubnet' + properties: { + addressPrefix: '10.241.0.0/16' + delegations: [ + { + name: 'aks-delegation' + properties: { + serviceName: 'Microsoft.ContainerService/managedClusters' + } + } + ] + } + } +*/ + ] + } + + resource nodesubnet 'subnets' existing = { name: 'nodesubnet' } +//resource podsubnet 'subnets' existing = { name: 'podsubnet' } +} + +resource aks 'Microsoft.ContainerService/managedClusters@2022-07-01' = { + location: location + name: aksname + identity: { + type: 'SystemAssigned' + } + sku: { + name: 'Basic' + tier: 'Paid' // better for scale perf testing + } + properties: { + dnsPrefix: dnsPrefix + agentPoolProfiles: [ + { + count: 1 + mode: 'System' + name: 'nodepool1' + type: 'VirtualMachines' // experimental + vmSize: 'Standard_D2pds_v5' + maxPods: 30 + vnetSubnetID: vnet::nodesubnet.id +// podSubnetID: vnet::podsubnet.id + } + ] + networkProfile: { + networkPlugin: 'azure' + serviceCidr: '10.0.0.0/16' + dnsServiceIP: '10.0.0.10' + dockerBridgeCidr: '172.17.0.1/16' + } + } +} + +// container registry + +resource acr 'Microsoft.ContainerRegistry/registries@2021-09-01' = { + location: location + name: acrname + sku: { name: 'Basic' } +} + +var AcrPull = subscriptionResourceId('Microsoft.Authorization/roleDefinition', '7f951dda-4ed3-4680-a7ca-43fe172d538d') + +@description('AKS can pull images from ACR') +resource aksAcrPull 'Microsoft.Authorization/roleAssignments@2022-04-01' = { + name: guid(resourceGroup().id, acr.name, aks.name, AcrPull) + scope: acr + properties: { + principalId: aks.properties.identityProfile.kubeletidentity.objectId + principalType: 'ServicePrincipal' + roleDefinitionId: AcrPull + } +} diff --git a/hack/azure/perftest.sh b/hack/azure/perftest.sh new file mode 100755 index 000000000..6909ae558 --- /dev/null +++ b/hack/azure/perftest.sh @@ -0,0 +1,54 @@ +#!/bin/bash + +# This deploys Provisioner requiring small instances (2 vCPU) and 'inflate' deployment with 1 cpu request, requiring VM per replica. +# It then scales the deployment up to the requested number of replicas (allocating the same number of VMs) and then scales it down. + +# TODO: obtain cluster ID programmatically +CLUSTER_ID=63559813aff5f40001dfadb5 +DASHBOARD=3052d470-b928-4e5e-bdbc-cc01e18ff318 + +set -euxo pipefail + +if [ -z "$1" ]; then echo pass number of replicas; exit 1; fi +replicas="$1" + +FMT='+%Y-%m-%dT%H-%M-%SZ' +START=$(date ${FMT}) +STARTKUBECTL=$(date --iso-8601=seconds) + +mkdir -p logs +exec > >(tee -i "logs/az-perftest-${START}-${replicas}.log") +exec 2>&1 +logk="logs/az-perftest-${START}-${replicas}-karpenter.log" + +# prep +kubectl apply -f examples/provisioner/general-purpose-azure-smallnodes.yaml +kubectl apply -f examples/workloads/inflate.yaml + +# scale up +date +kubectl scale --replicas="${replicas}" deployment/inflate +time kubectl rollout status deployment/inflate --watch --timeout=2h +date +ENDUP=$(date ${FMT}) +echo Scale up: "https://dataexplorer.azure.com/dashboards/${DASHBOARD}?p-_startTime=${START}&p-_endTime=${ENDUP}&p-_cluster_id=${CLUSTER_ID}&p-_bin_size=v-20s" +ENDUPKUBECTL=$(date --iso-8601=seconds) +kubectl logs deployment/karpenter -n karpenter --since-time="${STARTKUBECTL}" > "${logk}" + +# scale down +sleep 30 +kubectl scale --replicas=0 deployment/inflate +date +kubectl delete --wait=false nodes -l karpenter.sh/provisioner-name +time kubectl wait --for=delete nodes -l karpenter.sh/provisioner-name --timeout=30m +ENDDOWN=$(date ${FMT}) +date + +# review +kubectl logs deployment/karpenter -n karpenter --since-time="${ENDUPKUBECTL}" >> "${logk}" +az resource list -o table --tag=karpenter.sh_provisioner-name=default +# az resource wait --deleted --timeout 300 --tag=karpenter.sh_provisioner-name=default - can't wait on tags :( + +# Cluster Autoscaler dashboard links - handy for some metrics +echo Scale up: "https://dataexplorer.azure.com/dashboards/${DASHBOARD}?p-_startTime=${START}&p-_endTime=${ENDUP}&p-_cluster_id=${CLUSTER_ID}&p-_bin_size=v-20s" +echo Scale down: "https://dataexplorer.azure.com/dashboards/${DASHBOARD}?p-_startTime=${ENDUP}&p-_endTime=${ENDDOWN}&p-_cluster_id=${CLUSTER_ID}&p-_bin_size=v-20s" diff --git a/hack/boilerplate.go.txt b/hack/boilerplate.go.txt new file mode 100644 index 000000000..1c7d84977 --- /dev/null +++ b/hack/boilerplate.go.txt @@ -0,0 +1,15 @@ +/* +Portions Copyright (c) Microsoft Corporation. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ diff --git a/hack/boilerplate.sh b/hack/boilerplate.sh new file mode 100755 index 000000000..60087d2a1 --- /dev/null +++ b/hack/boilerplate.sh @@ -0,0 +1,10 @@ +#!/bin/bash +set -eu -o pipefail + +for i in $( + find ./cmd ./pkg ./test ./hack -name "*.go" +); do + if ! grep -q "Apache License" $i; then + cat hack/boilerplate.go.txt $i >$i.new && mv $i.new $i + fi +done diff --git a/hack/code/instancetype_testdata_gen.go b/hack/code/instancetype_testdata_gen.go new file mode 100644 index 000000000..d6bc1054f --- /dev/null +++ b/hack/code/instancetype_testdata_gen.go @@ -0,0 +1,163 @@ +/* +Portions Copyright (c) Microsoft Corporation. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "bytes" + "context" + "fmt" + "os" + "strings" + "time" + + "github.com/Azure/azure-sdk-for-go/sdk/azidentity" + "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute" + + "github.com/samber/lo" +) + +func main() { + fmt.Println("starting generation of sku data...") + tenantID := os.Getenv("TENANT_ID") + sub := os.Getenv("SUBSCRIPTION_ID") + aadClientID := os.Getenv("AAD_CLIENT_ID") + aadClientSecret := os.Getenv("AAD_CLIENT_SECRET") + path, region, selectedSkus := os.Args[2], os.Args[3], os.Args[4] + skus := strings.Split(selectedSkus, ",") + targetSkus := map[string]struct{}{} + for _, sku := range skus { + targetSkus[sku] = struct{}{} + } + if tenantID == "" || sub == "" || aadClientID == "" || aadClientSecret == "" || path == "" { + fmt.Println("missing env vars") + os.Exit(1) + } + cred, err := azidentity.NewClientSecretCredential(tenantID, aadClientID, aadClientSecret, nil) + if err != nil { + fmt.Printf("failed to create credential: %v", err) + } + + ctx := context.Background() + if err != nil { + fmt.Printf("failed to create client factory: %v", err) + } + + client, err := armcompute.NewResourceSKUsClient(sub, cred, nil) + if err != nil { + panic(err) + } + pager := client.NewListPager(&armcompute.ResourceSKUsClientListOptions{ + // filter by location + Filter: lo.ToPtr(fmt.Sprintf("location eq '%s'", region)), + }) + skuData := []*armcompute.ResourceSKU{} + for pager.More() { + page, err := pager.NextPage(ctx) + if err != nil { + fmt.Printf("failed to get next page: %v", err) + } + for _, sku := range page.Value { + if _, ok := targetSkus[*sku.Name]; !ok { + continue + } + skuData = append(skuData, sku) + } + } + fmt.Println("Successfully Fetched all the SKUs", len(skuData)) + writeSkuData(skuData, region, path) + fmt.Println("Successfully Generated all the SKUs") +} + +func writeSkuData(ResourceSkus []*armcompute.ResourceSKU, location, path string) { + src := &bytes.Buffer{} + fmt.Fprintln(src, "//go:build !ignore_autogenerated") + license := lo.Must(os.ReadFile("hack/boilerplate.go.txt")) + fmt.Fprintln(src, string(license)) + fmt.Fprintln(src, "package fake") + fmt.Fprintln(src, "import (") + fmt.Fprintln(src, ` "github.com/samber/lo"`) + fmt.Fprintln(src, ` // nolint SA1019 - deprecated package`) + fmt.Fprintln(src, ` "github.com/Azure/azure-sdk-for-go/services/compute/mgmt/2022-08-01/compute"`) + fmt.Fprintln(src, ")") + now := time.Now().UTC().Format(time.RFC3339) + fmt.Fprintf(src, "// generated at %s\n\n\n", now) + fmt.Fprintln(src, "// ResourceSkus is a list of all available skus from azure") + fmt.Fprintln(src, "var ResourceSkus = []compute.ResourceSku{") + for _, sku := range ResourceSkus { + fmt.Fprintln(src, " {") + fmt.Fprintf(src, " Name: lo.ToPtr(%q),\n", lo.FromPtrOr(sku.Name, "")) + fmt.Fprintf(src, " Tier: lo.ToPtr(%q),\n", lo.FromPtrOr(sku.Tier, "")) + fmt.Fprintf(src, " Kind: lo.ToPtr(%q),\n", lo.FromPtrOr(sku.Kind, "")) + fmt.Fprintf(src, " Size: lo.ToPtr(%q),\n", lo.FromPtrOr(sku.Size, "")) + fmt.Fprintf(src, " Family: lo.ToPtr(%q),\n", lo.FromPtrOr(sku.Family, "")) + fmt.Fprintf(src, " ResourceType: lo.ToPtr(%q),\n", lo.FromPtrOr(sku.ResourceType, "")) + fmt.Fprintln(src, " APIVersions: &[]string{") + for _, apiVersion := range sku.APIVersions { + fmt.Fprintf(src, " lo.ToPtr(%q),\n", lo.FromPtrOr(apiVersion, "")) + } + fmt.Fprintln(src, " },") + if sku.Capacity != nil { + fmt.Fprintln(src, " Capacity: compute.ResourceSkuCapacity{") + fmt.Fprintf(src, " Minimum: lo.ToPtr(%d),\n", lo.FromPtrOr(sku.Capacity.Minimum, 0)) + fmt.Fprintf(src, " Maximum: lo.ToPtr(%d),\n", lo.FromPtrOr(sku.Capacity.Maximum, 0)) + fmt.Fprintf(src, " Default: lo.ToPtr(%d),\n", lo.FromPtrOr(sku.Capacity.Default, 0)) + fmt.Fprintf(src, " },") + } + fmt.Fprintf(src, " Costs: &[]compute.ResourceSkuCosts{") + for _, cost := range sku.Costs { + fmt.Fprintf(src, " {MeterID: lo.ToPtr(%f), Quantity: lo.ToPtr(%q), ExtendedUnit: lo.ToPtr(%q)},", lo.FromPtrOr(cost.MeterID, ""), lo.FromPtrOr(cost.Quantity, 0.0), lo.FromPtrOr(cost.ExtendedUnit, "")) + } + fmt.Fprintln(src, " },") + fmt.Fprintln(src, " Restrictions: &[]compute.ResourceSkuRestrictions{") + for _, restriction := range sku.Restrictions { + fmt.Fprintln(src, " {") + fmt.Printf(" Type: lo.ToPtr(%s),", lo.FromPtrOr(restriction.Type, "")) + for _, value := range restriction.Values { + fmt.Fprintf(src, " Values: &[]string{%q},", lo.FromPtrOr(value, "")) + } + fmt.Fprintln(src, " },") + } + fmt.Fprintln(src, " },") + fmt.Fprintln(src, " Capabilities: &[]compute.ResourceSkuCapabilities{") + for _, capability := range sku.Capabilities { + fmt.Fprintf(src, " {Name: lo.ToPtr(%q), Value: lo.ToPtr(%q)},\n", *capability.Name, *capability.Value) + } + fmt.Fprintln(src, " },") + fmt.Fprintf(src, " Locations: &[]string{%q},\n", location) + fmt.Fprintf(src, " LocationInfo: &[]compute.ResourceSkuLocationInfo{") + for _, locationInfo := range sku.LocationInfo { + fmt.Fprintf(src, " {Location: lo.ToPtr(%q),", location) + fmt.Fprintln(src, " Zones: &[]string{") + for _, zone := range locationInfo.Zones { + fmt.Fprintf(src, " %q,\n", lo.FromPtr(zone)) + } + fmt.Fprintln(src, " },") + } + fmt.Fprintln(src, " },") + fmt.Fprintln(src, " },") + + fmt.Fprintln(src, "},") + } + + fmt.Fprintln(src, "}") + fmt.Println(src.String()) + fmt.Println("writing file to", path) + if err := os.WriteFile(path, src.Bytes(), 0600); err != nil { + fmt.Printf("failed to write file: %v", err) + panic(err) + } +} diff --git a/hack/code/prices_gen.go b/hack/code/prices_gen.go new file mode 100644 index 000000000..8cd9ac481 --- /dev/null +++ b/hack/code/prices_gen.go @@ -0,0 +1,190 @@ +/* +Portions Copyright (c) Microsoft Corporation. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "bytes" + "context" + "flag" + "fmt" + "go/format" + "log" + "os" + "runtime" + "runtime/pprof" + "sort" + "time" + + "github.com/Azure/karpenter/pkg/providers/pricing" + "github.com/samber/lo" +) + +var regions = []string{ + "australiacentral", + "australiacentral2", + "australiaeast", + "australiasoutheast", + "brazilsouth", + "brazilsoutheast", + "canadacentral", + "canadaeast", + "centralindia", + "centralus", + "eastasia", + "eastus", + "eastus2", + "francecentral", + "francesouth", + "germanynorth", + "germanywestcentral", + "japaneast", + "japanwest", + "jioindiacentral", + "jioindiawest", + "koreacentral", + "koreasouth", + "northcentralus", + "northeurope", + "norwayeast", + "norwaywest", + "polandcentral", + "qatarcentral", + "southafricanorth", + "southafricawest", + "southcentralus", + "southeastasia", + "southindia", + "swedencentral", + "swedensouth", + "switzerlandnorth", + "switzerlandwest", + "uaecentral", + "uaenorth", + "uksouth", + "ukwest", + "westcentralus", + "westeurope", + "westindia", + "westus", + "westus2", + "westus3", +} + +func main() { + flag.Parse() + if flag.NArg() != 1 { + log.Fatalf("Usage: %s pkg/providers/pricing/zz_generated.pricing.go", os.Args[0]) + } + + generatePricing(flag.Arg(0)) +} + +func generatePricing(filePath string) { + f, err := os.Create("pricing.heapprofile") + if err != nil { + log.Fatal("could not create memory profile: ", err) + } + defer f.Close() // error handling omitted for example + + ctx := context.Background() + updateStarted := time.Now() + src := &bytes.Buffer{} + fmt.Fprintln(src, "//go:build !ignore_autogenerated") + license := lo.Must(os.ReadFile("hack/boilerplate.go.txt")) + fmt.Fprintln(src, string(license)) + fmt.Fprintln(src, "package pricing") + fmt.Fprintln(src, `import "time"`) + now := time.Now().UTC().Format(time.RFC3339) + fmt.Fprintf(src, "// generated at %s\n\n\n", now) + fmt.Fprintf(src, "var initialPriceUpdate, _ = time.Parse(time.RFC3339, \"%s\")\n", now) + fmt.Fprintln(src, "var initialOnDemandPrices = map[string]map[string]float64{}") + fmt.Fprintln(src, "func init() {") + // record prices for each region + var pricingProviderByRegion = map[string]chan *pricing.Provider{} + for _, region := range regions { + resultsChan := make(chan *pricing.Provider) + log.Println("fetching pricing data in region", region) + go func(region string, resultsChan chan *pricing.Provider) { + pricingProvider := pricing.NewProvider(ctx, pricing.NewAPI(), region, make(chan struct{})) + attempts := 0 + for { + if pricingProvider.OnDemandLastUpdated().After(updateStarted) { + break + } + + if attempts == 0 { + log.Println("started wait loop for pricing update on region", region) + } else if attempts%10 == 0 { + log.Printf("waiting on pricing update on region %s...\n", region) + } else if time.Now().Sub(updateStarted) >= time.Minute*2 { + log.Fatalf("failed to update region %s within 2 minutes", region) + } + time.Sleep(1 * time.Second) + attempts += 1 + } + log.Printf("fetched pricing for region %s\n", region) + resultsChan <- pricingProvider + }(region, resultsChan) + pricingProviderByRegion[region] = resultsChan + } + for _, region := range regions { + pricingProviderChan := pricingProviderByRegion[region] + var pricingProvider *pricing.Provider = <-pricingProviderChan + log.Println("writing output for", region) + instanceTypes := pricingProvider.InstanceTypes() + sort.Strings(instanceTypes) + + writePricing(src, instanceTypes, region, pricingProvider.OnDemandPrice) + } + fmt.Fprintln(src, "}") + formatted, err := format.Source(src.Bytes()) + if err != nil { + if err := os.WriteFile(filePath, src.Bytes(), 0644); err != nil { + log.Fatalf("writing output, %s", err) + } + log.Fatalf("formatting generated source, %s", err) + } + + if err := os.WriteFile(filePath, formatted, 0644); err != nil { + log.Fatalf("writing output, %s", err) + } + runtime.GC() + if err := pprof.WriteHeapProfile(f); err != nil { + log.Fatal("could not write memory profile: ", err) + } + log.Printf("successfully generated pricing file: \"%s\"\n", filePath) +} + +func writePricing(src *bytes.Buffer, instanceNames []string, region string, getPrice func(instanceType string) (float64, bool)) { + fmt.Fprintf(src, "// %s\n", region) + fmt.Fprintf(src, "initialOnDemandPrices[%q] = map[string]float64{\n", region) + sort.Strings(instanceNames) + for _, instanceName := range instanceNames { + price, ok := getPrice(instanceName) + if !ok { + continue + } + + // TODO: look at grouping by families to make the generated output nicer: + // https://github.com/Azure/karpenter/pull/94#discussion_r1120901524 + _, err := fmt.Fprintf(src, "\"%s\":%f, \n", instanceName, price) + if err != nil { + log.Fatalf("error writing, %s", err) + } + } + fmt.Fprintln(src, "\n}") +} diff --git a/hack/codegen.sh b/hack/codegen.sh new file mode 100755 index 000000000..c0444eb9f --- /dev/null +++ b/hack/codegen.sh @@ -0,0 +1,114 @@ +#!/usr/bin/env bash +set -euo pipefail + +if [ -z ${ENABLE_GIT_PUSH+x} ];then + ENABLE_GIT_PUSH=false +fi + +echo "api-code-gen running ENABLE_GIT_PUSH: ${ENABLE_GIT_PUSH}" + +pricing() { + GENERATED_FILE="pkg/providers/pricing/zz_generated.pricing.go" + NO_UPDATE=$' pkg/providers/pricing/zz_generated.pricing.go | 4 ++--\n 1 file changed, 2 insertions(+), 2 deletions(-)' + SUBJECT="Pricing" + + go run hack/code/prices_gen.go -- "${GENERATED_FILE}" + + GIT_DIFF=$(git diff --stat "${GENERATED_FILE}") + checkForUpdates "${GIT_DIFF}" "${NO_UPDATE}" "${SUBJECT} beside timestamps since last update" "${GENERATED_FILE}" +} + +skugen() { + SUBSCRIPTION_ID=$(az account show --query 'id' --output tsv) + export SUBSCRIPTION_ID + if [ -z "${SUBSCRIPTION_ID}" ]; then + echo "No subscription is set. Please login and set a subscription." + return 1 + fi + + RG_NAME="skugen-sp" + RG_LOCATION="eastus" + + az group create --name "${RG_NAME}" --location "${RG_LOCATION}" > /dev/null 2>&1 + + SP_NAME="skugen-$(date +%s)" + + # Note to use skugen you need valid azure credentials, and references to token credentials + # you can create a token credential by running the following command: + # az ad sp create-for-rbac --name "skugen" --role contributor --scopes /subscriptions//resourceGroups/ + # and then export the following environment variables that use the returned Service Principal JSON: + # export TENANT_ID= + # export AAD_CLIENT_ID= + # export AAD_CLIENT_SECRET= + # This script automatically creates these variables and deletes the service principal after running, + # but just note these variables are required to interact with the skugen API. + + # Fix for SC2086 + SP_JSON=$(az ad sp create-for-rbac --name "${SP_NAME}" --role contributor --scopes /subscriptions/"${SUBSCRIPTION_ID}"/resourceGroups/"${RG_NAME}") + + # Fixes for SC2155 and SC2086 combined + TENANT_ID=$(echo "$SP_JSON" | jq -r '.tenant') + export TENANT_ID + + AAD_CLIENT_ID=$(echo "$SP_JSON" | jq -r '.appId') + export AAD_CLIENT_ID + + AAD_CLIENT_SECRET=$(echo "$SP_JSON" | jq -r '.password') + export AAD_CLIENT_SECRET + + GENERATED_FILE=$(pwd)/"pkg/fake/zz_generated.sku.go" + echo GENERATED_FILE: "${GENERATED_FILE}" + SUBJECT="SKUGEN" + NO_UPDATE=$' pkg/fake/zz_generated.sku.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)' + + go run hack/code/instancetype_testdata_gen.go -- "${GENERATED_FILE}" "eastus" "Standard_B1s,Standard_A0,Standard_D2_v2,Standard_D2_v3,Standard_DS2_v2,Standard_D2s_v3,Standard_D2_v5,Standard_F16s_v2,Standard_NC24ads_A100_v4,Standard_M8-2ms,Standard_D4s_v3,Standard_D64s_v3" + + GIT_DIFF=$(git diff --stat "${GENERATED_FILE}") + checkForUpdates "${GIT_DIFF}" "${NO_UPDATE}" "${SUBJECT} beside timestamps since last update" "${GENERATED_FILE}" + + az ad sp delete --id "${AAD_CLIENT_ID}" +} + + +checkForUpdates() { + GIT_DIFF=$1 + NO_UPDATE=$2 + SUBJECT=$3 + GENERATED_FILE=$4 + + echo "Checking git diff for updates. ${GIT_DIFF}, ${NO_UPDATE}" + if [[ "${GIT_DIFF}" == "${NO_UPDATE}" ]]; then + noUpdates "${SUBJECT}" + git checkout "${GENERATED_FILE}" + else + echo "true" >/tmp/api-code-gen-updates + git add "${GENERATED_FILE}" + if [[ $ENABLE_GIT_PUSH == true ]]; then + gitCommitAndPush "${SUBJECT}" + fi + fi +} + +gitOpenAndPullBranch() { + git fetch origin + git checkout api-code-gen || git checkout -b api-code-gen || true +} + +gitCommitAndPush() { + UPDATE_SUBJECT=$1 + git commit -m "APICodeGen updates from Azure API for ${UPDATE_SUBJECT}" + git push --set-upstream origin api-code-gen +} + +noUpdates() { + UPDATE_SUBJECT=$1 + echo "No updates from Azure API for ${UPDATE_SUBJECT}" +} + +if [[ $ENABLE_GIT_PUSH == true ]]; then + gitOpenAndPullBranch +fi + +# Run all the codegen scripts +pricing +skugen diff --git a/hack/go-helper-image/Dockerfile b/hack/go-helper-image/Dockerfile new file mode 100644 index 000000000..429f4700b --- /dev/null +++ b/hack/go-helper-image/Dockerfile @@ -0,0 +1,36 @@ +ARG GOVERSION=1.21 +FROM --platform=$BUILDPLATFORM golang:${GOVERSION} as delve +ARG BUILDPLATFORM +ARG TARGETOS +ARG TARGETARCH + +ARG DELVE_VERSION=1.21.0 + +# Patch delve to make defaults for --check-go-version and --only-same-user +# to be set at build time. We must install patch(1) to apply the patch. +# +# We default --check-go-version to false to support binaries compiled +# with unsupported versions of Go. Delve issues a prominent warning. +# +# We default --only-same-user to false as `kubectl port-forward` +# to dlv port is refused otherwise. +RUN apt-get update && apt-get install -y --no-install-recommends \ + patch +RUN curl --location --output delve.tar.gz https://github.com/go-delve/delve/archive/v$DELVE_VERSION.tar.gz \ + && tar xzf delve.tar.gz \ + && mv delve-$DELVE_VERSION delve-source +COPY delve-*.patch . +RUN patch -p0 -d delve-source < delve-as-options.patch + +# Produce an as-static-as-possible dlv binary to work on musl and glibc +RUN cd delve-source \ + && CGO_ENABLED=0 GOOS=$TARGETOS GOARCH=$TARGETARCH go build -o /go/dlv -ldflags '-s -w -X github.com/go-delve/delve/cmd/dlv/cmds.checkGoVersionDefault=false -X github.com/go-delve/delve/cmd/dlv/cmds.checkLocalConnUserDefault=false -extldflags "-static"' ./cmd/dlv/ + +# Now populate the duct-tape image with the language runtime debugging support files +# The debian image is about 95MB bigger +FROM busybox +# The install script copies all files in /duct-tape to /dbg +COPY install.sh / +CMD ["/bin/sh", "/install.sh"] +WORKDIR /duct-tape +COPY --from=delve /go/dlv go/bin/ diff --git a/hack/go-helper-image/delve-as-options.patch b/hack/go-helper-image/delve-as-options.patch new file mode 100644 index 000000000..2405efe6c --- /dev/null +++ b/hack/go-helper-image/delve-as-options.patch @@ -0,0 +1,41 @@ +diff --git cmd/dlv/cmds/commands.go cmd/dlv/cmds/commands.go +index 15df5f6..f145330 100644 +--- cmd/dlv/cmds/commands.go ++++ cmd/dlv/cmds/commands.go +@@ -46,6 +46,10 @@ var ( + apiVersion int + // acceptMulti allows multiple clients to connect to the same server + acceptMulti bool ++ // checkGoVersionDefault sets default for --check-go-version ++ checkGoVersionDefault = "true" ++ // checkLocalConnUserDefault sets default for --only-same-user ++ checkLocalConnUserDefault = "true" + // addr is the debugging server listen address. + addr string + // initFile is the path to initialization file. +@@ -139,8 +143,8 @@ func New(docCall bool) *cobra.Command { + rootCommand.PersistentFlags().StringVar(&initFile, "init", "", "Init file, executed by the terminal client.") + rootCommand.PersistentFlags().StringVar(&buildFlags, "build-flags", buildFlagsDefault, "Build flags, to be passed to the compiler. For example: --build-flags=\"-tags=integration -mod=vendor -cover -v\"") + rootCommand.PersistentFlags().StringVar(&workingDir, "wd", "", "Working directory for running the program.") +- rootCommand.PersistentFlags().BoolVarP(&checkGoVersion, "check-go-version", "", true, "Exits if the version of Go in use is not compatible (too old or too new) with the version of Delve.") +- rootCommand.PersistentFlags().BoolVarP(&checkLocalConnUser, "only-same-user", "", true, "Only connections from the same user that started this instance of Delve are allowed to connect.") ++ rootCommand.PersistentFlags().BoolVarP(&checkGoVersion, "check-go-version", "", parseBool(checkGoVersionDefault), "Exits if the version of Go in use is not compatible (too old or too new) with the version of Delve.") ++ rootCommand.PersistentFlags().BoolVarP(&checkLocalConnUser, "only-same-user", "", parseBool(checkLocalConnUserDefault), "Only connections from the same user that started this instance of Delve are allowed to connect.") + rootCommand.PersistentFlags().StringVar(&backend, "backend", "default", `Backend selection (see 'dlv help backend').`) + rootCommand.PersistentFlags().StringArrayVarP(&redirects, "redirect", "r", []string{}, "Specifies redirect rules for target process (see 'dlv help redirect')") + rootCommand.PersistentFlags().BoolVar(&allowNonTerminalInteractive, "allow-non-terminal-interactive", false, "Allows interactive sessions of Delve that don't have a terminal as stdin, stdout and stderr") +@@ -1020,3 +1024,14 @@ func parseRedirects(redirects []string) ([3]string, error) { + } + return r, nil + } ++ ++// parseBool parses a boolean value represented by a string, and panics if there is an error. ++// It is intended for boolean build-time constants that are set with 'go build -ldflags=-X xxx=bool' ++// and should only be a valid value. ++func parseBool(value string) bool { ++ b, err := strconv.ParseBool(value) ++ if err != nil { ++ panic(err) ++ } ++ return b ++} diff --git a/hack/go-helper-image/install.sh b/hack/go-helper-image/install.sh new file mode 100644 index 000000000..600c9e69e --- /dev/null +++ b/hack/go-helper-image/install.sh @@ -0,0 +1,11 @@ +#!/bin/sh +set -e + +if [ ! -d /dbg ]; then + echo "Error: installation requires a volume mount at /dbg" 1>&2 + exit 1 +fi + +echo "Installing runtime debugging support files in /dbg" +tar cf - -C /duct-tape . | tar xf - -C /dbg +echo "Installation complete" diff --git a/hack/toolchain.sh b/hack/toolchain.sh new file mode 100755 index 000000000..cd4a62328 --- /dev/null +++ b/hack/toolchain.sh @@ -0,0 +1,41 @@ +#!/usr/bin/env bash +set -euo pipefail + +K8S_VERSION="${K8S_VERSION:="1.27.x"}" +KUBEBUILDER_ASSETS="${KUBEBUILDER_ASSETS:="${HOME}/.kubebuilder/bin"}" + +main() { + tools + kubebuilder +} + +tools() { + go install github.com/google/go-licenses@v1.6.0 + go install github.com/golangci/golangci-lint/cmd/golangci-lint@v1.55.1 + go install github.com/google/ko@v0.15.0 + go install github.com/mikefarah/yq/v4@v4.35.2 + go install github.com/norwoodj/helm-docs/cmd/helm-docs@v1.11.3 + go install sigs.k8s.io/controller-runtime/tools/setup-envtest@v0.0.0-20231031112659-edb7fe9c520e + go install sigs.k8s.io/controller-tools/cmd/controller-gen@v0.13.0 + go install github.com/sigstore/cosign/cmd/cosign@v1.13.1 +# go install -tags extended github.com/gohugoio/hugo@v0.110.0 + go install golang.org/x/vuln/cmd/govulncheck@v1.0.1 + go install github.com/onsi/ginkgo/v2/ginkgo@latest + + if ! echo "$PATH" | grep -q "${GOPATH:-undefined}/bin\|$HOME/go/bin"; then + echo "Go workspace's \"bin\" directory is not in PATH. Run 'export PATH=\"\$PATH:\${GOPATH:-\$HOME/go}/bin\"'." + fi +} + +kubebuilder() { + mkdir -p "$KUBEBUILDER_ASSETS" + arch=$(go env GOARCH) + ## Kubebuilder does not support darwin/arm64, so use amd64 through Rosetta instead + if [[ $(go env GOOS) == "darwin" ]] && [[ $(go env GOARCH) == "arm64" ]]; then + arch="amd64" + fi + ln -sf "$(setup-envtest use -p path "${K8S_VERSION}" --arch="${arch}" --bin-dir="${KUBEBUILDER_ASSETS}")"/* "${KUBEBUILDER_ASSETS}" + find "$KUBEBUILDER_ASSETS" +} + +main "$@" diff --git a/pkg/alt/karpenter-core/pkg/operator/logger.go b/pkg/alt/karpenter-core/pkg/operator/logger.go new file mode 100644 index 000000000..6f69e388e --- /dev/null +++ b/pkg/alt/karpenter-core/pkg/operator/logger.go @@ -0,0 +1,59 @@ +/* +Portions Copyright (c) Microsoft Corporation. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Unmodified copy of ignoreDebugEventsSink from +// https://github.com/aws/karpenter-core/blob/v0.30.0/pkg/operator/logger.go + +package operator + +import ( + "github.com/go-logr/logr" +) + +type ignoreDebugEventsSink struct { + name string + sink logr.LogSink +} + +func (i ignoreDebugEventsSink) Init(ri logr.RuntimeInfo) { + i.sink.Init(ri) +} + +func (i ignoreDebugEventsSink) Enabled(level int) bool { return i.sink.Enabled(level) } + +func (i ignoreDebugEventsSink) Info(level int, msg string, keysAndValues ...interface{}) { + // ignore debug "events" logs + if level == 1 && i.name == "events" { + return + } + i.sink.Info(level, msg, keysAndValues...) +} + +func (i ignoreDebugEventsSink) Error(err error, msg string, keysAndValues ...interface{}) { + i.sink.Error(err, msg, keysAndValues...) +} + +func (i ignoreDebugEventsSink) WithValues(keysAndValues ...interface{}) logr.LogSink { + return i.sink.WithValues(keysAndValues...) +} + +func (i ignoreDebugEventsSink) WithName(name string) logr.LogSink { + return &ignoreDebugEventsSink{name: name, sink: i.sink.WithName(name)} +} + +func ignoreDebugEvents(logger logr.Logger) logr.Logger { + return logr.New(&ignoreDebugEventsSink{sink: logger.GetSink()}) +} diff --git a/pkg/alt/karpenter-core/pkg/operator/operator.go b/pkg/alt/karpenter-core/pkg/operator/operator.go new file mode 100644 index 000000000..5891bac73 --- /dev/null +++ b/pkg/alt/karpenter-core/pkg/operator/operator.go @@ -0,0 +1,231 @@ +/* +Portions Copyright (c) Microsoft Corporation. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Source: github.com/aws/karpenter-core@v0.30.0/pkg/operator/operator.go + +package operator + +import ( + "context" + "fmt" + "net/http" + "net/http/pprof" + "os" + "runtime/debug" + "time" + + "github.com/go-logr/zapr" + "github.com/samber/lo" + + "github.com/aws/karpenter-core/pkg/apis" + + "github.com/aws/karpenter-core/pkg/apis/v1alpha5" + "github.com/aws/karpenter-core/pkg/apis/v1beta1" + "github.com/aws/karpenter-core/pkg/events" + + coreoperator "github.com/aws/karpenter-core/pkg/operator" + coreoperatorlogging "github.com/aws/karpenter-core/pkg/operator/logging" + + "github.com/aws/karpenter-core/pkg/operator/injection" + "github.com/aws/karpenter-core/pkg/operator/options" + "github.com/aws/karpenter-core/pkg/operator/scheme" + coordinationv1 "k8s.io/api/coordination/v1" + v1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/fields" + + //"k8s.io/apimachinery/pkg/fields" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/rest" + "k8s.io/client-go/tools/leaderelection/resourcelock" + "k8s.io/client-go/util/flowcontrol" + "k8s.io/utils/clock" + knativeinjection "knative.dev/pkg/injection" + knativelogging "knative.dev/pkg/logging" + "knative.dev/pkg/signals" + "knative.dev/pkg/system" + controllerruntime "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/cache" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/healthz" + "sigs.k8s.io/controller-runtime/pkg/metrics/server" +) + +// Unmodified; for exposing private entity only +const ( + appName = "karpenter" + component = "controller" +) + +// Source: NewOperator() +// Modified behavior: +// - Allow Karpenter and most components to exist on control plane, but can reach the CRs on overlay +// - Webhooks not supported +// - Karpenter will not crash if CRDs are not found, but goes into a retry loop for a while +// Modified implementations: +// - Split the context into two: control plane and overlay +// - Remove webhooks-related code +// - Retry loop for getting CRDs +// - Introduce and retrieve overlay namespace from env +// - No profiling +// The code is purposefully kept in the similar structure as the original for easy comparison +// nolint:revive,stylecheck +func NewOperator() (context.Context, *coreoperator.Operator) { + overlayNamespace := os.Getenv("OVERLAY_NAMESPACE") + if overlayNamespace == "" { + overlayNamespace = "karpenter-system" + } + + // Root Context + originCtx := signals.NewContext() + ccPlaneCtx := knativeinjection.WithNamespaceScope(originCtx, system.Namespace()) + overlayCtx := knativeinjection.WithNamespaceScope(originCtx, overlayNamespace) + + // Options + ccPlaneCtx = injection.WithOptionsOrDie(ccPlaneCtx, options.Injectables...) + overlayCtx = injection.WithOptionsOrDie(overlayCtx, options.Injectables...) + + // Make the Karpenter binary aware of the container memory limit + // https://pkg.go.dev/runtime/debug#SetMemoryLimit + if options.FromContext(ccPlaneCtx).MemoryLimit > 0 { + newLimit := int64(float64(options.FromContext(ccPlaneCtx).MemoryLimit) * 0.9) + debug.SetMemoryLimit(newLimit) + } + + // Webhook + // Unsupported -- skipping + + // Client Config + ccPlaneConfig := lo.Must(rest.InClusterConfig()) + overlayConfig := controllerruntime.GetConfigOrDie() + ccPlaneConfig.RateLimiter = flowcontrol.NewTokenBucketRateLimiter(float32(options.FromContext(ccPlaneCtx).KubeClientQPS), options.FromContext(ccPlaneCtx).KubeClientBurst) + overlayConfig.RateLimiter = flowcontrol.NewTokenBucketRateLimiter(float32(options.FromContext(overlayCtx).KubeClientQPS), options.FromContext(overlayCtx).KubeClientBurst) + // config.UserAgent = fmt.Sprintf("%s/%s", appName, Version) + ccPlaneConfig.UserAgent = appName + overlayConfig.UserAgent = appName + + // Client + ccPlaneKubernetesInterface := kubernetes.NewForConfigOrDie(ccPlaneConfig) + overlayKubernetesInterface := kubernetes.NewForConfigOrDie(overlayConfig) + //configMapWatcher := informer.NewInformedWatcher(ccPlaneKubernetesInterface, system.Namespace()) + //lo.Must0(configMapWatcher.Start(ccPlaneCtx.Done())) + + // Inject settings from the ConfigMap(s) into the context + ccPlaneCtx = injection.WithSettingsOrDie(ccPlaneCtx, ccPlaneKubernetesInterface, apis.Settings...) + overlayCtx = injection.WithSettingsOrDie(overlayCtx, ccPlaneKubernetesInterface, apis.Settings...) // Note: still using ccPlaneKubernetesInterface + + // Temporarily merge settings into options until configmap is removed + // Note: injectables are pointer to those already in context + for _, o := range options.Injectables { + o.MergeSettings(ccPlaneCtx) + o.MergeSettings(overlayCtx) + } + + // Logging + logger := coreoperatorlogging.NewLogger(ccPlaneCtx, component, ccPlaneKubernetesInterface) + ccPlaneCtx = knativelogging.WithLogger(ccPlaneCtx, logger) + overlayCtx = knativelogging.WithLogger(overlayCtx, logger) + coreoperatorlogging.ConfigureGlobalLoggers(ccPlaneCtx) + + // Inject settings from the ConfigMap(s) into the context + ccPlaneCtx = injection.WithSettingsOrDie(ccPlaneCtx, ccPlaneKubernetesInterface, apis.Settings...) + overlayCtx = injection.WithSettingsOrDie(overlayCtx, ccPlaneKubernetesInterface, apis.Settings...) + + // Temporarily merge settings into options until configmap is removed + // Note: injectables are pointer to those already in context + for _, o := range options.Injectables { + o.MergeSettings(ccPlaneCtx) + o.MergeSettings(overlayCtx) + } + + // Manager + mgrOpts := controllerruntime.Options{ + Logger: ignoreDebugEvents(zapr.NewLogger(logger.Desugar())), + LeaderElection: options.FromContext(overlayCtx).EnableLeaderElection, + LeaderElectionID: "karpenter-leader-election", + LeaderElectionResourceLock: resourcelock.LeasesResourceLock, + Scheme: scheme.Scheme, + Metrics: server.Options{ + BindAddress: fmt.Sprintf(":%d", options.FromContext(overlayCtx).MetricsPort), + }, + HealthProbeBindAddress: fmt.Sprintf(":%d", options.FromContext(overlayCtx).HealthProbePort), + BaseContext: func() context.Context { + ctx := context.Background() + ctx = knativelogging.WithLogger(ctx, logger) + ctx = injection.WithSettingsOrDie(ctx, ccPlaneKubernetesInterface, apis.Settings...) + // ctx = injection.WithConfig(ctx, overlayConfig) + ctx = injection.WithOptionsOrDie(ctx, options.Injectables...) + return ctx + }, + Cache: cache.Options{ + ByObject: map[client.Object]cache.ByObject{ + &coordinationv1.Lease{}: { + Field: fields.SelectorFromSet(fields.Set{"metadata.namespace": "kube-node-lease"}), + }, + }, + }, + } + if options.FromContext(ccPlaneCtx).EnableProfiling { + mgrOpts.Metrics.ExtraHandlers = lo.Assign(mgrOpts.Metrics.ExtraHandlers, map[string]http.Handler{ + "/debug/pprof/": http.HandlerFunc(pprof.Index), + "/debug/pprof/cmdline": http.HandlerFunc(pprof.Cmdline), + "/debug/pprof/profile": http.HandlerFunc(pprof.Profile), + "/debug/pprof/symbol": http.HandlerFunc(pprof.Symbol), + "/debug/pprof/trace": http.HandlerFunc(pprof.Trace), + "/debug/pprof/allocs": pprof.Handler("allocs"), + "/debug/pprof/heap": pprof.Handler("heap"), + "/debug/pprof/block": pprof.Handler("block"), + "/debug/pprof/goroutine": pprof.Handler("goroutine"), + "/debug/pprof/threadcreate": pprof.Handler("threadcreate"), + }) + } + mgr, err := controllerruntime.NewManager(overlayConfig, mgrOpts) + mgr = lo.Must(mgr, err, "failed to setup manager") + + lo.Must0(mgr.GetFieldIndexer().IndexField(overlayCtx, &v1.Pod{}, "spec.nodeName", func(o client.Object) []string { + return []string{o.(*v1.Pod).Spec.NodeName} + }), "failed to setup pod indexer") + lo.Must0(mgr.GetFieldIndexer().IndexField(overlayCtx, &v1.Node{}, "spec.providerID", func(o client.Object) []string { + return []string{o.(*v1.Node).Spec.ProviderID} + }), "failed to setup node provider id indexer") + lo.Must0(func() error { + _, _, err := lo.AttemptWithDelay(42, 10*time.Second, func(index int, duration time.Duration) error { + err := mgr.GetFieldIndexer().IndexField(overlayCtx, &v1beta1.NodeClaim{}, "status.providerID", func(o client.Object) []string { + return []string{o.(*v1beta1.NodeClaim).Status.ProviderID} + }) + if err != nil { + knativelogging.FromContext(ccPlaneCtx).Infof("failed to setup machine provider id indexer, CRDs deployment may not be ready, index: %d, duration: %v, err: %v", index, duration, err) + } + return err + }) + return err + }(), "failed to setup nodeclaim provider id indexer, all attempts used") + lo.Must0(mgr.GetFieldIndexer().IndexField(overlayCtx, &v1alpha5.Machine{}, "status.providerID", func(o client.Object) []string { + return []string{o.(*v1alpha5.Machine).Status.ProviderID} + }), "failed to setup machine provider id indexer") + + lo.Must0(mgr.AddReadyzCheck("manager", func(req *http.Request) error { + return lo.Ternary(mgr.GetCache().WaitForCacheSync(req.Context()), nil, fmt.Errorf("failed to sync caches")) + })) + lo.Must0(mgr.AddHealthzCheck("healthz", healthz.Ping)) + lo.Must0(mgr.AddReadyzCheck("readyz", healthz.Ping)) + + return ccPlaneCtx, &coreoperator.Operator{ + Manager: mgr, + KubernetesInterface: overlayKubernetesInterface, + EventRecorder: events.NewRecorder(mgr.GetEventRecorderFor(appName)), + Clock: clock.RealClock{}, + } +} diff --git a/pkg/apis/apis.go b/pkg/apis/apis.go new file mode 100644 index 000000000..68014ec31 --- /dev/null +++ b/pkg/apis/apis.go @@ -0,0 +1,56 @@ +/* +Portions Copyright (c) Microsoft Corporation. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package apis contains Kubernetes API groups. +package apis + +import ( + _ "embed" + + v1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" + "k8s.io/apimachinery/pkg/runtime" + + "github.com/aws/karpenter-core/pkg/operator/scheme" + + "github.com/samber/lo" + + "github.com/Azure/karpenter/pkg/apis/settings" + "github.com/Azure/karpenter/pkg/apis/v1alpha2" + "github.com/aws/karpenter-core/pkg/apis" + coresettings "github.com/aws/karpenter-core/pkg/apis/settings" + "github.com/aws/karpenter-core/pkg/utils/functional" +) + +var ( + // Builder includes all types within the apis package + Builder = runtime.NewSchemeBuilder( + v1alpha2.SchemeBuilder.AddToScheme, + ) + // AddToScheme may be used to add all resources defined in the project to a Scheme + AddToScheme = Builder.AddToScheme + Settings = []coresettings.Injectable{&settings.Settings{}} +) + +//go:generate controller-gen crd object:headerFile="../../hack/boilerplate.go.txt" paths="./..." output:crd:artifacts:config=crds +var ( + //go:embed crds/karpenter.azure.com_aksnodeclasses.yaml + AKSNodeClassCRD []byte + CRDs = append(apis.CRDs, lo.Must(functional.Unmarshal[v1.CustomResourceDefinition](AKSNodeClassCRD))) +) + +func init() { + lo.Must0(AddToScheme(scheme.Scheme)) +} diff --git a/pkg/apis/crds/karpenter.azure.com_aksnodeclasses.yaml b/pkg/apis/crds/karpenter.azure.com_aksnodeclasses.yaml new file mode 100644 index 000000000..6af87f265 --- /dev/null +++ b/pkg/apis/crds/karpenter.azure.com_aksnodeclasses.yaml @@ -0,0 +1,72 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.13.0 + name: aksnodeclasses.karpenter.azure.com +spec: + group: karpenter.azure.com + names: + categories: + - karpenter + kind: AKSNodeClass + listKind: AKSNodeClassList + plural: aksnodeclasses + shortNames: + - aksnc + - aksncs + singular: aksnodeclass + scope: Cluster + versions: + - name: v1alpha2 + schema: + openAPIV3Schema: + description: AKSNodeClass is the Schema for the AKSNodeClass API + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: AKSNodeClassSpec is the top level specification for the AKS + Karpenter Provider. This will contain configuration necessary to launch + instances in AKS. + properties: + imageFamily: + default: Ubuntu2204 + description: ImageFamily is the image family that instances use. + enum: + - Ubuntu2204 + type: string + imageVersion: + description: ImageVersion is the image version that instances use. + type: string + osDiskSizeGB: + default: 128 + description: osDiskSizeGB is the size of the OS disk in GB. + format: int32 + minimum: 100 + type: integer + tags: + additionalProperties: + type: string + description: Tags to be applied on Azure resources like instances. + type: object + type: object + status: + description: AKSNodeClassStatus contains the resolved state of the AKSNodeClass + type: object + type: object + served: true + storage: true + subresources: + status: {} diff --git a/pkg/apis/crds/karpenter.sh_machines.yaml b/pkg/apis/crds/karpenter.sh_machines.yaml new file mode 100644 index 000000000..fa753deb5 --- /dev/null +++ b/pkg/apis/crds/karpenter.sh_machines.yaml @@ -0,0 +1,354 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.13.0 + name: machines.karpenter.sh +spec: + group: karpenter.sh + names: + categories: + - karpenter + kind: Machine + listKind: MachineList + plural: machines + singular: machine + scope: Cluster + versions: + - additionalPrinterColumns: + - jsonPath: .metadata.labels.node\.kubernetes\.io/instance-type + name: Type + type: string + - jsonPath: .metadata.labels.topology\.kubernetes\.io/zone + name: Zone + type: string + - jsonPath: .status.nodeName + name: Node + type: string + - jsonPath: .status.conditions[?(@.type=="Ready")].status + name: Ready + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + - jsonPath: .metadata.labels.karpenter\.sh/capacity-type + name: Capacity + priority: 1 + type: string + - jsonPath: .metadata.labels.karpenter\.sh/provisioner-name + name: Provisioner + priority: 1 + type: string + - jsonPath: .spec.machineTemplateRef.name + name: Template + priority: 1 + type: string + name: v1alpha5 + schema: + openAPIV3Schema: + description: Machine is the Schema for the Machines API + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: MachineSpec describes the desired state of the Machine + properties: + kubelet: + description: Kubelet are options passed to the kubelet when provisioning + nodes + properties: + clusterDNS: + description: clusterDNS is a list of IP addresses for the cluster + DNS server. Note that not all providers may use all addresses. + items: + type: string + type: array + containerRuntime: + description: ContainerRuntime is the container runtime to be used + with your worker nodes. + type: string + cpuCFSQuota: + description: CPUCFSQuota enables CPU CFS quota enforcement for + containers that specify CPU limits. + type: boolean + evictionHard: + additionalProperties: + type: string + description: EvictionHard is the map of signal names to quantities + that define hard eviction thresholds + type: object + evictionMaxPodGracePeriod: + description: EvictionMaxPodGracePeriod is the maximum allowed + grace period (in seconds) to use when terminating pods in response + to soft eviction thresholds being met. + format: int32 + type: integer + evictionSoft: + additionalProperties: + type: string + description: EvictionSoft is the map of signal names to quantities + that define soft eviction thresholds + type: object + evictionSoftGracePeriod: + additionalProperties: + type: string + description: EvictionSoftGracePeriod is the map of signal names + to quantities that define grace periods for each eviction signal + type: object + imageGCHighThresholdPercent: + description: ImageGCHighThresholdPercent is the percent of disk + usage after which image garbage collection is always run. The + percent is calculated by dividing this field value by 100, so + this field must be between 0 and 100, inclusive. When specified, + the value must be greater than ImageGCLowThresholdPercent. + format: int32 + maximum: 100 + minimum: 0 + type: integer + imageGCLowThresholdPercent: + description: ImageGCLowThresholdPercent is the percent of disk + usage before which image garbage collection is never run. Lowest + disk usage to garbage collect to. The percent is calculated + by dividing this field value by 100, so the field value must + be between 0 and 100, inclusive. When specified, the value must + be less than imageGCHighThresholdPercent + format: int32 + maximum: 100 + minimum: 0 + type: integer + kubeReserved: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: KubeReserved contains resources reserved for Kubernetes + system components. + type: object + maxPods: + description: MaxPods is an override for the maximum number of + pods that can run on a worker node instance. + format: int32 + minimum: 0 + type: integer + podsPerCore: + description: PodsPerCore is an override for the number of pods + that can run on a worker node instance based on the number of + cpu cores. This value cannot exceed MaxPods, so, if MaxPods + is a lower value, that value will be used. + format: int32 + minimum: 0 + type: integer + systemReserved: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: SystemReserved contains resources reserved for OS + system daemons and kernel memory. + type: object + type: object + machineTemplateRef: + description: MachineTemplateRef is a reference to an object that defines + provider specific configuration + properties: + apiVersion: + description: API version of the referent + type: string + kind: + description: 'Kind of the referent; More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds"' + type: string + name: + description: 'Name of the referent; More info: http://kubernetes.io/docs/user-guide/identifiers#names' + type: string + required: + - name + type: object + requirements: + description: Requirements are layered with Labels and applied to every + node. + items: + description: A node selector requirement is a selector that contains + values, a key, and an operator that relates the key and values. + properties: + key: + description: The label key that the selector applies to. + type: string + operator: + description: Represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and + Lt. + type: string + values: + description: An array of string values. If the operator is In + or NotIn, the values array must be non-empty. If the operator + is Exists or DoesNotExist, the values array must be empty. + If the operator is Gt or Lt, the values array must have a + single element, which will be interpreted as an integer. This + array is replaced during a strategic merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + resources: + description: Resources models the resource requirements for the Machine + to launch + properties: + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: Requests describes the minimum required resources + for the Machine to launch + type: object + type: object + startupTaints: + description: StartupTaints are taints that are applied to nodes upon + startup which are expected to be removed automatically within a + short period of time, typically by a DaemonSet that tolerates the + taint. These are commonly used by daemonsets to allow initialization + and enforce startup ordering. StartupTaints are ignored for provisioning + purposes in that pods are not required to tolerate a StartupTaint + in order to have nodes provisioned for them. + items: + description: The node this Taint is attached to has the "effect" + on any pod that does not tolerate the Taint. + properties: + effect: + description: Required. The effect of the taint on pods that + do not tolerate the taint. Valid effects are NoSchedule, PreferNoSchedule + and NoExecute. + type: string + key: + description: Required. The taint key to be applied to a node. + type: string + timeAdded: + description: TimeAdded represents the time at which the taint + was added. It is only written for NoExecute taints. + format: date-time + type: string + value: + description: The taint value corresponding to the taint key. + type: string + required: + - effect + - key + type: object + type: array + taints: + description: Taints will be applied to the machine's node. + items: + description: The node this Taint is attached to has the "effect" + on any pod that does not tolerate the Taint. + properties: + effect: + description: Required. The effect of the taint on pods that + do not tolerate the taint. Valid effects are NoSchedule, PreferNoSchedule + and NoExecute. + type: string + key: + description: Required. The taint key to be applied to a node. + type: string + timeAdded: + description: TimeAdded represents the time at which the taint + was added. It is only written for NoExecute taints. + format: date-time + type: string + value: + description: The taint value corresponding to the taint key. + type: string + required: + - effect + - key + type: object + type: array + type: object + status: + description: MachineStatus defines the observed state of Machine + properties: + allocatable: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: Allocatable is the estimated allocatable capacity of + the machine + type: object + capacity: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: Capacity is the estimated full capacity of the machine + type: object + conditions: + description: Conditions contains signals for health and readiness + items: + description: 'Condition defines a readiness condition for a Knative + resource. See: https://github.com/kubernetes/community/blob/master/contributors/devel/sig-architecture/api-conventions.md#typical-status-properties' + properties: + lastTransitionTime: + description: LastTransitionTime is the last time the condition + transitioned from one status to another. We use VolatileTime + in place of metav1.Time to exclude this from creating equality.Semantic + differences (all other things held constant). + type: string + message: + description: A human readable message indicating details about + the transition. + type: string + reason: + description: The reason for the condition's last transition. + type: string + severity: + description: Severity with which to treat failures of this type + of condition. When this is not specified, it defaults to Error. + type: string + status: + description: Status of the condition, one of True, False, Unknown. + type: string + type: + description: Type of condition. + type: string + required: + - status + - type + type: object + type: array + nodeName: + description: NodeName is the name of the corresponding node object + type: string + providerID: + description: ProviderID of the corresponding node object + type: string + type: object + type: object + served: true + storage: true + subresources: + status: {} diff --git a/pkg/apis/crds/karpenter.sh_nodeclaims.yaml b/pkg/apis/crds/karpenter.sh_nodeclaims.yaml new file mode 100644 index 000000000..e99170cff --- /dev/null +++ b/pkg/apis/crds/karpenter.sh_nodeclaims.yaml @@ -0,0 +1,364 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.13.0 + name: nodeclaims.karpenter.sh +spec: + group: karpenter.sh + names: + categories: + - karpenter + kind: NodeClaim + listKind: NodeClaimList + plural: nodeclaims + singular: nodeclaim + scope: Cluster + versions: + - additionalPrinterColumns: + - jsonPath: .metadata.labels.node\.kubernetes\.io/instance-type + name: Type + type: string + - jsonPath: .metadata.labels.karpenter\.azure\.com/zone + name: Zone + type: string + - jsonPath: .status.nodeName + name: Node + type: string + - jsonPath: .status.conditions[?(@.type=="Ready")].status + name: Ready + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + - jsonPath: .metadata.labels.karpenter\.sh/capacity-type + name: Capacity + priority: 1 + type: string + - jsonPath: .metadata.labels.karpenter\.sh/nodepool + name: NodePool + priority: 1 + type: string + - jsonPath: .spec.nodeClassRef.name + name: NodeClass + priority: 1 + type: string + name: v1beta1 + schema: + openAPIV3Schema: + description: NodeClaim is the Schema for the NodeClaims API + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: NodeClaimSpec describes the desired state of the NodeClaim + properties: + kubelet: + description: Kubelet defines args to be used when configuring kubelet on provisioned nodes. They are a subset of the upstream types, recognizing not all options may be supported. Wherever possible, the types and names should reflect the upstream kubelet types. + properties: + clusterDNS: + description: clusterDNS is a list of IP addresses for the cluster DNS server. Note that not all providers may use all addresses. + items: + type: string + type: array + cpuCFSQuota: + description: CPUCFSQuota enables CPU CFS quota enforcement for containers that specify CPU limits. + type: boolean + evictionHard: + additionalProperties: + type: string + pattern: ^((\d{1,2}(\.\d{1,2})?|100(\.0{1,2})?)%||(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?)$ + description: EvictionHard is the map of signal names to quantities that define hard eviction thresholds + type: object + x-kubernetes-validations: + - message: valid keys for evictionHard are ['memory.available','nodefs.available','nodefs.inodesFree','imagefs.available','imagefs.inodesFree','pid.available'] + rule: self.all(x, x in ['memory.available','nodefs.available','nodefs.inodesFree','imagefs.available','imagefs.inodesFree','pid.available']) + evictionMaxPodGracePeriod: + description: EvictionMaxPodGracePeriod is the maximum allowed grace period (in seconds) to use when terminating pods in response to soft eviction thresholds being met. + format: int32 + type: integer + evictionSoft: + additionalProperties: + type: string + pattern: ^((\d{1,2}(\.\d{1,2})?|100(\.0{1,2})?)%||(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?)$ + description: EvictionSoft is the map of signal names to quantities that define soft eviction thresholds + type: object + x-kubernetes-validations: + - message: valid keys for evictionSoft are ['memory.available','nodefs.available','nodefs.inodesFree','imagefs.available','imagefs.inodesFree','pid.available'] + rule: self.all(x, x in ['memory.available','nodefs.available','nodefs.inodesFree','imagefs.available','imagefs.inodesFree','pid.available']) + evictionSoftGracePeriod: + additionalProperties: + type: string + description: EvictionSoftGracePeriod is the map of signal names to quantities that define grace periods for each eviction signal + type: object + x-kubernetes-validations: + - message: valid keys for evictionSoftGracePeriod are ['memory.available','nodefs.available','nodefs.inodesFree','imagefs.available','imagefs.inodesFree','pid.available'] + rule: self.all(x, x in ['memory.available','nodefs.available','nodefs.inodesFree','imagefs.available','imagefs.inodesFree','pid.available']) + imageGCHighThresholdPercent: + description: ImageGCHighThresholdPercent is the percent of disk usage after which image garbage collection is always run. The percent is calculated by dividing this field value by 100, so this field must be between 0 and 100, inclusive. When specified, the value must be greater than ImageGCLowThresholdPercent. + format: int32 + maximum: 100 + minimum: 0 + type: integer + imageGCLowThresholdPercent: + description: ImageGCLowThresholdPercent is the percent of disk usage before which image garbage collection is never run. Lowest disk usage to garbage collect to. The percent is calculated by dividing this field value by 100, so the field value must be between 0 and 100, inclusive. When specified, the value must be less than imageGCHighThresholdPercent + format: int32 + maximum: 100 + minimum: 0 + type: integer + kubeReserved: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: KubeReserved contains resources reserved for Kubernetes system components. + type: object + x-kubernetes-validations: + - message: valid keys for kubeReserved are ['cpu','memory','ephemeral-storage','pid'] + rule: self.all(x, x=='cpu' || x=='memory' || x=='ephemeral-storage' || x=='pid') + - message: kubeReserved value cannot be a negative resource quantity + rule: self.all(x, !self[x].startsWith('-')) + maxPods: + description: MaxPods is an override for the maximum number of pods that can run on a worker node instance. + format: int32 + minimum: 0 + type: integer + podsPerCore: + description: PodsPerCore is an override for the number of pods that can run on a worker node instance based on the number of cpu cores. This value cannot exceed MaxPods, so, if MaxPods is a lower value, that value will be used. + format: int32 + minimum: 0 + type: integer + systemReserved: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: SystemReserved contains resources reserved for OS system daemons and kernel memory. + type: object + x-kubernetes-validations: + - message: valid keys for systemReserved are ['cpu','memory','ephemeral-storage','pid'] + rule: self.all(x, x=='cpu' || x=='memory' || x=='ephemeral-storage' || x=='pid') + - message: systemReserved value cannot be a negative resource quantity + rule: self.all(x, !self[x].startsWith('-')) + type: object + x-kubernetes-validations: + - message: imageGCHighThresholdPercent must be greater than imageGCLowThresholdPercent + rule: 'has(self.imageGCHighThresholdPercent) && has(self.imageGCLowThresholdPercent) ? self.imageGCHighThresholdPercent > self.imageGCLowThresholdPercent : true' + - message: evictionSoft OwnerKey does not have a matching evictionSoftGracePeriod + rule: has(self.evictionSoft) ? self.evictionSoft.all(e, (e in self.evictionSoftGracePeriod)):true + - message: evictionSoftGracePeriod OwnerKey does not have a matching evictionSoft + rule: has(self.evictionSoftGracePeriod) ? self.evictionSoftGracePeriod.all(e, (e in self.evictionSoft)):true + nodeClassRef: + description: NodeClassRef is a reference to an object that defines provider specific configuration + properties: + apiVersion: + description: API version of the referent + type: string + kind: + description: 'Kind of the referent; More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds"' + type: string + name: + description: 'Name of the referent; More info: http://kubernetes.io/docs/user-guide/identifiers#names' + type: string + required: + - name + type: object + requirements: + description: Requirements are layered with GetLabels and applied to every node. + items: + description: A node selector requirement is a selector that contains values, a key, and an operator that relates the key and values. + properties: + key: + description: The label key that the selector applies to. + type: string + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*(\/))?([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9]$ + x-kubernetes-validations: + - message: label domain "kubernetes.io" is restricted + rule: self in ["beta.kubernetes.io/instance-type", "failure-domain.beta.kubernetes.io/region", "beta.kubernetes.io/os", "beta.kubernetes.io/arch", "failure-domain.beta.kubernetes.io/zone", "topology.kubernetes.io/zone", "topology.kubernetes.io/region", "node.kubernetes.io/instance-type", "kubernetes.io/arch", "kubernetes.io/os", "node.kubernetes.io/windows-build"] || self.startsWith("node.kubernetes.io/") || self.startsWith("node-restriction.kubernetes.io/") || !self.find("^([^/]+)").endsWith("kubernetes.io") + - message: label domain "k8s.io" is restricted + rule: self.startsWith("kops.k8s.io/") || !self.find("^([^/]+)").endsWith("k8s.io") + - message: label domain "karpenter.sh" is restricted + rule: self in ["karpenter.sh/capacity-type", "karpenter.sh/nodepool"] || !self.find("^([^/]+)").endsWith("karpenter.sh") + - message: label "kubernetes.io/hostname" is restricted + rule: self != "kubernetes.io/hostname" + operator: + description: Represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. + type: string + enum: + - In + - NotIn + - Exists + - DoesNotExist + - Gt + - Lt + values: + description: An array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. If the operator is Gt or Lt, the values array must have a single element, which will be interpreted as an integer. This array is replaced during a strategic merge patch. + items: + type: string + type: array + maxLength: 63 + pattern: ^(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])?$ + required: + - key + - operator + type: object + maxItems: 30 + type: array + x-kubernetes-validations: + - message: requirements with operator 'In' must have a value defined + rule: 'self.all(x, x.operator == ''In'' ? x.values.size() != 0 : true)' + - message: requirements operator 'Gt' or 'Lt' must have a single positive integer value + rule: 'self.all(x, (x.operator == ''Gt'' || x.operator == ''Lt'') ? (x.values.size() == 1 && int(x.values[0]) >= 0) : true)' + resources: + description: Resources models the resource requirements for the NodeClaim to launch + properties: + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: Requests describes the minimum required resources for the NodeClaim to launch + type: object + type: object + startupTaints: + description: StartupTaints are taints that are applied to nodes upon startup which are expected to be removed automatically within a short period of time, typically by a DaemonSet that tolerates the taint. These are commonly used by daemonsets to allow initialization and enforce startup ordering. StartupTaints are ignored for provisioning purposes in that pods are not required to tolerate a StartupTaint in order to have nodes provisioned for them. + items: + description: The node this Taint is attached to has the "effect" on any pod that does not tolerate the Taint. + properties: + effect: + description: Required. The effect of the taint on pods that do not tolerate the taint. Valid effects are NoSchedule, PreferNoSchedule and NoExecute. + type: string + enum: + - NoSchedule + - PreferNoSchedule + - NoExecute + key: + description: Required. The taint key to be applied to a node. + type: string + minLength: 1 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*(\/))?([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9]$ + timeAdded: + description: TimeAdded represents the time at which the taint was added. It is only written for NoExecute taints. + format: date-time + type: string + value: + description: The taint value corresponding to the taint key. + type: string + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*(\/))?([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9]$ + required: + - effect + - key + type: object + type: array + taints: + description: Taints will be applied to the NodeClaim's node. + items: + description: The node this Taint is attached to has the "effect" on any pod that does not tolerate the Taint. + properties: + effect: + description: Required. The effect of the taint on pods that do not tolerate the taint. Valid effects are NoSchedule, PreferNoSchedule and NoExecute. + type: string + enum: + - NoSchedule + - PreferNoSchedule + - NoExecute + key: + description: Required. The taint key to be applied to a node. + type: string + minLength: 1 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*(\/))?([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9]$ + timeAdded: + description: TimeAdded represents the time at which the taint was added. It is only written for NoExecute taints. + format: date-time + type: string + value: + description: The taint value corresponding to the taint key. + type: string + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*(\/))?([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9]$ + required: + - effect + - key + type: object + type: array + required: + - nodeClassRef + - requirements + type: object + status: + description: NodeClaimStatus defines the observed state of NodeClaim + properties: + allocatable: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: Allocatable is the estimated allocatable capacity of the node + type: object + capacity: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: Capacity is the estimated full capacity of the node + type: object + conditions: + description: Conditions contains signals for health and readiness + items: + description: 'Condition defines a readiness condition for a Knative resource. See: https://github.com/kubernetes/community/blob/master/contributors/devel/sig-architecture/api-conventions.md#typical-status-properties' + properties: + lastTransitionTime: + description: LastTransitionTime is the last time the condition transitioned from one status to another. We use VolatileTime in place of metav1.Time to exclude this from creating equality.Semantic differences (all other things held constant). + type: string + message: + description: A human readable message indicating details about the transition. + type: string + reason: + description: The reason for the condition's last transition. + type: string + severity: + description: Severity with which to treat failures of this type of condition. When this is not specified, it defaults to Error. + type: string + status: + description: Status of the condition, one of True, False, Unknown. + type: string + type: + description: Type of condition. + type: string + required: + - status + - type + type: object + type: array + imageID: + description: ImageID is an identifier for the image that runs on the node + type: string + nodeName: + description: NodeName is the name of the corresponding node object + type: string + providerID: + description: ProviderID of the corresponding node object + type: string + type: object + type: object + served: true + storage: true + subresources: + status: {} diff --git a/pkg/apis/crds/karpenter.sh_nodepools.yaml b/pkg/apis/crds/karpenter.sh_nodepools.yaml new file mode 100644 index 000000000..265f193af --- /dev/null +++ b/pkg/apis/crds/karpenter.sh_nodepools.yaml @@ -0,0 +1,378 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.13.0 + name: nodepools.karpenter.sh +spec: + group: karpenter.sh + names: + categories: + - karpenter + kind: NodePool + listKind: NodePoolList + plural: nodepools + singular: nodepool + scope: Cluster + versions: + - additionalPrinterColumns: + - jsonPath: .spec.template.spec.nodeClassRef.name + name: NodeClass + type: string + - jsonPath: .spec.weight + name: Weight + priority: 1 + type: string + name: v1beta1 + schema: + openAPIV3Schema: + description: NodePool is the Schema for the NodePools API + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: NodePoolSpec is the top level provisioner specification. Provisioners launch nodes in response to pods that are unschedulable. A single provisioner is capable of managing a diverse set of nodes. Node properties are determined from a combination of provisioner and pod scheduling constraints. + properties: + disruption: + default: + consolidationPolicy: WhenUnderutilized + expireAfter: 720h + description: Disruption contains the parameters that relate to Karpenter's disruption logic + properties: + consolidateAfter: + description: ConsolidateAfter is the duration the controller will wait before attempting to terminate nodes that are underutilized. Refer to ConsolidationPolicy for how underutilization is considered. + pattern: ^(([0-9]+(s|m|h))+)|(Never)$ + type: string + consolidationPolicy: + default: WhenUnderutilized + description: ConsolidationPolicy describes which nodes Karpenter can disrupt through its consolidation algorithm. This policy defaults to "WhenUnderutilized" if not specified + enum: + - WhenEmpty + - WhenUnderutilized + type: string + expireAfter: + default: 720h + description: ExpireAfter is the duration the controller will wait before terminating a node, measured from when the node is created. This is useful to implement features like eventually consistent node upgrade, memory leak protection, and disruption testing. + pattern: ^(([0-9]+(s|m|h))+)|(Never)$ + type: string + type: object + x-kubernetes-validations: + - message: consolidateAfter cannot be combined with consolidationPolicy=WhenUnderutilized + rule: 'has(self.consolidateAfter) ? self.consolidationPolicy != ''WhenUnderutilized'' || self.consolidateAfter == ''Never'' : true' + - message: consolidateAfter must be specified with consolidationPolicy=WhenEmpty + rule: 'self.consolidationPolicy == ''WhenEmpty'' ? has(self.consolidateAfter) : true' + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: Limits define a set of bounds for provisioning capacity. + type: object + template: + description: Template contains the template of possibilities for the provisioning logic to launch a NodeClaim with. NodeClaims launched from this NodePool will often be further constrained than the template specifies. + properties: + metadata: + properties: + annotations: + additionalProperties: + type: string + description: 'Annotations is an unstructured key value map stored with a resource that may be set by external tools to store and retrieve arbitrary metadata. They are not queryable and should be preserved when modifying objects. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations' + type: object + labels: + additionalProperties: + type: string + maxLength: 63 + pattern: ^(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])?$ + description: 'Map of string keys and values that can be used to organize and categorize (scope and select) objects. May match selectors of replication controllers and services. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/labels' + type: object + maxProperties: 100 + x-kubernetes-validations: + - message: label domain "kubernetes.io" is restricted + rule: self.all(x, x in ["beta.kubernetes.io/instance-type", "failure-domain.beta.kubernetes.io/region", "beta.kubernetes.io/os", "beta.kubernetes.io/arch", "failure-domain.beta.kubernetes.io/zone", "topology.kubernetes.io/zone", "topology.kubernetes.io/region", "kubernetes.io/arch", "kubernetes.io/os", "node.kubernetes.io/windows-build"] || x.startsWith("node.kubernetes.io") || x.startsWith("node-restriction.kubernetes.io") || !x.find("^([^/]+)").endsWith("kubernetes.io")) + - message: label domain "k8s.io" is restricted + rule: self.all(x, x.startsWith("kops.k8s.io") || !x.find("^([^/]+)").endsWith("k8s.io")) + - message: label domain "karpenter.sh" is restricted + rule: self.all(x, x in ["karpenter.sh/capacity-type", "karpenter.sh/nodepool"] || !x.find("^([^/]+)").endsWith("karpenter.sh")) + - message: label "karpenter.sh/nodepool" is restricted + rule: self.all(x, x != "karpenter.sh/nodepool") + - message: label "kubernetes.io/hostname" is restricted + rule: self.all(x, x != "kubernetes.io/hostname") + type: object + spec: + description: NodeClaimSpec describes the desired state of the NodeClaim + properties: + kubelet: + description: Kubelet defines args to be used when configuring kubelet on provisioned nodes. They are a subset of the upstream types, recognizing not all options may be supported. Wherever possible, the types and names should reflect the upstream kubelet types. + properties: + clusterDNS: + description: clusterDNS is a list of IP addresses for the cluster DNS server. Note that not all providers may use all addresses. + items: + type: string + type: array + cpuCFSQuota: + description: CPUCFSQuota enables CPU CFS quota enforcement for containers that specify CPU limits. + type: boolean + evictionHard: + additionalProperties: + type: string + pattern: ^((\d{1,2}(\.\d{1,2})?|100(\.0{1,2})?)%||(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?)$ + description: EvictionHard is the map of signal names to quantities that define hard eviction thresholds + type: object + x-kubernetes-validations: + - message: valid keys for evictionHard are ['memory.available','nodefs.available','nodefs.inodesFree','imagefs.available','imagefs.inodesFree','pid.available'] + rule: self.all(x, x in ['memory.available','nodefs.available','nodefs.inodesFree','imagefs.available','imagefs.inodesFree','pid.available']) + evictionMaxPodGracePeriod: + description: EvictionMaxPodGracePeriod is the maximum allowed grace period (in seconds) to use when terminating pods in response to soft eviction thresholds being met. + format: int32 + type: integer + evictionSoft: + additionalProperties: + type: string + pattern: ^((\d{1,2}(\.\d{1,2})?|100(\.0{1,2})?)%||(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?)$ + description: EvictionSoft is the map of signal names to quantities that define soft eviction thresholds + type: object + x-kubernetes-validations: + - message: valid keys for evictionSoft are ['memory.available','nodefs.available','nodefs.inodesFree','imagefs.available','imagefs.inodesFree','pid.available'] + rule: self.all(x, x in ['memory.available','nodefs.available','nodefs.inodesFree','imagefs.available','imagefs.inodesFree','pid.available']) + evictionSoftGracePeriod: + additionalProperties: + type: string + description: EvictionSoftGracePeriod is the map of signal names to quantities that define grace periods for each eviction signal + type: object + x-kubernetes-validations: + - message: valid keys for evictionSoftGracePeriod are ['memory.available','nodefs.available','nodefs.inodesFree','imagefs.available','imagefs.inodesFree','pid.available'] + rule: self.all(x, x in ['memory.available','nodefs.available','nodefs.inodesFree','imagefs.available','imagefs.inodesFree','pid.available']) + imageGCHighThresholdPercent: + description: ImageGCHighThresholdPercent is the percent of disk usage after which image garbage collection is always run. The percent is calculated by dividing this field value by 100, so this field must be between 0 and 100, inclusive. When specified, the value must be greater than ImageGCLowThresholdPercent. + format: int32 + maximum: 100 + minimum: 0 + type: integer + imageGCLowThresholdPercent: + description: ImageGCLowThresholdPercent is the percent of disk usage before which image garbage collection is never run. Lowest disk usage to garbage collect to. The percent is calculated by dividing this field value by 100, so the field value must be between 0 and 100, inclusive. When specified, the value must be less than imageGCHighThresholdPercent + format: int32 + maximum: 100 + minimum: 0 + type: integer + kubeReserved: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: KubeReserved contains resources reserved for Kubernetes system components. + type: object + x-kubernetes-validations: + - message: valid keys for kubeReserved are ['cpu','memory','ephemeral-storage','pid'] + rule: self.all(x, x=='cpu' || x=='memory' || x=='ephemeral-storage' || x=='pid') + - message: kubeReserved value cannot be a negative resource quantity + rule: self.all(x, !self[x].startsWith('-')) + maxPods: + description: MaxPods is an override for the maximum number of pods that can run on a worker node instance. + format: int32 + minimum: 0 + type: integer + podsPerCore: + description: PodsPerCore is an override for the number of pods that can run on a worker node instance based on the number of cpu cores. This value cannot exceed MaxPods, so, if MaxPods is a lower value, that value will be used. + format: int32 + minimum: 0 + type: integer + systemReserved: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: SystemReserved contains resources reserved for OS system daemons and kernel memory. + type: object + x-kubernetes-validations: + - message: valid keys for systemReserved are ['cpu','memory','ephemeral-storage','pid'] + rule: self.all(x, x=='cpu' || x=='memory' || x=='ephemeral-storage' || x=='pid') + - message: systemReserved value cannot be a negative resource quantity + rule: self.all(x, !self[x].startsWith('-')) + type: object + x-kubernetes-validations: + - message: imageGCHighThresholdPercent must be greater than imageGCLowThresholdPercent + rule: 'has(self.imageGCHighThresholdPercent) && has(self.imageGCLowThresholdPercent) ? self.imageGCHighThresholdPercent > self.imageGCLowThresholdPercent : true' + - message: evictionSoft OwnerKey does not have a matching evictionSoftGracePeriod + rule: has(self.evictionSoft) ? self.evictionSoft.all(e, (e in self.evictionSoftGracePeriod)):true + - message: evictionSoftGracePeriod OwnerKey does not have a matching evictionSoft + rule: has(self.evictionSoftGracePeriod) ? self.evictionSoftGracePeriod.all(e, (e in self.evictionSoft)):true + nodeClassRef: + description: NodeClassRef is a reference to an object that defines provider specific configuration + properties: + apiVersion: + description: API version of the referent + type: string + kind: + description: 'Kind of the referent; More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds"' + type: string + name: + description: 'Name of the referent; More info: http://kubernetes.io/docs/user-guide/identifiers#names' + type: string + required: + - name + type: object + requirements: + description: Requirements are layered with GetLabels and applied to every node. + items: + description: A node selector requirement is a selector that contains values, a key, and an operator that relates the key and values. + properties: + key: + description: The label key that the selector applies to. + type: string + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*(\/))?([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9]$ + x-kubernetes-validations: + - message: label domain "kubernetes.io" is restricted + rule: self in ["beta.kubernetes.io/instance-type", "failure-domain.beta.kubernetes.io/region", "beta.kubernetes.io/os", "beta.kubernetes.io/arch", "failure-domain.beta.kubernetes.io/zone", "topology.kubernetes.io/zone", "topology.kubernetes.io/region", "node.kubernetes.io/instance-type", "kubernetes.io/arch", "kubernetes.io/os", "node.kubernetes.io/windows-build"] || self.startsWith("node.kubernetes.io/") || self.startsWith("node-restriction.kubernetes.io/") || !self.find("^([^/]+)").endsWith("kubernetes.io") + - message: label domain "k8s.io" is restricted + rule: self.startsWith("kops.k8s.io/") || !self.find("^([^/]+)").endsWith("k8s.io") + - message: label domain "karpenter.sh" is restricted + rule: self in ["karpenter.sh/capacity-type", "karpenter.sh/nodepool"] || !self.find("^([^/]+)").endsWith("karpenter.sh") + - message: label "karpenter.sh/nodepool" is restricted + rule: self != "karpenter.sh/nodepool" + - message: label "kubernetes.io/hostname" is restricted + rule: self != "kubernetes.io/hostname" + operator: + description: Represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. + type: string + enum: + - In + - NotIn + - Exists + - DoesNotExist + - Gt + - Lt + values: + description: An array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. If the operator is Gt or Lt, the values array must have a single element, which will be interpreted as an integer. This array is replaced during a strategic merge patch. + items: + type: string + type: array + maxLength: 63 + pattern: ^(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])?$ + required: + - key + - operator + type: object + maxItems: 30 + type: array + x-kubernetes-validations: + - message: requirements with operator 'In' must have a value defined + rule: 'self.all(x, x.operator == ''In'' ? x.values.size() != 0 : true)' + - message: requirements operator 'Gt' or 'Lt' must have a single positive integer value + rule: 'self.all(x, (x.operator == ''Gt'' || x.operator == ''Lt'') ? (x.values.size() == 1 && int(x.values[0]) >= 0) : true)' + resources: + description: Resources models the resource requirements for the NodeClaim to launch + properties: + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: Requests describes the minimum required resources for the NodeClaim to launch + type: object + type: object + startupTaints: + description: StartupTaints are taints that are applied to nodes upon startup which are expected to be removed automatically within a short period of time, typically by a DaemonSet that tolerates the taint. These are commonly used by daemonsets to allow initialization and enforce startup ordering. StartupTaints are ignored for provisioning purposes in that pods are not required to tolerate a StartupTaint in order to have nodes provisioned for them. + items: + description: The node this Taint is attached to has the "effect" on any pod that does not tolerate the Taint. + properties: + effect: + description: Required. The effect of the taint on pods that do not tolerate the taint. Valid effects are NoSchedule, PreferNoSchedule and NoExecute. + type: string + enum: + - NoSchedule + - PreferNoSchedule + - NoExecute + key: + description: Required. The taint key to be applied to a node. + type: string + minLength: 1 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*(\/))?([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9]$ + timeAdded: + description: TimeAdded represents the time at which the taint was added. It is only written for NoExecute taints. + format: date-time + type: string + value: + description: The taint value corresponding to the taint key. + type: string + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*(\/))?([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9]$ + required: + - effect + - key + type: object + type: array + taints: + description: Taints will be applied to the NodeClaim's node. + items: + description: The node this Taint is attached to has the "effect" on any pod that does not tolerate the Taint. + properties: + effect: + description: Required. The effect of the taint on pods that do not tolerate the taint. Valid effects are NoSchedule, PreferNoSchedule and NoExecute. + type: string + enum: + - NoSchedule + - PreferNoSchedule + - NoExecute + key: + description: Required. The taint key to be applied to a node. + type: string + minLength: 1 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*(\/))?([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9]$ + timeAdded: + description: TimeAdded represents the time at which the taint was added. It is only written for NoExecute taints. + format: date-time + type: string + value: + description: The taint value corresponding to the taint key. + type: string + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*(\/))?([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9]$ + required: + - effect + - key + type: object + type: array + required: + - nodeClassRef + - requirements + type: object + type: object + weight: + description: Weight is the priority given to the provisioner during scheduling. A higher numerical weight indicates that this provisioner will be ordered ahead of other provisioners with lower weights. A provisioner with no weight will be treated as if it is a provisioner with a weight of 0. + format: int32 + maximum: 100 + minimum: 1 + type: integer + type: object + status: + description: NodePoolStatus defines the observed state of NodePool + properties: + resources: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: Resources is the list of resources that have been provisioned. + type: object + type: object + type: object + served: true + storage: true + subresources: + status: {} diff --git a/pkg/apis/crds/karpenter.sh_provisioners.yaml b/pkg/apis/crds/karpenter.sh_provisioners.yaml new file mode 100644 index 000000000..192412474 --- /dev/null +++ b/pkg/apis/crds/karpenter.sh_provisioners.yaml @@ -0,0 +1,381 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.13.0 + name: provisioners.karpenter.sh +spec: + group: karpenter.sh + names: + categories: + - karpenter + kind: Provisioner + listKind: ProvisionerList + plural: provisioners + singular: provisioner + scope: Cluster + versions: + - additionalPrinterColumns: + - jsonPath: .spec.providerRef.name + name: Template + type: string + - jsonPath: .spec.weight + name: Weight + priority: 1 + type: string + name: v1alpha5 + schema: + openAPIV3Schema: + description: Provisioner is the Schema for the Provisioners API + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: ProvisionerSpec is the top level provisioner specification. + Provisioners launch nodes in response to pods that are unschedulable. + A single provisioner is capable of managing a diverse set of nodes. + Node properties are determined from a combination of provisioner and + pod scheduling constraints. + properties: + annotations: + additionalProperties: + type: string + description: Annotations are applied to every node. + type: object + consolidation: + description: Consolidation are the consolidation parameters + properties: + enabled: + description: Enabled enables consolidation if it has been set + type: boolean + type: object + kubeletConfiguration: + description: KubeletConfiguration are options passed to the kubelet + when provisioning nodes + properties: + clusterDNS: + description: clusterDNS is a list of IP addresses for the cluster + DNS server. Note that not all providers may use all addresses. + items: + type: string + type: array + containerRuntime: + description: ContainerRuntime is the container runtime to be used + with your worker nodes. + type: string + cpuCFSQuota: + description: CPUCFSQuota enables CPU CFS quota enforcement for + containers that specify CPU limits. + type: boolean + evictionHard: + additionalProperties: + type: string + description: EvictionHard is the map of signal names to quantities + that define hard eviction thresholds + type: object + evictionMaxPodGracePeriod: + description: EvictionMaxPodGracePeriod is the maximum allowed + grace period (in seconds) to use when terminating pods in response + to soft eviction thresholds being met. + format: int32 + type: integer + evictionSoft: + additionalProperties: + type: string + description: EvictionSoft is the map of signal names to quantities + that define soft eviction thresholds + type: object + evictionSoftGracePeriod: + additionalProperties: + type: string + description: EvictionSoftGracePeriod is the map of signal names + to quantities that define grace periods for each eviction signal + type: object + imageGCHighThresholdPercent: + description: ImageGCHighThresholdPercent is the percent of disk + usage after which image garbage collection is always run. The + percent is calculated by dividing this field value by 100, so + this field must be between 0 and 100, inclusive. When specified, + the value must be greater than ImageGCLowThresholdPercent. + format: int32 + maximum: 100 + minimum: 0 + type: integer + imageGCLowThresholdPercent: + description: ImageGCLowThresholdPercent is the percent of disk + usage before which image garbage collection is never run. Lowest + disk usage to garbage collect to. The percent is calculated + by dividing this field value by 100, so the field value must + be between 0 and 100, inclusive. When specified, the value must + be less than imageGCHighThresholdPercent + format: int32 + maximum: 100 + minimum: 0 + type: integer + kubeReserved: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: KubeReserved contains resources reserved for Kubernetes + system components. + type: object + maxPods: + description: MaxPods is an override for the maximum number of + pods that can run on a worker node instance. + format: int32 + minimum: 0 + type: integer + podsPerCore: + description: PodsPerCore is an override for the number of pods + that can run on a worker node instance based on the number of + cpu cores. This value cannot exceed MaxPods, so, if MaxPods + is a lower value, that value will be used. + format: int32 + minimum: 0 + type: integer + systemReserved: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: SystemReserved contains resources reserved for OS + system daemons and kernel memory. + type: object + type: object + labels: + additionalProperties: + type: string + description: Labels are layered with Requirements and applied to every + node. + type: object + limits: + description: Limits define a set of bounds for provisioning capacity. + properties: + resources: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: Resources contains all the allocatable resources + that Karpenter supports for limiting. + type: object + type: object + provider: + description: Provider contains fields specific to your cloudprovider. + type: object + x-kubernetes-preserve-unknown-fields: true + providerRef: + description: ProviderRef is a reference to a dedicated CRD for the + chosen provider, that holds additional configuration options + properties: + apiVersion: + description: API version of the referent + type: string + kind: + description: 'Kind of the referent; More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds"' + type: string + name: + description: 'Name of the referent; More info: http://kubernetes.io/docs/user-guide/identifiers#names' + type: string + required: + - name + type: object + requirements: + description: Requirements are layered with Labels and applied to every + node. + items: + description: A node selector requirement is a selector that contains + values, a key, and an operator that relates the key and values. + properties: + key: + description: The label key that the selector applies to. + type: string + operator: + description: Represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and + Lt. + type: string + values: + description: An array of string values. If the operator is In + or NotIn, the values array must be non-empty. If the operator + is Exists or DoesNotExist, the values array must be empty. + If the operator is Gt or Lt, the values array must have a + single element, which will be interpreted as an integer. This + array is replaced during a strategic merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + startupTaints: + description: StartupTaints are taints that are applied to nodes upon + startup which are expected to be removed automatically within a + short period of time, typically by a DaemonSet that tolerates the + taint. These are commonly used by daemonsets to allow initialization + and enforce startup ordering. StartupTaints are ignored for provisioning + purposes in that pods are not required to tolerate a StartupTaint + in order to have nodes provisioned for them. + items: + description: The node this Taint is attached to has the "effect" + on any pod that does not tolerate the Taint. + properties: + effect: + description: Required. The effect of the taint on pods that + do not tolerate the taint. Valid effects are NoSchedule, PreferNoSchedule + and NoExecute. + type: string + key: + description: Required. The taint key to be applied to a node. + type: string + timeAdded: + description: TimeAdded represents the time at which the taint + was added. It is only written for NoExecute taints. + format: date-time + type: string + value: + description: The taint value corresponding to the taint key. + type: string + required: + - effect + - key + type: object + type: array + taints: + description: Taints will be applied to every node launched by the + Provisioner. If specified, the provisioner will not provision nodes + for pods that do not have matching tolerations. Additional taints + will be created that match pod tolerations on a per-node basis. + items: + description: The node this Taint is attached to has the "effect" + on any pod that does not tolerate the Taint. + properties: + effect: + description: Required. The effect of the taint on pods that + do not tolerate the taint. Valid effects are NoSchedule, PreferNoSchedule + and NoExecute. + type: string + key: + description: Required. The taint key to be applied to a node. + type: string + timeAdded: + description: TimeAdded represents the time at which the taint + was added. It is only written for NoExecute taints. + format: date-time + type: string + value: + description: The taint value corresponding to the taint key. + type: string + required: + - effect + - key + type: object + type: array + ttlSecondsAfterEmpty: + description: "TTLSecondsAfterEmpty is the number of seconds the controller + will wait before attempting to delete a node, measured from when + the node is detected to be empty. A Node is considered to be empty + when it does not have pods scheduled to it, excluding daemonsets. + \n Termination due to no utilization is disabled if this field is + not set." + format: int64 + type: integer + ttlSecondsUntilExpired: + description: "TTLSecondsUntilExpired is the number of seconds the + controller will wait before terminating a node, measured from when + the node is created. This is useful to implement features like eventually + consistent node upgrade, memory leak protection, and disruption + testing. \n Termination due to expiration is disabled if this field + is not set." + format: int64 + type: integer + weight: + description: Weight is the priority given to the provisioner during + scheduling. A higher numerical weight indicates that this provisioner + will be ordered ahead of other provisioners with lower weights. + A provisioner with no weight will be treated as if it is a provisioner + with a weight of 0. + format: int32 + maximum: 100 + minimum: 1 + type: integer + type: object + status: + description: ProvisionerStatus defines the observed state of Provisioner + properties: + conditions: + description: Conditions is the set of conditions required for this + provisioner to scale its target, and indicates whether or not those + conditions are met. + items: + description: 'Condition defines a readiness condition for a Knative + resource. See: https://github.com/kubernetes/community/blob/master/contributors/devel/sig-architecture/api-conventions.md#typical-status-properties' + properties: + lastTransitionTime: + description: LastTransitionTime is the last time the condition + transitioned from one status to another. We use VolatileTime + in place of metav1.Time to exclude this from creating equality.Semantic + differences (all other things held constant). + type: string + message: + description: A human readable message indicating details about + the transition. + type: string + reason: + description: The reason for the condition's last transition. + type: string + severity: + description: Severity with which to treat failures of this type + of condition. When this is not specified, it defaults to Error. + type: string + status: + description: Status of the condition, one of True, False, Unknown. + type: string + type: + description: Type of condition. + type: string + required: + - status + - type + type: object + type: array + lastScaleTime: + description: LastScaleTime is the last time the Provisioner scaled + the number of nodes + format: date-time + type: string + resources: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: Resources is the list of resources that have been provisioned. + type: object + type: object + type: object + served: true + storage: true + subresources: + status: {} diff --git a/pkg/apis/settings/settings.go b/pkg/apis/settings/settings.go new file mode 100644 index 000000000..a7cbaa4c5 --- /dev/null +++ b/pkg/apis/settings/settings.go @@ -0,0 +1,195 @@ +/* +Portions Copyright (c) Microsoft Corporation. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package settings + +import ( + "context" + "encoding/json" + "fmt" + "hash/fnv" + "math/rand" + "net/url" + + "github.com/go-playground/validator/v10" + "go.uber.org/multierr" + v1 "k8s.io/api/core/v1" + "knative.dev/pkg/configmap" +) + +type settingsKeyType struct{} + +var ContextKey = settingsKeyType{} + +var defaultSettings = Settings{ + ClusterName: "", + ClusterEndpoint: "", + VMMemoryOverheadPercent: 0.075, + Tags: map[string]string{}, + ClusterID: "", + KubeletClientTLSBootstrapToken: "", + SSHPublicKey: "", + NetworkPlugin: "", + NetworkPolicy: "", +} + +// +k8s:deepcopy-gen=true +type Settings struct { + ClusterName string `validate:"required"` + ClusterEndpoint string `validate:"required"` // => APIServerName in bootstrap, except needs to be w/o https/port + VMMemoryOverheadPercent float64 `validate:"min=0"` + Tags map[string]string + + // Cluster-level settings required for nodebootstrap (category "x") + // (Candidates for exposure/accessibility via API) + // TODO: consider making these AKS-specific (e.g. subkey?) + + ClusterID string + + KubeletClientTLSBootstrapToken string `validate:"required"` // => TLSBootstrapToken in bootstrap (may need to be per node/nodepool) + SSHPublicKey string // ssh.publicKeys.keyData => VM SSH public key // TODO: move to node template? + NetworkPlugin string `validate:"required"` // => NetworkPlugin in bootstrap + NetworkPolicy string // => NetworkPolicy in bootstrap + NodeIdentities []string // => Applied onto each VM +} + +func (*Settings) ConfigMap() string { + return "karpenter-global-settings" +} + +// Inject creates a Settings from the supplied ConfigMap +func (*Settings) Inject(ctx context.Context, cm *v1.ConfigMap) (context.Context, error) { + s := defaultSettings.DeepCopy() + + if err := configmap.Parse(cm.Data, + configmap.AsString("azure.clusterName", &s.ClusterName), + configmap.AsString("azure.clusterEndpoint", &s.ClusterEndpoint), + configmap.AsFloat64("azure.vmMemoryOverheadPercent", &s.VMMemoryOverheadPercent), + AsStringMap("azure.tags", &s.Tags), + configmap.AsString("azure.clusterID", &s.ClusterID), + configmap.AsString("azure.kubeletClientTLSBootstrapToken", &s.KubeletClientTLSBootstrapToken), + configmap.AsString("azure.sshPublicKey", &s.SSHPublicKey), + configmap.AsString("azure.networkPlugin", &s.NetworkPlugin), + configmap.AsString("azure.networkPolicy", &s.NetworkPolicy), + AsStringSlice("azure.nodeIdentities", &s.NodeIdentities), + ); err != nil { + return ctx, fmt.Errorf("parsing settings, %w", err) + } + if err := s.Validate(); err != nil { + return ctx, fmt.Errorf("validating settings, %w", err) + } + + // if clusterID is not set, generate it from cluster endpoint + if s.ClusterID == "" { + s.ClusterID = getAKSClusterID(s.GetAPIServerName()) + } + + return ToContext(ctx, s), nil +} + +func (s Settings) Data() (map[string]string, error) { + d := map[string]string{} + + raw, err := json.Marshal(s) + if err != nil { + return nil, fmt.Errorf("marshaling settings, %w", err) + } + if err = json.Unmarshal(raw, &d); err != nil { + return d, fmt.Errorf("unmarshalling settings, %w", err) + } + return d, nil +} + +// Validate leverages struct tags with go-playground/validator so you can define a struct with custom +// validation on fields i.e. +// +// type ExampleStruct struct { +// Example metav1.Duration `json:"example" validate:"required,min=10m"` +// } +func (s Settings) Validate() error { + validate := validator.New() + return multierr.Combine( + s.validateEndpoint(), + validate.Struct(s), + ) +} + +func (s Settings) validateEndpoint() error { + endpoint, err := url.Parse(s.ClusterEndpoint) + // url.Parse() will accept a lot of input without error; make + // sure it's a real URL + if err != nil || !endpoint.IsAbs() || endpoint.Hostname() == "" { + return fmt.Errorf("\"%s\" not a valid clusterEndpoint URL", s.ClusterEndpoint) + } + return nil +} + +func (s Settings) GetAPIServerName() string { + endpoint, _ := url.Parse(s.ClusterEndpoint) // already validated + return endpoint.Hostname() +} + +func ToContext(ctx context.Context, s *Settings) context.Context { + return context.WithValue(ctx, ContextKey, s) +} + +func FromContext(ctx context.Context) *Settings { + data := ctx.Value(ContextKey) + if data == nil { + // This is developer error if this happens, so we should panic + panic("settings doesn't exist in context") + } + return data.(*Settings) +} + +// AsStringMap parses a value as a JSON map of map[string]string. +func AsStringMap(key string, target *map[string]string) configmap.ParseFunc { + return func(data map[string]string) error { + if raw, ok := data[key]; ok { + m := map[string]string{} + if err := json.Unmarshal([]byte(raw), &m); err != nil { + return err + } + *target = m + } + return nil + } +} + +// AsStringMap parses a value as a JSON map of map[string]string. +func AsStringSlice(key string, target *[]string) configmap.ParseFunc { + return func(data map[string]string) error { + if raw, ok := data[key]; ok { + var s []string + if err := json.Unmarshal([]byte(raw), &s); err != nil { + return err + } + *target = s + } + return nil + } +} + +// getAKSClusterID returns cluster ID based on the DNS prefix of the cluster. +// The logic comes from AgentBaker and other places, originally from aks-engine +// with the additional assumption of DNS prefix being the first 33 chars of FQDN +func getAKSClusterID(apiServerFQDN string) string { + dnsPrefix := apiServerFQDN[:33] + h := fnv.New64a() + h.Write([]byte(dnsPrefix)) + r := rand.New(rand.NewSource(int64(h.Sum64()))) //nolint:gosec + return fmt.Sprintf("%08d", r.Uint32())[:8] +} diff --git a/pkg/apis/settings/suite_test.go b/pkg/apis/settings/suite_test.go new file mode 100644 index 000000000..94aa48e27 --- /dev/null +++ b/pkg/apis/settings/suite_test.go @@ -0,0 +1,125 @@ +/* +Portions Copyright (c) Microsoft Corporation. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package settings_test + +import ( + "context" + "testing" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + v1 "k8s.io/api/core/v1" + . "knative.dev/pkg/logging/testing" + + "github.com/Azure/karpenter/pkg/apis/settings" +) + +var ctx context.Context + +func TestAPIs(t *testing.T) { + ctx = TestContextWithLogger(t) + RegisterFailHandler(Fail) + RunSpecs(t, "Settings") +} + +var _ = Describe("Validation", func() { + It("should succeed to set defaults", func() { + cm := &v1.ConfigMap{ + Data: map[string]string{ + "azure.clusterEndpoint": "https://karpenter-000000000000.hcp.westus2.staging.azmk8s.io", + "azure.clusterName": "my-cluster", + "azure.clusterID": "my-cluster-id", + "azure.kubeletClientTLSBootstrapToken": "my-bootstrap-token", + "azure.sshPublicKey": "my-ssh-public-key", + "azure.networkPlugin": "kubenet", + "azure.networkPolicy": "azure", + }, + } + ctx, err := (&settings.Settings{}).Inject(ctx, cm) + Expect(err).ToNot(HaveOccurred()) + s := settings.FromContext(ctx) + Expect(s.VMMemoryOverheadPercent).To(Equal(0.075)) + Expect(len(s.Tags)).To(BeZero()) + }) + It("should succeed to set custom values", func() { + cm := &v1.ConfigMap{ + Data: map[string]string{ + "azure.clusterEndpoint": "https://karpenter-000000000000.hcp.westus2.staging.azmk8s.io", + "azure.clusterName": "my-cluster", + "azure.vmMemoryOverheadPercent": "0.1", + "azure.tags": `{"tag1": "value1", "tag2": "value2", "example.com/tag": "my-value"}`, + "azure.clusterID": "my-cluster-id", + "azure.kubeletClientTLSBootstrapToken": "my-bootstrap-token", + "azure.sshPublicKey": "my-ssh-public-key", + "azure.networkPlugin": "kubenet", + "azure.networkPolicy": "azure", + "azure.nodeIdentities": "[\"/subscriptions/1234/resourceGroups/mcrg/providers/Microsoft.ManagedIdentity/userAssignedIdentities/myid1\",\"/subscriptions/1234/resourceGroups/mcrg/providers/Microsoft.ManagedIdentity/userAssignedIdentities/myid2\"]", + }, + } + ctx, err := (&settings.Settings{}).Inject(ctx, cm) + Expect(err).ToNot(HaveOccurred()) + s := settings.FromContext(ctx) + Expect(s.VMMemoryOverheadPercent).To(Equal(0.1)) + Expect(len(s.Tags)).To(Equal(3)) + Expect(s.Tags).To(HaveKeyWithValue("tag1", "value1")) + Expect(s.Tags).To(HaveKeyWithValue("tag2", "value2")) + Expect(s.Tags).To(HaveKeyWithValue("example.com/tag", "my-value")) + Expect(s.NodeIdentities).To(HaveLen(2)) + Expect(s.NodeIdentities[0]).To(Equal("/subscriptions/1234/resourceGroups/mcrg/providers/Microsoft.ManagedIdentity/userAssignedIdentities/myid1")) + Expect(s.NodeIdentities[1]).To(Equal("/subscriptions/1234/resourceGroups/mcrg/providers/Microsoft.ManagedIdentity/userAssignedIdentities/myid2")) + }) + It("should fail validation with panic when clusterName not included", func() { + cm := &v1.ConfigMap{ + Data: map[string]string{ + "azure.clusterEndpoint": "https://karpenter-000000000000.hcp.westus2.staging.azmk8s.io", + }, + } + _, err := (&settings.Settings{}).Inject(ctx, cm) + Expect(err).To(HaveOccurred()) + }) + It("should fail validation with panic when clusterEndpoint not included", func() { + cm := &v1.ConfigMap{ + Data: map[string]string{ + "azure.clusterName": "my-name", + }, + } + _, err := (&settings.Settings{}).Inject(ctx, cm) + Expect(err).To(HaveOccurred()) + }) + It("should fail validation with panic when clusterEndpoint is invalid (not absolute)", func() { + cm := &v1.ConfigMap{ + Data: map[string]string{ + "azure.clusterName": "my-name", + "azure.clusterEndpoint": "karpenter-000000000000.hcp.westus2.staging.azmk8s.io", + }, + } + _, err := (&settings.Settings{}).Inject(ctx, cm) + Expect(err).To(HaveOccurred()) + }) + It("should fail validation with panic when vmMemoryOverheadPercent is negative", func() { + cm := &v1.ConfigMap{ + Data: map[string]string{ + "azure.clusterEndpoint": "https://karpenter-000000000000.hcp.westus2.staging.azmk8s.io", + "azure.clusterName": "my-cluster", + "azure.vmMemoryOverheadPercent": "-0.01", + }, + } + _, err := (&settings.Settings{}).Inject(ctx, cm) + Expect(err).To(HaveOccurred()) + }) + // TODO: more validation tests +}) diff --git a/pkg/apis/settings/zz_generated.deepcopy.go b/pkg/apis/settings/zz_generated.deepcopy.go new file mode 100644 index 000000000..2aff41bba --- /dev/null +++ b/pkg/apis/settings/zz_generated.deepcopy.go @@ -0,0 +1,48 @@ +/* +Portions Copyright (c) Microsoft Corporation. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Code generated by controller-gen. DO NOT EDIT. + +package settings + +import () + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *Settings) DeepCopyInto(out *Settings) { + *out = *in + if in.Tags != nil { + in, out := &in.Tags, &out.Tags + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } + if in.NodeIdentities != nil { + in, out := &in.NodeIdentities, &out.NodeIdentities + *out = make([]string, len(*in)) + copy(*out, *in) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Settings. +func (in *Settings) DeepCopy() *Settings { + if in == nil { + return nil + } + out := new(Settings) + in.DeepCopyInto(out) + return out +} diff --git a/pkg/apis/v1alpha2/aksnodeclass.go b/pkg/apis/v1alpha2/aksnodeclass.go new file mode 100644 index 000000000..d251c8a5b --- /dev/null +++ b/pkg/apis/v1alpha2/aksnodeclass.go @@ -0,0 +1,75 @@ +/* +Portions Copyright (c) Microsoft Corporation. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1alpha2 + +import ( + "fmt" + + "github.com/mitchellh/hashstructure/v2" + "github.com/samber/lo" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +// AKSNodeClassSpec is the top level specification for the AKS Karpenter Provider. +// This will contain configuration necessary to launch instances in AKS. +type AKSNodeClassSpec struct { + // +kubebuilder:default=128 + // +kubebuilder:validation:Minimum=100 + // osDiskSizeGB is the size of the OS disk in GB. + OSDiskSizeGB *int32 `json:"osDiskSizeGB,omitempty"` + // ImageID is the ID of the image that instances use. + // Not exposed in the API yet + ImageID *string `json:"-"` + // ImageFamily is the image family that instances use. + // +kubebuilder:default=Ubuntu2204 + // +kubebuilder:validation:Enum:={Ubuntu2204} + ImageFamily *string `json:"imageFamily,omitempty"` + // ImageVersion is the image version that instances use. + // +optional + ImageVersion *string `json:"imageVersion,omitempty"` + // Tags to be applied on Azure resources like instances. + // +optional + Tags map[string]string `json:"tags,omitempty"` +} + +// AKSNodeClass is the Schema for the AKSNodeClass API +// +kubebuilder:object:root=true +// +kubebuilder:resource:path=aksnodeclasses,scope=Cluster,categories=karpenter,shortName={aksnc,aksncs} +// +kubebuilder:subresource:status +type AKSNodeClass struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata,omitempty"` + + Spec AKSNodeClassSpec `json:"spec,omitempty"` + Status AKSNodeClassStatus `json:"status,omitempty"` +} + +func (in *AKSNodeClass) Hash() string { + return fmt.Sprint(lo.Must(hashstructure.Hash(in.Spec, hashstructure.FormatV2, &hashstructure.HashOptions{ + SlicesAsSets: true, + IgnoreZeroValue: true, + ZeroNil: true, + }))) +} + +// AKSNodeClassList contains a list of AKSNodeClass +// +kubebuilder:object:root=true +type AKSNodeClassList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata,omitempty"` + Items []AKSNodeClass `json:"items"` +} diff --git a/pkg/apis/v1alpha2/aksnodeclass_defaults.go b/pkg/apis/v1alpha2/aksnodeclass_defaults.go new file mode 100644 index 000000000..d11dd173c --- /dev/null +++ b/pkg/apis/v1alpha2/aksnodeclass_defaults.go @@ -0,0 +1,25 @@ +/* +Portions Copyright (c) Microsoft Corporation. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1alpha2 + +import ( + "context" +) + +// SetDefaults for the AKSNodeClass +func (in *AKSNodeClass) SetDefaults(_ context.Context) { +} diff --git a/pkg/apis/v1alpha2/aksnodeclass_status.go b/pkg/apis/v1alpha2/aksnodeclass_status.go new file mode 100644 index 000000000..9d93d5179 --- /dev/null +++ b/pkg/apis/v1alpha2/aksnodeclass_status.go @@ -0,0 +1,33 @@ +/* +Portions Copyright (c) Microsoft Corporation. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1alpha2 + +import v1 "k8s.io/api/core/v1" + +// Image contains resolved image selector values utilized for node launch +type Image struct { + // ID of the image + // +required + ID string `json:"id"` + // Requirements of the image to be utilized on an instance type + // +required + Requirements []v1.NodeSelectorRequirement `json:"requirements"` +} + +// AKSNodeClassStatus contains the resolved state of the AKSNodeClass +type AKSNodeClassStatus struct { +} diff --git a/pkg/apis/v1alpha2/aksnodeclass_util.go b/pkg/apis/v1alpha2/aksnodeclass_util.go new file mode 100644 index 000000000..7018a908a --- /dev/null +++ b/pkg/apis/v1alpha2/aksnodeclass_util.go @@ -0,0 +1,28 @@ +/* +Portions Copyright (c) Microsoft Corporation. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1alpha2 + +func (in *AKSNodeClassSpec) IsEmptyImageID() bool { + return in.ImageID == nil || *in.ImageID == "" +} + +func (in *AKSNodeClassSpec) GetImageVersion() string { + if in.ImageVersion == nil { + return "" + } + return *in.ImageVersion +} diff --git a/pkg/apis/v1alpha2/aksnodeclass_validation.go b/pkg/apis/v1alpha2/aksnodeclass_validation.go new file mode 100644 index 000000000..51c1f70d4 --- /dev/null +++ b/pkg/apis/v1alpha2/aksnodeclass_validation.go @@ -0,0 +1,69 @@ +/* +Portions Copyright (c) Microsoft Corporation. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1alpha2 + +import ( + "context" + "fmt" + "regexp" + + admissionregistrationv1 "k8s.io/api/admissionregistration/v1" + + "knative.dev/pkg/apis" +) + +var ( + SubscriptionShape = regexp.MustCompile(`[a-fA-F0-9]{8}-[a-fA-F0-9]{4}-[a-fA-F0-9]{4}-[a-fA-F0-9]{4}-[a-fA-F0-9]{12}`) + ComputeGalleryImageVersionRegex = regexp.MustCompile(`(?i)/subscriptions/` + SubscriptionShape.String() + `/resourceGroups/[\w-]+/providers/Microsoft\.Compute/galleries/[\w-]+/images/[\w-]+/versions/[\d.]+`) + CommunityImageVersionRegex = regexp.MustCompile(`(?i)/CommunityGalleries/[\w-]+/images/[\w-]+/versions/[\d.]+`) +) + +func (in *AKSNodeClass) SupportedVerbs() []admissionregistrationv1.OperationType { + return []admissionregistrationv1.OperationType{ + admissionregistrationv1.Create, + admissionregistrationv1.Update, + } +} + +func IsComputeGalleryImageID(imageID string) bool { + return ComputeGalleryImageVersionRegex.MatchString(imageID) +} + +func (in *AKSNodeClass) Validate(ctx context.Context) (errs *apis.FieldError) { + //if apis.IsInUpdate(ctx) { + // original := apis.GetBaseline(ctx).(*AKSNodeClass) + // errs = in.validateImmutableFields(original) + //} + return errs.Also( + apis.ValidateObjectMetadata(in).ViaField("metadata"), + in.Spec.validate(ctx).ViaField("spec"), + ) +} + +func (in *AKSNodeClassSpec) validate(_ context.Context) (errs *apis.FieldError) { + return errs.Also( + in.validateImageID(), + ) +} + +func (in *AKSNodeClassSpec) validateImageID() (errs *apis.FieldError) { + if in.IsEmptyImageID() || ComputeGalleryImageVersionRegex.MatchString(*in.ImageID) || CommunityImageVersionRegex.MatchString(*in.ImageID) { + return nil + } + return apis.ErrInvalidValue(fmt.Sprintf( + "the provided image ID: '%s' is invalid because it doesn't match the expected format", *in.ImageID), "ImageID") +} diff --git a/pkg/apis/v1alpha2/annotations.go b/pkg/apis/v1alpha2/annotations.go new file mode 100644 index 000000000..28916fafe --- /dev/null +++ b/pkg/apis/v1alpha2/annotations.go @@ -0,0 +1,22 @@ +/* +Portions Copyright (c) Microsoft Corporation. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1alpha2 + +// Annotations +var ( + AnnotationInPlaceUpdateHash = Group + "/in-place-update-hash" +) diff --git a/pkg/apis/v1alpha2/doc.go b/pkg/apis/v1alpha2/doc.go new file mode 100644 index 000000000..21c67ba76 --- /dev/null +++ b/pkg/apis/v1alpha2/doc.go @@ -0,0 +1,23 @@ +/* +Portions Copyright (c) Microsoft Corporation. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// +k8s:openapi-gen=true +// +k8s:deepcopy-gen=package,register +// +k8s:defaulter-gen=TypeMeta +// +groupName=karpenter.azure.com +package v1alpha2 // doc.go is discovered by codegen + +// TODO: tests diff --git a/pkg/apis/v1alpha2/labels.go b/pkg/apis/v1alpha2/labels.go new file mode 100644 index 000000000..7ce9c72f3 --- /dev/null +++ b/pkg/apis/v1alpha2/labels.go @@ -0,0 +1,109 @@ +/* +Portions Copyright (c) Microsoft Corporation. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1alpha2 + +import ( + corev1beta1 "github.com/aws/karpenter-core/pkg/apis/v1beta1" +) + +func init() { + corev1beta1.RestrictedLabelDomains = corev1beta1.RestrictedLabelDomains.Insert(RestrictedLabelDomains...) + corev1beta1.WellKnownLabels = corev1beta1.WellKnownLabels.Insert( + LabelSKUName, + LabelSKUFamily, + LabelSKUVersion, + + LabelSKUCPU, + LabelSKUMemory, + LabelSKUAccelerator, + + LabelSKUConfidential, + LabelSKUIsolatedSize, + + LabelSKUHyperVGeneration, + LabelSKUAcceleratedNetworking, + + LabelSKUStoragePremiumCapable, + LabelSKUStorageCacheSize, + LabelSKUStorageTempMaxSize, + LabelSKUStorageEphemeralOSMaxSize, + + LabelSKUEncryptionAtHostSupported, + + LabelSKUGPUName, + LabelSKUGPUManufacturer, + LabelSKUGPUCount, + + AKSLabelCluster, + ) +} + +var ( + AzureToKubeArchitectures = map[string]string{ + // TODO: consider using constants like compute.ArchitectureArm64 + "x64": corev1beta1.ArchitectureAmd64, + "Arm64": corev1beta1.ArchitectureArm64, + } + RestrictedLabelDomains = []string{ + Group, + } + + // alternative zone label for Machine (the standard one is protected for AKS nodes) + AlternativeLabelTopologyZone = Group + "/zone" + + HyperVGenerationV1 = "1" + HyperVGenerationV2 = "2" + ManufacturerNvidia = "nvidia" + + LabelSKUName = Group + "/sku-name" // Standard_A1_v2 + LabelSKUFamily = Group + "/sku-family" // A + LabelSKUVersion = Group + "/sku-version" // numerical (without v), with 1 backfilled + + LabelSKUCPU = Group + "/sku-cpu" // sku.vCPUs + LabelSKUMemory = Group + "/sku-memory" // sku.MemoryGB + LabelSKUAccelerator = Group + "/sku-accelerator" + + // selected capabilities (from additive features in VM size name, or from SKU capabilities) + // https://learn.microsoft.com/en-us/azure/virtual-machines/vm-naming-conventions + LabelSKUConfidential = Group + "/sku-confidential" // c + LabelSKUIsolatedSize = Group + "/sku-isolated-size" // i + + LabelSKUHyperVGeneration = Group + "/sku-hyperv-generation" // sku.HyperVGenerations + LabelSKUAcceleratedNetworking = Group + "/sku-networking-accelerated" // sku.AcceleratedNetworkingEnabled + + LabelSKUStoragePremiumCapable = Group + "/sku-storage-premium-capable" // sku.IsPremiumIO + LabelSKUStorageCacheSize = Group + "/sku-storage-cache-size" // sku.CachedDiskBytes + LabelSKUStorageTempMaxSize = Group + "/sku-storage-temp-maxsize" // sku.MaxResourceVolumeMB + LabelSKUStorageEphemeralOSMaxSize = Group + "/sku-storage-ephemeralos-maxsize" // calculated as max(sku.CachedDiskBytes, sku.MaxResourceVolumeMB) + + LabelSKUEncryptionAtHostSupported = Group + "/sku-encryptionathost-capable" // sku.EncryptionAtHostSupported + + // GPU labels + LabelSKUGPUName = Group + "/sku-gpu-name" // ie GPU Accelerator type we parse from vmSize + LabelSKUGPUManufacturer = Group + "/sku-gpu-manufacturer" // ie NVIDIA, AMD, etc + LabelSKUGPUCount = Group + "/sku-gpu-count" // ie 16, 32, etc + + // AKS labels + AKSLabelDomain = "kubernetes.azure.com" + + AKSLabelCluster = AKSLabelDomain + "/cluster" + + SkuFeatureToLabel = map[rune]string{ + 'c': LabelSKUConfidential, + 'i': LabelSKUIsolatedSize, + } +) diff --git a/pkg/apis/v1alpha2/register.go b/pkg/apis/v1alpha2/register.go new file mode 100644 index 000000000..c13d66350 --- /dev/null +++ b/pkg/apis/v1alpha2/register.go @@ -0,0 +1,37 @@ +/* +Portions Copyright (c) Microsoft Corporation. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1alpha2 + +import ( + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" +) + +const Group = "karpenter.azure.com" + +var ( + SchemeGroupVersion = schema.GroupVersion{Group: Group, Version: "v1alpha2"} + SchemeBuilder = runtime.NewSchemeBuilder(func(scheme *runtime.Scheme) error { + scheme.AddKnownTypes(SchemeGroupVersion, + &AKSNodeClass{}, + &AKSNodeClassList{}, + ) + metav1.AddToGroupVersion(scheme, SchemeGroupVersion) + return nil + }) +) diff --git a/pkg/apis/v1alpha2/zz_generated.deepcopy.go b/pkg/apis/v1alpha2/zz_generated.deepcopy.go new file mode 100644 index 000000000..4b5ecdea5 --- /dev/null +++ b/pkg/apis/v1alpha2/zz_generated.deepcopy.go @@ -0,0 +1,162 @@ +/* +Portions Copyright (c) Microsoft Corporation. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Code generated by controller-gen. DO NOT EDIT. + +package v1alpha2 + +import ( + "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/runtime" +) + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *AKSNodeClass) DeepCopyInto(out *AKSNodeClass) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + in.Spec.DeepCopyInto(&out.Spec) + out.Status = in.Status +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new AKSNodeClass. +func (in *AKSNodeClass) DeepCopy() *AKSNodeClass { + if in == nil { + return nil + } + out := new(AKSNodeClass) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *AKSNodeClass) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *AKSNodeClassList) DeepCopyInto(out *AKSNodeClassList) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ListMeta.DeepCopyInto(&out.ListMeta) + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]AKSNodeClass, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new AKSNodeClassList. +func (in *AKSNodeClassList) DeepCopy() *AKSNodeClassList { + if in == nil { + return nil + } + out := new(AKSNodeClassList) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *AKSNodeClassList) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *AKSNodeClassSpec) DeepCopyInto(out *AKSNodeClassSpec) { + *out = *in + if in.OSDiskSizeGB != nil { + in, out := &in.OSDiskSizeGB, &out.OSDiskSizeGB + *out = new(int32) + **out = **in + } + if in.ImageID != nil { + in, out := &in.ImageID, &out.ImageID + *out = new(string) + **out = **in + } + if in.ImageFamily != nil { + in, out := &in.ImageFamily, &out.ImageFamily + *out = new(string) + **out = **in + } + if in.ImageVersion != nil { + in, out := &in.ImageVersion, &out.ImageVersion + *out = new(string) + **out = **in + } + if in.Tags != nil { + in, out := &in.Tags, &out.Tags + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new AKSNodeClassSpec. +func (in *AKSNodeClassSpec) DeepCopy() *AKSNodeClassSpec { + if in == nil { + return nil + } + out := new(AKSNodeClassSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *AKSNodeClassStatus) DeepCopyInto(out *AKSNodeClassStatus) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new AKSNodeClassStatus. +func (in *AKSNodeClassStatus) DeepCopy() *AKSNodeClassStatus { + if in == nil { + return nil + } + out := new(AKSNodeClassStatus) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *Image) DeepCopyInto(out *Image) { + *out = *in + if in.Requirements != nil { + in, out := &in.Requirements, &out.Requirements + *out = make([]v1.NodeSelectorRequirement, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Image. +func (in *Image) DeepCopy() *Image { + if in == nil { + return nil + } + out := new(Image) + in.DeepCopyInto(out) + return out +} diff --git a/pkg/auth/autorest_auth.go b/pkg/auth/autorest_auth.go new file mode 100644 index 000000000..28e8fb398 --- /dev/null +++ b/pkg/auth/autorest_auth.go @@ -0,0 +1,92 @@ +/* +Portions Copyright (c) Microsoft Corporation. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package auth + +import ( + "fmt" + "os" + + "github.com/Azure/go-autorest/autorest" + "github.com/Azure/go-autorest/autorest/adal" + "github.com/Azure/go-autorest/autorest/azure" + klog "k8s.io/klog/v2" +) + +func NewAuthorizer(config *Config, env *azure.Environment) (autorest.Authorizer, error) { + token, err := newServicePrincipalTokenFromCredentials(config, env) + if err != nil { + return nil, fmt.Errorf("retrieve service principal token: %w", err) + } + return autorest.NewBearerAuthorizer(token), nil +} + +// newServicePrincipalTokenFromCredentials creates a new ServicePrincipalToken using values of the +// passed credentials map. +func newServicePrincipalTokenFromCredentials(config *Config, env *azure.Environment) (*adal.ServicePrincipalToken, error) { + oauthConfig, err := adal.NewOAuthConfig(env.ActiveDirectoryEndpoint, config.TenantID) + if err != nil { + return nil, fmt.Errorf("creating the OAuth config: %w", err) + } + + if config.UseManagedIdentityExtension { + klog.V(2).Infoln("azure: using managed identity extension to retrieve access token") + msiEndpoint, err := adal.GetMSIVMEndpoint() + if err != nil { + return nil, fmt.Errorf("getting the managed service identity endpoint: %w", err) + } + + if len(config.UserAssignedIdentityID) > 0 { + klog.V(4).Info("azure: using User Assigned MSI ID to retrieve access token") + return adal.NewServicePrincipalTokenFromMSIWithUserAssignedID(msiEndpoint, + env.ServiceManagementEndpoint, + config.UserAssignedIdentityID) + } + klog.V(4).Info("azure: using System Assigned MSI to retrieve access token") + return adal.NewServicePrincipalTokenFromMSI( + msiEndpoint, + env.ServiceManagementEndpoint) + } + + if len(config.AADClientSecret) > 0 { + klog.V(2).Infoln("azure: using client_id+client_secret to retrieve access token") + return adal.NewServicePrincipalToken( + *oauthConfig, + config.AADClientID, + config.AADClientSecret, + env.ServiceManagementEndpoint) + } + + if len(config.AADClientCertPath) > 0 && len(config.AADClientCertPassword) > 0 { + klog.V(2).Infoln("azure: using jwt client_assertion (client_cert+client_private_key) to retrieve access token") + certData, err := os.ReadFile(config.AADClientCertPath) + if err != nil { + return nil, fmt.Errorf("reading the client certificate from file %s: %w", config.AADClientCertPath, err) + } + certificate, privateKey, err := decodePkcs12(certData, config.AADClientCertPassword) + if err != nil { + return nil, fmt.Errorf("decoding the client certificate: %w", err) + } + return adal.NewServicePrincipalTokenFromCertificate( + *oauthConfig, + config.AADClientID, + certificate, + privateKey, + env.ServiceManagementEndpoint) + } + + return nil, fmt.Errorf("no credentials provided for AAD application %s", config.AADClientID) +} diff --git a/pkg/auth/config.go b/pkg/auth/config.go new file mode 100644 index 000000000..d27ae9307 --- /dev/null +++ b/pkg/auth/config.go @@ -0,0 +1,213 @@ +/* +Portions Copyright (c) Microsoft Corporation. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package auth + +import ( + "fmt" + "os" + "strconv" + "strings" + + "github.com/Azure/go-autorest/autorest" + "github.com/Azure/go-autorest/autorest/azure" +) + +const ( + // auth methods + authMethodPrincipal = "principal" + authMethodCLI = "cli" +) + +const ( + // from azure_manager + vmTypeVMSS = "vmss" +) + +type cfgField struct { + val string + name string +} + +// ClientConfig contains all essential information to create an Azure client. +type ClientConfig struct { + CloudName string + Location string + SubscriptionID string + ResourceManagerEndpoint string + Authorizer autorest.Authorizer + UserAgent string +} + +// Config holds the configuration parsed from the --cloud-config flag +type Config struct { + Cloud string `json:"cloud" yaml:"cloud"` + Location string `json:"location" yaml:"location"` + TenantID string `json:"tenantId" yaml:"tenantId"` + SubscriptionID string `json:"subscriptionId" yaml:"subscriptionId"` + ResourceGroup string `json:"resourceGroup" yaml:"resourceGroup"` + VMType string `json:"vmType" yaml:"vmType"` + + // AuthMethod determines how to authorize requests for the Azure + // cloud. Valid options are "principal" (= the traditional + // service principle approach) and "cli" (= load az command line + // config file). The default is "principal". + AuthMethod string `json:"authMethod" yaml:"authMethod"` + + // Settings for a service principal. + AADClientID string `json:"aadClientId" yaml:"aadClientId"` + AADClientSecret string `json:"aadClientSecret" yaml:"aadClientSecret"` + AADClientCertPath string `json:"aadClientCertPath" yaml:"aadClientCertPath"` + AADClientCertPassword string `json:"aadClientCertPassword" yaml:"aadClientCertPassword"` + UseManagedIdentityExtension bool `json:"useManagedIdentityExtension" yaml:"useManagedIdentityExtension"` + UserAssignedIdentityID string `json:"userAssignedIdentityID" yaml:"userAssignedIdentityID"` + + //Configs only for AKS + ClusterName string `json:"clusterName" yaml:"clusterName"` + //Config only for AKS + NodeResourceGroup string `json:"nodeResourceGroup" yaml:"nodeResourceGroup"` + //SubnetId is the resource ID of the subnet that VM network interfaces should use + SubnetID string `json:"subnetId" yaml:"subnetId"` + VnetName string `json:"vnetName" yaml:"vnetName"` + SubnetName string `json:"subnetName" yaml:"subnetName"` +} + +func (cfg *Config) PrepareConfig() error { + cfg.BaseVars() + err := cfg.prepareMSI() + if err != nil { + return err + } + return nil +} + +func (cfg *Config) BaseVars() { + cfg.Cloud = os.Getenv("ARM_CLOUD") + cfg.Location = os.Getenv("LOCATION") + cfg.ResourceGroup = os.Getenv("ARM_RESOURCE_GROUP") + cfg.TenantID = os.Getenv("ARM_TENANT_ID") + cfg.SubscriptionID = os.Getenv("ARM_SUBSCRIPTION_ID") + cfg.AADClientID = os.Getenv("ARM_CLIENT_ID") + cfg.AADClientSecret = os.Getenv("ARM_CLIENT_SECRET") + cfg.VMType = strings.ToLower(os.Getenv("ARM_VM_TYPE")) + cfg.AADClientCertPath = os.Getenv("ARM_CLIENT_CERT_PATH") + cfg.AADClientCertPassword = os.Getenv("ARM_CLIENT_CERT_PASSWORD") + cfg.ClusterName = os.Getenv("AZURE_CLUSTER_NAME") + cfg.NodeResourceGroup = os.Getenv("AZURE_NODE_RESOURCE_GROUP") + cfg.SubnetID = os.Getenv("AZURE_SUBNET_ID") + cfg.SubnetName = os.Getenv("AZURE_SUBNET_NAME") + cfg.VnetName = os.Getenv("AZURE_VNET_NAME") + // cfg.VnetGuid = os.Getenv("AZURE_VNET_GUID") // This field needs to be resolved inside of karpenter, so we will get it in the azClient initialization +} + +func (cfg *Config) prepareMSI() error { + useManagedIdentityExtensionFromEnv := os.Getenv("ARM_USE_MANAGED_IDENTITY_EXTENSION") + if len(useManagedIdentityExtensionFromEnv) > 0 { + shouldUse, err := strconv.ParseBool(useManagedIdentityExtensionFromEnv) + if err != nil { + return err + } + cfg.UseManagedIdentityExtension = shouldUse + } + userAssignedIdentityIDFromEnv := os.Getenv("ARM_USER_ASSIGNED_IDENTITY_ID") + if userAssignedIdentityIDFromEnv != "" { + cfg.UserAssignedIdentityID = userAssignedIdentityIDFromEnv + } + return nil +} + +// BuildAzureConfig returns a Config object for the Azure clients +func BuildAzureConfig() (*Config, error) { + var err error + cfg := &Config{} + err = cfg.PrepareConfig() + if err != nil { + return nil, err + } + cfg.TrimSpace() + setVMType(cfg) + + if err := cfg.validate(); err != nil { + return nil, err + } + return cfg, nil +} + +func setVMType(cfg *Config) { + // Defaulting vmType to vmss. + if cfg.VMType == "" { + cfg.VMType = vmTypeVMSS + } +} + +func (cfg *Config) GetAzureClientConfig(authorizer autorest.Authorizer, env *azure.Environment) *ClientConfig { + azClientConfig := &ClientConfig{ + Location: cfg.Location, + SubscriptionID: cfg.SubscriptionID, + ResourceManagerEndpoint: env.ResourceManagerEndpoint, + Authorizer: authorizer, + } + + return azClientConfig +} + +// TrimSpace removes all leading and trailing white spaces. +func (cfg *Config) TrimSpace() { + cfg.Cloud = strings.TrimSpace(cfg.Cloud) + cfg.TenantID = strings.TrimSpace(cfg.TenantID) + cfg.SubscriptionID = strings.TrimSpace(cfg.SubscriptionID) + cfg.ResourceGroup = strings.TrimSpace(cfg.ResourceGroup) + cfg.VMType = strings.TrimSpace(cfg.VMType) + cfg.AADClientID = strings.TrimSpace(cfg.AADClientID) + cfg.AADClientSecret = strings.TrimSpace(cfg.AADClientSecret) + cfg.AADClientCertPath = strings.TrimSpace(cfg.AADClientCertPath) + cfg.AADClientCertPassword = strings.TrimSpace(cfg.AADClientCertPassword) + cfg.ClusterName = strings.TrimSpace(cfg.ClusterName) + cfg.NodeResourceGroup = strings.TrimSpace(cfg.NodeResourceGroup) + cfg.SubnetID = strings.TrimSpace(cfg.SubnetID) + cfg.SubnetName = strings.TrimSpace(cfg.SubnetName) + cfg.VnetName = strings.TrimSpace(cfg.VnetName) +} + +func (cfg *Config) validate() error { + // Setup fields and validate all of them are not empty + fields := []cfgField{ + {cfg.SubscriptionID, "subscription ID"}, + {cfg.NodeResourceGroup, "node resource group"}, + {cfg.VMType, "VM type"}, + // Even though the config doesnt use some of these, + // its good to validate they were set in the environment + {cfg.SubnetID, "subnet ID"}, + {cfg.SubnetName, "subnet name"}, + {cfg.VnetName, "vnet name"}, + } + + for _, field := range fields { + if field.val == "" { + return fmt.Errorf("%s not set", field.name) + } + } + + if cfg.UseManagedIdentityExtension { + return nil + } + + if cfg.AuthMethod != "" && cfg.AuthMethod != authMethodPrincipal && cfg.AuthMethod != authMethodCLI { + return fmt.Errorf("unsupported authorization method: %s", cfg.AuthMethod) + } + + return nil +} diff --git a/pkg/auth/config_test.go b/pkg/auth/config_test.go new file mode 100644 index 000000000..3b35790d9 --- /dev/null +++ b/pkg/auth/config_test.go @@ -0,0 +1,146 @@ +/* +Portions Copyright (c) Microsoft Corporation. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package auth + +import ( + "os" + "reflect" + "testing" +) + +func TestBuildAzureConfig(t *testing.T) { + tests := []struct { + name string + expected *Config + wantErr bool + env map[string]string + }{ + { + name: "required env vars not present", + expected: nil, + wantErr: true, + }, + { + name: "default", + expected: &Config{ + SubscriptionID: "12345", + ResourceGroup: "my-rg", + NodeResourceGroup: "my-node-rg", + SubnetID: "12345", + SubnetName: "my-subnet", + VnetName: "my-vnet", + VMType: "vmss", + }, + wantErr: false, + env: map[string]string{ + "ARM_RESOURCE_GROUP": "my-rg", + "ARM_SUBSCRIPTION_ID": "12345", + "AZURE_NODE_RESOURCE_GROUP": "my-node-rg", + "AZURE_SUBNET_ID": "12345", + "AZURE_SUBNET_NAME": "my-subnet", + "AZURE_VNET_NAME": "my-vnet", + }, + }, + { + name: "vmType=vm", // tests setVMType() + expected: &Config{ + SubscriptionID: "12345", + ResourceGroup: "my-rg", + NodeResourceGroup: "my-node-rg", + SubnetID: "12345", + SubnetName: "my-subnet", + VnetName: "my-vnet", + VMType: "vm", + }, + wantErr: false, + env: map[string]string{ + "ARM_RESOURCE_GROUP": "my-rg", + "ARM_SUBSCRIPTION_ID": "12345", + "AZURE_NODE_RESOURCE_GROUP": "my-node-rg", + "AZURE_SUBNET_ID": "12345", + "AZURE_SUBNET_NAME": "my-subnet", + "AZURE_VNET_NAME": "my-vnet", + "ARM_VM_TYPE": "vm", + }, + }, + { + name: "bogus ARM_USE_MANAGED_IDENTITY_EXTENSION", + expected: nil, + wantErr: true, + env: map[string]string{ + "ARM_RESOURCE_GROUP": "my-rg", + "ARM_SUBSCRIPTION_ID": "12345", + "AZURE_NODE_RESOURCE_GROUP": "my-node-rg", + "AZURE_SUBNET_ID": "12345", + "AZURE_SUBNET_NAME": "my-subnet", + "AZURE_VNET_NAME": "my-vnet", + "ARM_USE_MANAGED_IDENTITY_EXTENSION": "foo", // this is not a supported value + }, + }, + { + name: "valid msi", + expected: &Config{ + SubscriptionID: "12345", + ResourceGroup: "my-rg", + NodeResourceGroup: "my-node-rg", + SubnetID: "12345", + SubnetName: "my-subnet", + VnetName: "my-vnet", + VMType: "vmss", + UseManagedIdentityExtension: true, + UserAssignedIdentityID: "12345", + }, + wantErr: false, + env: map[string]string{ + "ARM_RESOURCE_GROUP": "my-rg", + "ARM_SUBSCRIPTION_ID": "12345", + "AZURE_NODE_RESOURCE_GROUP": "my-node-rg", + "AZURE_SUBNET_ID": "12345", + "AZURE_SUBNET_NAME": "my-subnet", + "AZURE_VNET_NAME": "my-vnet", + "ARM_USE_MANAGED_IDENTITY_EXTENSION": "true", + "ARM_USER_ASSIGNED_IDENTITY_ID": "12345", + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + for k, v := range tt.env { + err := os.Setenv(k, v) + if err != nil { + t.Errorf("error setting environment %v = %s", tt.env, err) + return + } + } + got, err := BuildAzureConfig() + if (err != nil) != tt.wantErr { + t.Errorf("BuildAzureConfig() error = %v, wantErr %v", err, tt.wantErr) + return + } + if equal := reflect.DeepEqual(got, tt.expected); !equal { + t.Errorf("BuildAzureConfig() = %v, want %v", got, tt.expected) + } + for k := range tt.env { + err := os.Unsetenv(k) + if err != nil { + t.Errorf("error unsetting environment %v = %s", tt.env, err) + return + } + } + }) + } +} diff --git a/pkg/auth/cred.go b/pkg/auth/cred.go new file mode 100644 index 000000000..f0f49f819 --- /dev/null +++ b/pkg/auth/cred.go @@ -0,0 +1,47 @@ +/* +Portions Copyright (c) Microsoft Corporation. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package auth + +import ( + "fmt" + + "github.com/Azure/azure-sdk-for-go/sdk/azcore" + "github.com/Azure/azure-sdk-for-go/sdk/azidentity" +) + +// NewCredential provides a token credential for msi and service principal auth +func NewCredential(cfg *Config) (azcore.TokenCredential, error) { + if cfg == nil { + return nil, fmt.Errorf("failed to create credential, nil config provided") + } + + if cfg.UseManagedIdentityExtension || cfg.AADClientID == "msi" { + msiCred, err := azidentity.NewManagedIdentityCredential(&azidentity.ManagedIdentityCredentialOptions{ + ID: azidentity.ClientID(cfg.UserAssignedIdentityID), + }) + if err != nil { + return nil, err + } + return msiCred, nil + } + // service principal case + cred, err := azidentity.NewClientSecretCredential(cfg.TenantID, cfg.AADClientID, cfg.AADClientSecret, nil) + if err != nil { + return nil, err + } + return cred, nil +} diff --git a/pkg/auth/cred_test.go b/pkg/auth/cred_test.go new file mode 100644 index 000000000..22d48ab93 --- /dev/null +++ b/pkg/auth/cred_test.go @@ -0,0 +1,100 @@ +/* +Portions Copyright (c) Microsoft Corporation. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package auth + +import ( + "reflect" + "testing" + + "github.com/Azure/azure-sdk-for-go/sdk/azidentity" +) + +func TestNewCredential(t *testing.T) { + tests := []struct { + name string + cfg *Config + want reflect.Type + wantErr bool + wantErrStr string + }{ + { + name: "nil Config", + cfg: nil, + want: nil, + wantErr: true, + wantErrStr: "failed to create credential, nil config provided", + }, + { + name: "AAD client ID is MSI", + cfg: &Config{ + AADClientID: "msi", + TenantID: "00000000-0000-0000-0000-000000000000", + UserAssignedIdentityID: "12345678-1234-1234-1234-123456789012", + }, + want: reflect.TypeOf(&azidentity.ManagedIdentityCredential{}), + wantErr: false, + }, + { + name: "AAD client ID is using MSI extension", + cfg: &Config{ + UseManagedIdentityExtension: true, + AADClientID: "msi", + TenantID: "00000000-0000-0000-0000-000000000000", + UserAssignedIdentityID: "12345678-1234-1234-1234-123456789012", + }, + want: reflect.TypeOf(&azidentity.ManagedIdentityCredential{}), + wantErr: false, + }, + { + name: "AADClientID is not MSI", + cfg: &Config{ + AADClientID: "test-client-id", + AADClientSecret: "test-client-secret", + TenantID: "00000000-0000-0000-0000-000000000000", + }, + want: reflect.TypeOf(&azidentity.ClientSecretCredential{}), + wantErr: false, + }, + { + name: "AADClientID is not MSI and UserAssignedIdentityID is set", + cfg: &Config{ + AADClientID: "test-client-id", + AADClientSecret: "test-client-secret", + TenantID: "00000000-0000-0000-0000-000000000000", + UserAssignedIdentityID: "12345678-1234-1234-1234-123456789012", + }, + want: reflect.TypeOf(&azidentity.ClientSecretCredential{}), + wantErr: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := NewCredential(tt.cfg) + if (err != nil) != tt.wantErr { + t.Errorf("NewCredential() error = %v, wantErr %v", err, tt.wantErr) + return + } + if err != nil && err.Error() != tt.wantErrStr { + t.Errorf("NewCredential() error = %v, wantErrStr %v", err, tt.wantErrStr) + return + } + if reflect.TypeOf(got) != tt.want { + t.Errorf("NewCredential() = %v, want %v", reflect.TypeOf(got), tt.want) + } + }) + } +} diff --git a/pkg/auth/util.go b/pkg/auth/util.go new file mode 100644 index 000000000..96ebbc7e5 --- /dev/null +++ b/pkg/auth/util.go @@ -0,0 +1,46 @@ +/* +Portions Copyright (c) Microsoft Corporation. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package auth + +import ( + "crypto/rsa" + "crypto/x509" + "fmt" + + "golang.org/x/crypto/pkcs12" + + "github.com/Azure/karpenter/pkg/utils/project" +) + +// decodePkcs12 decodes a PKCS#12 client certificate by extracting the public certificate and +// the private RSA key +func decodePkcs12(pkcs []byte, password string) (*x509.Certificate, *rsa.PrivateKey, error) { + privateKey, certificate, err := pkcs12.Decode(pkcs, password) + if err != nil { + return nil, nil, fmt.Errorf("decoding the PKCS#12 client certificate: %w", err) + } + rsaPrivateKey, isRsaKey := privateKey.(*rsa.PrivateKey) + if !isRsaKey { + return nil, nil, fmt.Errorf("PKCS#12 certificate must contain a RSA private key") + } + + return certificate, rsaPrivateKey, nil +} + +func GetUserAgentExtension() string { + return fmt.Sprintf("karpenter-aks/v%s", project.Version) +} diff --git a/pkg/cache/cache.go b/pkg/cache/cache.go new file mode 100644 index 000000000..f25900c8b --- /dev/null +++ b/pkg/cache/cache.go @@ -0,0 +1,30 @@ +/* +Portions Copyright (c) Microsoft Corporation. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package cache + +import "time" + +const ( + // KubernetesVersionTTL is the time before the detected Kubernetes version is removed from cache, + // to be re-detected next time it is needed. + KubernetesVersionTTL = 15 * time.Minute + // UnavailableOfferingsTTL is the time before offerings that were marked as unavailable + // are removed from the cache and are available for launch again + UnavailableOfferingsTTL = 3 * time.Minute + // DefaultCleanupInterval triggers cache cleanup (lazy eviction) at this interval. + DefaultCleanupInterval = 10 * time.Minute +) diff --git a/pkg/cache/unavailableofferings.go b/pkg/cache/unavailableofferings.go new file mode 100644 index 000000000..9ff1597e5 --- /dev/null +++ b/pkg/cache/unavailableofferings.go @@ -0,0 +1,79 @@ +/* +Portions Copyright (c) Microsoft Corporation. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package cache + +import ( + "context" + "fmt" + "time" + + "github.com/patrickmn/go-cache" + "knative.dev/pkg/logging" +) + +// UnavailableOfferings stores any offerings that return ICE (insufficient capacity errors) when +// attempting to launch the capacity. These offerings are ignored as long as they are in the cache on +// GetInstanceTypes responses +type UnavailableOfferings struct { + // key: ::, value: struct{}{} + cache *cache.Cache +} + +func NewUnavailableOfferingsWithCache(c *cache.Cache) *UnavailableOfferings { + return &UnavailableOfferings{ + cache: c, + } +} + +func NewUnavailableOfferings() *UnavailableOfferings { + c := cache.New(UnavailableOfferingsTTL, DefaultCleanupInterval) + return &UnavailableOfferings{ + cache: c, + } +} + +// IsUnavailable returns true if the offering appears in the cache +func (u *UnavailableOfferings) IsUnavailable(instanceType, zone, capacityType string) bool { + _, found := u.cache.Get(u.key(instanceType, zone, capacityType)) + return found +} + +// MarkUnavailableWithTTL allows us to mark an offering unavailable with a custom TTL +func (u *UnavailableOfferings) MarkUnavailableWithTTL(ctx context.Context, unavailableReason, instanceType, zone, capacityType string, ttl time.Duration) { + // even if the key is already in the cache, we still need to call Set to extend the cached entry's TTL + logging.FromContext(ctx).With( + "unavailable", unavailableReason, + "instance-type", instanceType, + "zone", zone, + "capacity-type", capacityType, + "ttl", ttl).Debugf("removing offering from offerings") + u.cache.Set(u.key(instanceType, zone, capacityType), struct{}{}, ttl) +} + +// MarkUnavailable communicates recently observed temporary capacity shortages in the provided offerings +func (u *UnavailableOfferings) MarkUnavailable(ctx context.Context, unavailableReason, instanceType, zone, capacityType string) { + u.MarkUnavailableWithTTL(ctx, unavailableReason, instanceType, zone, capacityType, UnavailableOfferingsTTL) +} + +func (u *UnavailableOfferings) Flush() { + u.cache.Flush() +} + +// key returns the cache key for all offerings in the cache +func (u *UnavailableOfferings) key(instanceType string, zone string, capacityType string) string { + return fmt.Sprintf("%s:%s:%s", capacityType, instanceType, zone) +} diff --git a/pkg/cache/unavailableofferings_test.go b/pkg/cache/unavailableofferings_test.go new file mode 100644 index 000000000..7b82c3135 --- /dev/null +++ b/pkg/cache/unavailableofferings_test.go @@ -0,0 +1,64 @@ +/* +Portions Copyright (c) Microsoft Corporation. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package cache + +import ( + "context" + "testing" + "time" + + "github.com/patrickmn/go-cache" +) + +func TestUnavailableOfferings(t *testing.T) { + // create a new cache with a short TTL + c := cache.New(time.Second, time.Second) + u := NewUnavailableOfferingsWithCache(c) + + // test that an offering is not marked as unavailable initially + if u.IsUnavailable("NV16as_v4", "westus", "spot") { + t.Error("Offering should not be marked as unavailable initially") + } + + // mark the offering as unavailable + u.MarkUnavailableWithTTL(context.TODO(), "test reason", "NV16as_v4", "westus", "spot", time.Second) + + // test that the offering is now marked as unavailable + if !u.IsUnavailable("NV16as_v4", "westus", "spot") { + t.Error("Offering should be marked as unavailable after being marked as such") + } + + // wait for the cache entry to expire + time.Sleep(time.Second) + + // test that the offering is no longer marked as unavailable + if u.IsUnavailable("NV16as_v4", "westus", "spot") { + t.Error("Offering should not be marked as unavailable after cache entry has expired") + } +} + +func TestUnavailableOfferings_KeyGeneration(t *testing.T) { + c := cache.New(time.Second, time.Second) + u := NewUnavailableOfferingsWithCache(c) + + // test that the key is generated correctly + expectedKey := "spot:NV16as_v4:westus" + key := u.key("NV16as_v4", "westus", "spot") + if key != expectedKey { + t.Errorf("Expected key to be %s, but got %s", expectedKey, key) + } +} diff --git a/pkg/cloudprovider/cloudprovider.go b/pkg/cloudprovider/cloudprovider.go new file mode 100644 index 000000000..e19206514 --- /dev/null +++ b/pkg/cloudprovider/cloudprovider.go @@ -0,0 +1,358 @@ +/* +Portions Copyright (c) Microsoft Corporation. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package cloudprovider + +import ( + "context" + "fmt" + "net/http" + "strings" + + v1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime/schema" + + "k8s.io/apimachinery/pkg/types" + "knative.dev/pkg/logging" + "sigs.k8s.io/controller-runtime/pkg/client" + + // nolint SA1019 - deprecated package + "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute" + + "github.com/Azure/karpenter/pkg/apis" + "github.com/Azure/karpenter/pkg/apis/v1alpha2" + "github.com/Azure/karpenter/pkg/controllers/nodeclaim/inplaceupdate" + + cloudproviderevents "github.com/Azure/karpenter/pkg/cloudprovider/events" + "github.com/Azure/karpenter/pkg/providers/imagefamily" + "github.com/Azure/karpenter/pkg/providers/instance" + "github.com/Azure/karpenter/pkg/providers/instancetype" + "github.com/Azure/karpenter/pkg/utils" + nodeclaimutil "github.com/aws/karpenter-core/pkg/utils/nodeclaim" + "github.com/samber/lo" + + coreapis "github.com/aws/karpenter-core/pkg/apis" + corev1beta1 "github.com/aws/karpenter-core/pkg/apis/v1beta1" + "github.com/aws/karpenter-core/pkg/events" + + "github.com/aws/karpenter-core/pkg/cloudprovider" + + "github.com/aws/karpenter-core/pkg/scheduling" + "github.com/aws/karpenter-core/pkg/utils/functional" + "github.com/aws/karpenter-core/pkg/utils/resources" +) + +func init() { + coreapis.Settings = append(coreapis.Settings, apis.Settings...) +} + +var _ cloudprovider.CloudProvider = (*CloudProvider)(nil) + +type CloudProvider struct { + instanceTypeProvider *instancetype.Provider + instanceProvider *instance.Provider + kubeClient client.Client + imageProvider *imagefamily.Provider + recorder events.Recorder +} + +func New(instanceTypeProvider *instancetype.Provider, instanceProvider *instance.Provider, recorder events.Recorder, + kubeClient client.Client, imageProvider *imagefamily.Provider) *CloudProvider { + return &CloudProvider{ + instanceTypeProvider: instanceTypeProvider, + instanceProvider: instanceProvider, + kubeClient: kubeClient, + imageProvider: imageProvider, + recorder: recorder, + } +} + +// Create a node given the constraints. +func (c *CloudProvider) Create(ctx context.Context, nodeClaim *corev1beta1.NodeClaim) (*corev1beta1.NodeClaim, error) { + nodeClass, err := c.resolveNodeClassFromNodeClaim(ctx, nodeClaim) + if err != nil { + if errors.IsNotFound(err) { + c.recorder.Publish(cloudproviderevents.NodeClaimFailedToResolveNodeClass(nodeClaim)) + } + // We treat a failure to resolve the NodeClass as an ICE since this means there is no capacity possibilities for this NodeClaim + return nil, cloudprovider.NewInsufficientCapacityError(fmt.Errorf("resolving node class, %w", err)) + } + + instanceTypes, err := c.resolveInstanceTypes(ctx, nodeClaim, nodeClass) + if err != nil { + return nil, fmt.Errorf("resolving instance types, %w", err) + } + if len(instanceTypes) == 0 { + return nil, cloudprovider.NewInsufficientCapacityError(fmt.Errorf("all requested instance types were unavailable during launch")) + } + instance, err := c.instanceProvider.Create(ctx, nodeClass, nodeClaim, instanceTypes) + if err != nil { + return nil, fmt.Errorf("creating instance, %w", err) + } + instanceType, _ := lo.Find(instanceTypes, func(i *cloudprovider.InstanceType) bool { + return i.Name == string(*instance.Properties.HardwareProfile.VMSize) + }) + + return c.instanceToNodeClaim(ctx, instance, instanceType) +} + +func (c *CloudProvider) List(ctx context.Context) ([]*corev1beta1.NodeClaim, error) { + instances, err := c.instanceProvider.List(ctx) + if err != nil { + return nil, fmt.Errorf("listing instances, %w", err) + } + var nodeClaims []*corev1beta1.NodeClaim + for _, instance := range instances { + instanceType, err := c.resolveInstanceTypeFromInstance(ctx, instance) + if err != nil { + return nil, fmt.Errorf("resolving instance type, %w", err) + } + nodeClaim, err := c.instanceToNodeClaim(ctx, instance, instanceType) + if err != nil { + return nil, fmt.Errorf("converting instance to node claim, %w", err) + } + + nodeClaims = append(nodeClaims, nodeClaim) + } + return nodeClaims, nil +} + +func (c *CloudProvider) Link(ctx context.Context, nodeClaim *corev1beta1.NodeClaim) error { + ctx = logging.WithLogger(ctx, logging.FromContext(ctx).With("nodeclaim", nodeClaim.Name)) + vmName, err := utils.GetVMName(nodeClaim.Status.ProviderID) + if err != nil { + return fmt.Errorf("getting vm name, %w", err) + } + ctx = logging.WithLogger(ctx, logging.FromContext(ctx).With("vmName", vmName)) + return c.instanceProvider.Link(ctx, vmName, nodeClaim.Labels[corev1beta1.NodePoolLabelKey]) +} + +func (c *CloudProvider) Get(ctx context.Context, providerID string) (*corev1beta1.NodeClaim, error) { + vmName, err := utils.GetVMName(providerID) + if err != nil { + return nil, fmt.Errorf("getting vm name, %w", err) + } + ctx = logging.WithLogger(ctx, logging.FromContext(ctx).With("id", vmName)) + instance, err := c.instanceProvider.Get(ctx, vmName) + if err != nil { + return nil, fmt.Errorf("getting instance, %w", err) + } + instanceType, err := c.resolveInstanceTypeFromInstance(ctx, instance) + if err != nil { + return nil, fmt.Errorf("resolving instance type, %w", err) + } + return c.instanceToNodeClaim(ctx, instance, instanceType) +} + +func (c *CloudProvider) LivenessProbe(req *http.Request) error { + return c.instanceTypeProvider.LivenessProbe(req) +} + +// GetInstanceTypes returns all available InstanceTypes +func (c *CloudProvider) GetInstanceTypes(ctx context.Context, nodePool *corev1beta1.NodePool) ([]*cloudprovider.InstanceType, error) { + if nodePool == nil { + return c.instanceTypeProvider.List(ctx, &corev1beta1.KubeletConfiguration{}, &v1alpha2.AKSNodeClass{}) + } + + nodeClass, err := c.resolveNodeClassFromNodePool(ctx, nodePool) + if err != nil { + if errors.IsNotFound(err) { + c.recorder.Publish(cloudproviderevents.NodePoolFailedToResolveNodeClass(nodePool)) + } + // We must return an error here in the event of the node class not being found. Otherwise users just get + // no instance types and a failure to schedule with no indicator pointing to a bad configuration + // as the cause. + return nil, fmt.Errorf("resolving node class, %w", err) + } + instanceTypes, err := c.instanceTypeProvider.List(ctx, nodePool.Spec.Template.Spec.Kubelet, nodeClass) + if err != nil { + return nil, err + } + return instanceTypes, nil +} + +func (c *CloudProvider) Delete(ctx context.Context, nodeClaim *corev1beta1.NodeClaim) error { + return c.instanceProvider.Delete(ctx, nodeClaim) +} + +func (c *CloudProvider) IsDrifted(ctx context.Context, nodeClaim *corev1beta1.NodeClaim) (cloudprovider.DriftReason, error) { + // Not needed when GetInstanceTypes removes nodepool dependency + nodePool, err := nodeclaimutil.Owner(ctx, c.kubeClient, nodeClaim) + if err != nil { + return "", client.IgnoreNotFound(fmt.Errorf("resolving owner, %w", err)) + } + if nodePool.Spec.Template.Spec.NodeClassRef == nil { + return "", nil + } + nodeClass, err := c.resolveNodeClassFromNodePool(ctx, nodePool) + if err != nil { + if errors.IsNotFound(err) { + c.recorder.Publish(cloudproviderevents.NodePoolFailedToResolveNodeClass(nodePool)) + } + return "", client.IgnoreNotFound(fmt.Errorf("resolving node class, %w", err)) + } + + k8sVersionDrifted, err := c.isK8sVersionDrifted(ctx, nodeClaim) + if err != nil { + return "", err + } + if k8sVersionDrifted != "" { + return k8sVersionDrifted, nil + } + imageVersionDrifted, err := c.isImageVersionDrifted(ctx, nodeClaim, nodeClass) + if err != nil { + return "", err + } + if imageVersionDrifted != "" { + return imageVersionDrifted, nil + } + imageDrifted, err := c.isImageDrifted(ctx, nodeClaim, nodePool, nodeClass) + if err != nil { + return "", err + } + return imageDrifted, nil +} + +// Name returns the CloudProvider implementation name. +func (c *CloudProvider) Name() string { + return "azure" +} + +func (c *CloudProvider) resolveNodeClassFromNodeClaim(ctx context.Context, nodeClaim *corev1beta1.NodeClaim) (*v1alpha2.AKSNodeClass, error) { + nodeClass := &v1alpha2.AKSNodeClass{} + if err := c.kubeClient.Get(ctx, types.NamespacedName{Name: nodeClaim.Spec.NodeClassRef.Name}, nodeClass); err != nil { + return nil, err + } + // For the purposes of NodeClass CloudProvider resolution, we treat deleting NodeClasses as NotFound + if !nodeClass.DeletionTimestamp.IsZero() { + return nil, errors.NewNotFound(v1alpha2.SchemeGroupVersion.WithResource("aksnodeclasses").GroupResource(), nodeClass.Name) + } + return nodeClass, nil +} + +func (c *CloudProvider) resolveNodeClassFromNodePool(ctx context.Context, nodePool *corev1beta1.NodePool) (*v1alpha2.AKSNodeClass, error) { + nodeClass := &v1alpha2.AKSNodeClass{} + if err := c.kubeClient.Get(ctx, types.NamespacedName{Name: nodePool.Spec.Template.Spec.NodeClassRef.Name}, nodeClass); err != nil { + return nil, err + } + // For the purposes of NodeClass CloudProvider resolution, we treat deleting NodeClasses as NotFound + if !nodeClass.DeletionTimestamp.IsZero() { + return nil, errors.NewNotFound(v1alpha2.SchemeGroupVersion.WithResource("aksnodeclasses").GroupResource(), nodeClass.Name) + } + return nodeClass, nil +} +func (c *CloudProvider) resolveInstanceTypes(ctx context.Context, nodeClaim *corev1beta1.NodeClaim, nodeClass *v1alpha2.AKSNodeClass) ([]*cloudprovider.InstanceType, error) { + instanceTypes, err := c.instanceTypeProvider.List(ctx, nodeClaim.Spec.Kubelet, nodeClass) + if err != nil { + return nil, fmt.Errorf("getting instance types, %w", err) + } + + reqs := scheduling.NewNodeSelectorRequirements(nodeClaim.Spec.Requirements...) + return lo.Filter(instanceTypes, func(i *cloudprovider.InstanceType, _ int) bool { + return reqs.Compatible(i.Requirements, scheduling.AllowUndefinedWellKnownLabelsV1Beta1) == nil && + len(i.Offerings.Requirements(reqs).Available()) > 0 && + resources.Fits(nodeClaim.Spec.Resources.Requests, i.Allocatable()) + }), nil + + // Old logic + // return lo.Filter(instanceTypes, func(i *cloudprovider.InstanceType, _ int) bool { + // return reqs.Get(v1.LabelInstanceTypeStable).Has(i.Name) && + // len(i.Offerings.Requirements(reqs).Available()) > 0 + // }), nil +} + +func (c *CloudProvider) resolveInstanceTypeFromInstance(ctx context.Context, instance *armcompute.VirtualMachine) (*cloudprovider.InstanceType, error) { + nodePool, err := c.resolveNodePoolFromInstance(ctx, instance) + if err != nil { + // If we can't resolve the provisioner, we fallback to not getting instance type info + return nil, client.IgnoreNotFound(fmt.Errorf("resolving node pool, %w", err)) + } + instanceTypes, err := c.GetInstanceTypes(ctx, nodePool) + if err != nil { + // If we can't resolve the nodepool, we fallback to not getting instance type info + return nil, client.IgnoreNotFound(fmt.Errorf("resolving node template, %w", err)) + } + instanceType, _ := lo.Find(instanceTypes, func(i *cloudprovider.InstanceType) bool { + return i.Name == string(*instance.Properties.HardwareProfile.VMSize) + }) + return instanceType, nil +} + +func (c *CloudProvider) resolveNodePoolFromInstance(ctx context.Context, instance *armcompute.VirtualMachine) (*corev1beta1.NodePool, error) { + nodePoolName, ok := instance.Tags[corev1beta1.NodePoolLabelKey] + if ok && *nodePoolName != "" { + nodePool := &corev1beta1.NodePool{} + if err := c.kubeClient.Get(ctx, types.NamespacedName{Name: *nodePoolName}, nodePool); err != nil { + return nil, err + } + return nodePool, nil + } + + return nil, errors.NewNotFound(schema.GroupResource{Group: corev1beta1.Group, Resource: "NodePool"}, "") +} + +func (c *CloudProvider) instanceToNodeClaim(ctx context.Context, vm *armcompute.VirtualMachine, instanceType *cloudprovider.InstanceType) (*corev1beta1.NodeClaim, error) { + nodeClaim := &corev1beta1.NodeClaim{} + labels := map[string]string{} + annotations := map[string]string{} + + if instanceType != nil { + labels = instance.GetAllSingleValuedRequirementLabels(instanceType) + nodeClaim.Status.Capacity = functional.FilterMap(instanceType.Capacity, func(_ v1.ResourceName, v resource.Quantity) bool { return !resources.IsZero(v) }) + nodeClaim.Status.Allocatable = functional.FilterMap(instanceType.Allocatable(), func(_ v1.ResourceName, v resource.Quantity) bool { return !resources.IsZero(v) }) + } + + if zoneID, err := instance.GetZoneID(vm); err != nil { + logging.FromContext(ctx).Warnf("Failed to get zone for VM %s, %v", *vm.Name, err) + } else { + zone := makeZone(*vm.Location, zoneID) + // aks-node-validating-webhook protects v1.LabelTopologyZone, will be set elsewhere, so we use a different label + labels[v1alpha2.AlternativeLabelTopologyZone] = zone + } + + labels[corev1beta1.CapacityTypeLabelKey] = instance.GetCapacityType(vm) + labels[v1alpha2.LabelSKUHyperVGeneration] = instance.GetHyperVGeneration(vm) + + // TODO: v1beta1 new kes/labels + if tag, ok := vm.Tags[instance.NodePoolTagKey]; ok { + labels[corev1beta1.NodePoolLabelKey] = *tag + } + + inPlaceUpdateHash, err := inplaceupdate.HashFromVM(vm) + if err != nil { + return nil, fmt.Errorf("failed to calculate in place update hash, %w", err) + } + annotations[v1alpha2.AnnotationInPlaceUpdateHash] = inPlaceUpdateHash + + nodeClaim.Name = GenerateNodeClaimName(*vm.Name) + nodeClaim.Labels = labels + nodeClaim.Annotations = annotations + nodeClaim.CreationTimestamp = metav1.Time{Time: *vm.Properties.TimeCreated} + nodeClaim.Status.ProviderID = utils.ResourceIDToProviderID(ctx, *vm.ID) + return nodeClaim, nil +} + +func GenerateNodeClaimName(vmName string) string { + return strings.TrimLeft("aks-", vmName) +} + +// makeZone returns the zone value in format of -. +func makeZone(location string, zoneID string) string { + return fmt.Sprintf("%s-%s", strings.ToLower(location), zoneID) +} diff --git a/pkg/cloudprovider/drift.go b/pkg/cloudprovider/drift.go new file mode 100644 index 000000000..036a78ef5 --- /dev/null +++ b/pkg/cloudprovider/drift.go @@ -0,0 +1,145 @@ +/* +Portions Copyright (c) Microsoft Corporation. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package cloudprovider + +import ( + "context" + "fmt" + "strings" + + "knative.dev/pkg/logging" + + "github.com/Azure/karpenter/pkg/apis/v1alpha2" + "github.com/Azure/karpenter/pkg/providers/imagefamily" + "github.com/Azure/karpenter/pkg/utils" + "github.com/samber/lo" + + v1 "k8s.io/api/core/v1" + + "sigs.k8s.io/controller-runtime/pkg/client" + + corev1beta1 "github.com/aws/karpenter-core/pkg/apis/v1beta1" + "github.com/aws/karpenter-core/pkg/cloudprovider" +) + +const ( + K8sVersionDrift cloudprovider.DriftReason = "K8sVersionDrift" + ImageVersionDrift cloudprovider.DriftReason = "ImageVersionDrift" +) + +func (c *CloudProvider) isK8sVersionDrifted(ctx context.Context, nodeClaim *corev1beta1.NodeClaim) (cloudprovider.DriftReason, error) { + logger := logging.FromContext(ctx) + + nodeName := nodeClaim.Status.NodeName + + if nodeName == "" { + return "", nil + } + + n := &v1.Node{} + if err := c.kubeClient.Get(ctx, client.ObjectKey{Name: nodeName}, n); err != nil { + // TODO (charliedmcb): should we ignore is not found errors? Will it ever be trying to check for drift before the node/vm exist? + return "", err + } + + k8sVersion, err := c.imageProvider.KubeServerVersion(ctx) + if err != nil { + return "", err + } + + nodeK8sVersion := strings.TrimPrefix(n.Status.NodeInfo.KubeletVersion, "v") + if nodeK8sVersion != k8sVersion { + logger.Debugf("drift triggered for %s, with expected k8s version %s, and actual k8s version %s", K8sVersionDrift, k8sVersion, nodeK8sVersion) + return K8sVersionDrift, nil + } + return "", nil +} + +// TODO (charliedmcb): remove nolint on gocyclo. Added for now in order to pass "make verify +// Was looking at a way to breakdown the function to pass gocyclo, but didn't feel like the best code. +// Feel reassessing this within the future with a potential minor refactor would be best to fix the gocyclo. +// nolint: gocyclo +func (c *CloudProvider) isImageVersionDrifted( + ctx context.Context, nodeClaim *corev1beta1.NodeClaim, nodeClass *v1alpha2.AKSNodeClass) (cloudprovider.DriftReason, error) { + logger := logging.FromContext(ctx) + + if !nodeClass.Spec.IsEmptyImageID() { + // Note: ImageID takes priority ATM + return "", nil + } + + id, err := utils.GetVMName(nodeClaim.Status.ProviderID) + if err != nil { + // TODO (charliedmcb): Do we need to handle vm not found here before its provisioned? + return "", err + } + + vm, err := c.instanceProvider.Get(ctx, id) + if err != nil { + // TODO (charliedmcb): Do we need to handle vm not found here before its provisioned? + return "", err + } + if vm == nil { + // TODO (charliedmcb): Do we need to handle vm not found here before its provisioned? + return "", fmt.Errorf("vm with id %s missing", id) + } + + if vm.Properties == nil || + vm.Properties.StorageProfile == nil || + vm.Properties.StorageProfile.ImageReference == nil || + vm.Properties.StorageProfile.ImageReference.CommunityGalleryImageID == nil || + *vm.Properties.StorageProfile.ImageReference.CommunityGalleryImageID == "" { + logger.Debug("not using a CommunityGalleryImageID for nodeClaim %s", nodeClaim.Name) + return "", nil + } + + vmImageID := *vm.Properties.StorageProfile.ImageReference.CommunityGalleryImageID + + publicGalleryURL, communityImageName, _, err := imagefamily.ParseCommunityImageIDInfo(vmImageID) + if err != nil { + return "", err + } + + expectedImageID, err := c.imageProvider.GetImageID(communityImageName, publicGalleryURL, nodeClass.Spec.GetImageVersion()) + if err != nil { + return "", err + } + + if vmImageID != expectedImageID { + logger.Debugf("drift triggered for %s, with expected image id %s, and actual image id %s", ImageVersionDrift, expectedImageID, vmImageID) + return ImageVersionDrift, nil + } + return "", nil +} + +// TODO: remove nolint on unparam. Added for now in order to pass "make verify" +// nolint: unparam +func (c *CloudProvider) isImageDrifted( + ctx context.Context, nodeClaim *corev1beta1.NodeClaim, nodePool *corev1beta1.NodePool, _ *v1alpha2.AKSNodeClass) (cloudprovider.DriftReason, error) { + instanceTypes, err := c.GetInstanceTypes(ctx, nodePool) + if err != nil { + return "", fmt.Errorf("getting instanceTypes, %w", err) + } + _, found := lo.Find(instanceTypes, func(instType *cloudprovider.InstanceType) bool { + return instType.Name == nodeClaim.Labels[v1.LabelInstanceTypeStable] + }) + if !found { + return "", fmt.Errorf(`finding node instance type "%s"`, nodeClaim.Labels[v1.LabelInstanceTypeStable]) + } + + return "", nil +} diff --git a/pkg/cloudprovider/events/events.go b/pkg/cloudprovider/events/events.go new file mode 100644 index 000000000..7e09a886a --- /dev/null +++ b/pkg/cloudprovider/events/events.go @@ -0,0 +1,42 @@ +/* +Portions Copyright (c) Microsoft Corporation. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package events + +import ( + v1 "k8s.io/api/core/v1" + + "github.com/aws/karpenter-core/pkg/apis/v1beta1" + "github.com/aws/karpenter-core/pkg/events" +) + +func NodePoolFailedToResolveNodeClass(nodePool *v1beta1.NodePool) events.Event { + return events.Event{ + InvolvedObject: nodePool, + Type: v1.EventTypeWarning, + Message: "Failed resolving AKSNodeClass", + DedupeValues: []string{string(nodePool.UID)}, + } +} + +func NodeClaimFailedToResolveNodeClass(nodeClaim *v1beta1.NodeClaim) events.Event { + return events.Event{ + InvolvedObject: nodeClaim, + Type: v1.EventTypeWarning, + Message: "Failed resolving AKSNodeClass", + DedupeValues: []string{string(nodeClaim.UID)}, + } +} diff --git a/pkg/cloudprovider/suite_test.go b/pkg/cloudprovider/suite_test.go new file mode 100644 index 000000000..821797c01 --- /dev/null +++ b/pkg/cloudprovider/suite_test.go @@ -0,0 +1,219 @@ +/* +Portions Copyright (c) Microsoft Corporation. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package cloudprovider + +// TODO v1beta1 extra refactor into suite_test.go / cloudprovider_test.go +import ( + "context" + "testing" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + "k8s.io/client-go/tools/record" + clock "k8s.io/utils/clock/testing" + + "github.com/Azure/karpenter/pkg/utils" + coresettings "github.com/aws/karpenter-core/pkg/apis/settings" + corev1beta1 "github.com/aws/karpenter-core/pkg/apis/v1beta1" + corecloudprovider "github.com/aws/karpenter-core/pkg/cloudprovider" + + "github.com/aws/karpenter-core/pkg/controllers/provisioning" + "github.com/aws/karpenter-core/pkg/controllers/state" + "github.com/aws/karpenter-core/pkg/events" + coreoptions "github.com/aws/karpenter-core/pkg/operator/options" + "github.com/aws/karpenter-core/pkg/operator/scheme" + coretest "github.com/aws/karpenter-core/pkg/test" + + . "knative.dev/pkg/logging/testing" + + "github.com/Azure/karpenter/pkg/apis" + "github.com/Azure/karpenter/pkg/apis/settings" + "github.com/Azure/karpenter/pkg/apis/v1alpha2" + "github.com/Azure/karpenter/pkg/providers/instance" + "github.com/Azure/karpenter/pkg/test" + . "github.com/aws/karpenter-core/pkg/test/expectations" +) + +var ctx context.Context +var stop context.CancelFunc +var env *coretest.Environment +var azureEnv *test.Environment +var fakeClock *clock.FakeClock +var coreProvisioner *provisioning.Provisioner + +var nodePool *corev1beta1.NodePool +var nodeClass *v1alpha2.AKSNodeClass +var nodeClaim *corev1beta1.NodeClaim +var cluster *state.Cluster +var cloudProvider *CloudProvider + +func TestCloudProvider(t *testing.T) { + ctx = TestContextWithLogger(t) + RegisterFailHandler(Fail) + RunSpecs(t, "CloudProvider") +} + +var _ = BeforeSuite(func() { + env = coretest.NewEnvironment(scheme.Scheme, coretest.WithCRDs(apis.CRDs...)) + ctx = coresettings.ToContext(ctx, coretest.Settings(coresettings.Settings{DriftEnabled: true})) + ctx = settings.ToContext(ctx, test.Settings()) + ctx, stop = context.WithCancel(ctx) + azureEnv = test.NewEnvironment(ctx, env) + + fakeClock = &clock.FakeClock{} + cloudProvider = New(azureEnv.InstanceTypesProvider, azureEnv.InstanceProvider, events.NewRecorder(&record.FakeRecorder{}), env.Client, azureEnv.ImageProvider) + cluster = state.NewCluster(fakeClock, env.Client, cloudProvider) + coreProvisioner = provisioning.NewProvisioner(env.Client, env.KubernetesInterface.CoreV1(), events.NewRecorder(&record.FakeRecorder{}), cloudProvider, cluster) +}) + +var _ = AfterSuite(func() { + stop() + Expect(env.Stop()).To(Succeed(), "Failed to stop environment") +}) + +var _ = BeforeEach(func() { + ctx = coreoptions.ToContext(ctx, coretest.Options()) + // TODO v1beta1 options + // ctx = options.ToContext(ctx, test.Options()) + ctx = settings.ToContext(ctx, test.Settings()) + nodeClass = test.AKSNodeClass() + nodePool = coretest.NodePool(corev1beta1.NodePool{ + Spec: corev1beta1.NodePoolSpec{ + Template: corev1beta1.NodeClaimTemplate{ + Spec: corev1beta1.NodeClaimSpec{ + NodeClassRef: &corev1beta1.NodeClassReference{ + Name: nodeClass.Name, + }, + }, + }, + }, + }) + nodeClaim = coretest.NodeClaim(corev1beta1.NodeClaim{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{corev1beta1.NodePoolLabelKey: nodePool.Name}, + }, + Spec: corev1beta1.NodeClaimSpec{ + NodeClassRef: &corev1beta1.NodeClassReference{ + Name: nodeClass.Name, + }, + }, + }) + + cluster.Reset() + azureEnv.Reset() +}) + +var _ = AfterEach(func() { + ExpectCleanedUp(ctx, env.Client) +}) + +var _ = Describe("CloudProvider", func() { + It("should list nodeclaim created by the CloudProvider", func() { + ExpectApplied(ctx, env.Client, nodeClass, nodePool) + pod := coretest.UnschedulablePod(coretest.PodOptions{}) + ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, coreProvisioner, pod) + ExpectScheduled(ctx, env.Client, pod) + Expect(azureEnv.VirtualMachinesAPI.VirtualMachineCreateOrUpdateBehavior.CalledWithInput.Len()).To(Equal(1)) + + nodeClaims, _ := cloudProvider.List(ctx) + Expect(azureEnv.AzureResourceGraphAPI.AzureResourceGraphResourcesBehavior.CalledWithInput.Len()).To(Equal(1)) + queryRequest := azureEnv.AzureResourceGraphAPI.AzureResourceGraphResourcesBehavior.CalledWithInput.Pop().Query + Expect(*queryRequest.Query).To(Equal(instance.GetListQueryBuilder(azureEnv.AzureResourceGraphAPI.ResourceGroup).String())) + Expect(nodeClaims).To(HaveLen(1)) + Expect(nodeClaims[0]).ToNot(BeNil()) + resp, _ := azureEnv.VirtualMachinesAPI.Get(ctx, azureEnv.AzureResourceGraphAPI.ResourceGroup, nodeClaims[0].Name, nil) + Expect(resp.VirtualMachine).ToNot(BeNil()) + }) + It("should return an ICE error when there are no instance types to launch", func() { + // Specify no instance types and expect to receive a capacity error + nodeClaim.Spec.Requirements = []v1.NodeSelectorRequirement{ + { + Key: v1.LabelInstanceTypeStable, + Operator: v1.NodeSelectorOpIn, + Values: []string{"doesnotexist"}, // will not match any instance types + }, + } + ExpectApplied(ctx, env.Client, nodePool, nodeClass, nodeClaim) + cloudProviderMachine, err := cloudProvider.Create(ctx, nodeClaim) + Expect(corecloudprovider.IsInsufficientCapacityError(err)).To(BeTrue()) + Expect(cloudProviderMachine).To(BeNil()) + }) + Context("Drift", func() { + var nodeClaim *corev1beta1.NodeClaim + var pod *v1.Pod + BeforeEach(func() { + instanceType := "Standard_D2_v2" + ExpectApplied(ctx, env.Client, nodePool, nodeClass) + pod = coretest.UnschedulablePod(coretest.PodOptions{ + NodeSelector: map[string]string{v1.LabelInstanceTypeStable: instanceType}, + }) + ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, coreProvisioner, pod) + ExpectScheduled(ctx, env.Client, pod) + + Expect(azureEnv.VirtualMachinesAPI.VirtualMachineCreateOrUpdateBehavior.CalledWithInput.Len()).To(Equal(1)) + input := azureEnv.VirtualMachinesAPI.VirtualMachineCreateOrUpdateBehavior.CalledWithInput.Pop() + rg := input.ResourceGroupName + vmName := input.VMName + + // Corresponding NodeClaim + nodeClaim = coretest.NodeClaim(corev1beta1.NodeClaim{ + Status: corev1beta1.NodeClaimStatus{ + ProviderID: utils.ResourceIDToProviderID(ctx, utils.MkVMID(rg, vmName)), + }, + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{ + corev1beta1.NodePoolLabelKey: nodePool.Name, + v1.LabelInstanceTypeStable: instanceType, + }, + }, + }) + }) + It("should not fail if nodeClass does not exist", func() { + ExpectDeleted(ctx, env.Client, nodeClass) + drifted, err := cloudProvider.IsDrifted(ctx, nodeClaim) + Expect(err).ToNot(HaveOccurred()) + Expect(drifted).To(BeEmpty()) + }) + It("should not fail if nodePool does not exist", func() { + ExpectDeleted(ctx, env.Client, nodePool) + drifted, err := cloudProvider.IsDrifted(ctx, nodeClaim) + Expect(err).ToNot(HaveOccurred()) + Expect(drifted).To(BeEmpty()) + }) + It("should not return drifted if the NodeClaim is valid", func() { + drifted, err := cloudProvider.IsDrifted(ctx, nodeClaim) + Expect(err).ToNot(HaveOccurred()) + Expect(drifted).To(BeEmpty()) + }) + It("should error if the NodeClaim doesn't have the instance-type label", func() { + nodeClaim.Labels = map[string]string{ + corev1beta1.NodePoolLabelKey: nodePool.Name, + } + _, err := cloudProvider.IsDrifted(ctx, nodeClaim) + Expect(err).To(HaveOccurred()) + }) + It("should error drift if NodeClaim doesn't have provider id", func() { + nodeClaim.Status = corev1beta1.NodeClaimStatus{} + drifted, err := cloudProvider.IsDrifted(ctx, nodeClaim) + Expect(err).To(HaveOccurred()) + Expect(drifted).To(BeEmpty()) + }) + }) +}) diff --git a/pkg/controllers/controllers.go b/pkg/controllers/controllers.go new file mode 100644 index 000000000..0517c4e17 --- /dev/null +++ b/pkg/controllers/controllers.go @@ -0,0 +1,43 @@ +/* +Portions Copyright (c) Microsoft Corporation. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package controllers + +import ( + "context" + + "github.com/aws/karpenter-core/pkg/operator/controller" + "knative.dev/pkg/logging" + "sigs.k8s.io/controller-runtime/pkg/client" + + "github.com/Azure/karpenter/pkg/cloudprovider" + nodeclaimgarbagecollection "github.com/Azure/karpenter/pkg/controllers/nodeclaim/garbagecollection" + "github.com/Azure/karpenter/pkg/controllers/nodeclaim/inplaceupdate" + nodeclaimlink "github.com/Azure/karpenter/pkg/controllers/nodeclaim/link" + "github.com/Azure/karpenter/pkg/providers/instance" + "github.com/Azure/karpenter/pkg/utils/project" +) + +func NewControllers(ctx context.Context, kubeClient client.Client, cloudProvider *cloudprovider.CloudProvider, instanceProvider *instance.Provider) []controller.Controller { + logging.FromContext(ctx).With("version", project.Version).Debugf("discovered version") + linkController := nodeclaimlink.NewController(kubeClient, cloudProvider) + controllers := []controller.Controller{ + nodeclaimgarbagecollection.NewController(kubeClient, cloudProvider, linkController), + linkController, + inplaceupdate.NewController(kubeClient, instanceProvider), + } + return controllers +} diff --git a/pkg/controllers/nodeclaim/garbagecollection/controller.go b/pkg/controllers/nodeclaim/garbagecollection/controller.go new file mode 100644 index 000000000..a576da8eb --- /dev/null +++ b/pkg/controllers/nodeclaim/garbagecollection/controller.go @@ -0,0 +1,126 @@ +/* +Portions Copyright (c) Microsoft Corporation. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package garbagecollection + +import ( + "context" + "fmt" + "time" + + "github.com/Azure/karpenter/pkg/cloudprovider" + "github.com/samber/lo" + "go.uber.org/multierr" + v1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/util/sets" + "k8s.io/client-go/util/workqueue" + "knative.dev/pkg/logging" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/manager" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + + "github.com/aws/karpenter-core/pkg/apis/v1alpha5" + corev1beta1 "github.com/aws/karpenter-core/pkg/apis/v1beta1" + + link "github.com/Azure/karpenter/pkg/controllers/nodeclaim/link" + corecloudprovider "github.com/aws/karpenter-core/pkg/cloudprovider" + "github.com/aws/karpenter-core/pkg/operator/controller" +) + +type Controller struct { + kubeClient client.Client + cloudProvider *cloudprovider.CloudProvider + successfulCount uint64 // keeps track of successful reconciles for more aggressive requeueing near the start of the controller + linkController *link.Controller +} + +func NewController(kubeClient client.Client, cloudProvider *cloudprovider.CloudProvider, linkController *link.Controller) *Controller { + return &Controller{ + kubeClient: kubeClient, + cloudProvider: cloudProvider, + successfulCount: 0, + linkController: linkController, + } +} + +func (c *Controller) Name() string { + return "nodeclaim.garbagecollection" +} + +func (c *Controller) Reconcile(ctx context.Context, _ reconcile.Request) (reconcile.Result, error) { + // We LIST VMs on the CloudProvider BEFORE we grab NodeClaims/Nodes on the cluster so that we make sure that, if + // LISTing instances takes a long time, our information is more updated by the time we get to nodeclaim and Node LIST + // This works since our CloudProvider instances are deleted based on whether the NodeClaim exists or not, not vice-versa + retrieved, err := c.cloudProvider.List(ctx) + if err != nil { + return reconcile.Result{}, fmt.Errorf("listing cloudprovider VMs, %w", err) + } + managedRetrieved := lo.Filter(retrieved, func(m *corev1beta1.NodeClaim, _ int) bool { + return m.DeletionTimestamp.IsZero() + }) + nodeClaims := &corev1beta1.NodeClaimList{} + if err := c.kubeClient.List(ctx, nodeClaims); err != nil { + return reconcile.Result{}, err + } + nodeList := &v1.NodeList{} + if err := c.kubeClient.List(ctx, nodeList); err != nil { + return reconcile.Result{}, err + } + resolvedNodeClaims := lo.Filter(nodeClaims.Items, func(m corev1beta1.NodeClaim, _ int) bool { + return m.Status.ProviderID != "" || m.Annotations[v1alpha5.MachineLinkedAnnotationKey] != "" + }) + resolvedProviderIDs := sets.New[string](lo.Map(resolvedNodeClaims, func(m corev1beta1.NodeClaim, _ int) string { + if m.Status.ProviderID != "" { + return m.Status.ProviderID + } + return m.Annotations[v1alpha5.MachineLinkedAnnotationKey] + })...) + errs := make([]error, len(retrieved)) + workqueue.ParallelizeUntil(ctx, 100, len(managedRetrieved), func(i int) { + _, recentlyLinked := c.linkController.Cache.Get(managedRetrieved[i].Status.ProviderID) + + if !recentlyLinked && + !resolvedProviderIDs.Has(managedRetrieved[i].Status.ProviderID) && + time.Since(managedRetrieved[i].CreationTimestamp.Time) > time.Minute*5 { + errs[i] = c.garbageCollect(ctx, managedRetrieved[i], nodeList) + } + }) + c.successfulCount++ + return reconcile.Result{RequeueAfter: lo.Ternary(c.successfulCount <= 20, time.Second*10, time.Minute*2)}, multierr.Combine(errs...) +} + +func (c *Controller) garbageCollect(ctx context.Context, nodeClaim *corev1beta1.NodeClaim, nodeList *v1.NodeList) error { + ctx = logging.WithLogger(ctx, logging.FromContext(ctx).With("provider-id", nodeClaim.Status.ProviderID)) + if err := c.cloudProvider.Delete(ctx, nodeClaim); err != nil { + return corecloudprovider.IgnoreNodeClaimNotFoundError(err) + } + logging.FromContext(ctx).Debugf("garbage collected cloudprovider instance") + + // Go ahead and cleanup the node if we know that it exists to make scheduling go quicker + if node, ok := lo.Find(nodeList.Items, func(n v1.Node) bool { + return n.Spec.ProviderID == nodeClaim.Status.ProviderID + }); ok { + if err := c.kubeClient.Delete(ctx, &node); err != nil { + return client.IgnoreNotFound(err) + } + logging.FromContext(ctx).With("node", node.Name).Debugf("garbage collected node") + } + return nil +} + +func (c *Controller) Builder(_ context.Context, m manager.Manager) controller.Builder { + return controller.NewSingletonManagedBy(m) +} diff --git a/pkg/controllers/nodeclaim/garbagecollection/suite_test.go b/pkg/controllers/nodeclaim/garbagecollection/suite_test.go new file mode 100644 index 000000000..a81c83b28 --- /dev/null +++ b/pkg/controllers/nodeclaim/garbagecollection/suite_test.go @@ -0,0 +1,342 @@ +/* +Portions Copyright (c) Microsoft Corporation. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package garbagecollection_test + +import ( + "context" + "fmt" + "sync" + "testing" + "time" + + "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute" + "github.com/Azure/karpenter/pkg/apis" + "github.com/Azure/karpenter/pkg/apis/settings" + "github.com/Azure/karpenter/pkg/apis/v1alpha2" + "github.com/Azure/karpenter/pkg/cloudprovider" + "github.com/Azure/karpenter/pkg/controllers/nodeclaim/garbagecollection" + link "github.com/Azure/karpenter/pkg/controllers/nodeclaim/link" + "github.com/Azure/karpenter/pkg/providers/instance" + "github.com/Azure/karpenter/pkg/utils" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "github.com/patrickmn/go-cache" + "github.com/samber/lo" + "k8s.io/client-go/tools/record" + clock "k8s.io/utils/clock/testing" + . "knative.dev/pkg/logging/testing" + "sigs.k8s.io/controller-runtime/pkg/client" + + "github.com/Azure/karpenter/pkg/test" + corecloudprovider "github.com/aws/karpenter-core/pkg/cloudprovider" + "github.com/aws/karpenter-core/pkg/controllers/state" + "github.com/aws/karpenter-core/pkg/events" + "github.com/aws/karpenter-core/pkg/operator/controller" + coreoptions "github.com/aws/karpenter-core/pkg/operator/options" + "github.com/aws/karpenter-core/pkg/operator/scheme" + coretest "github.com/aws/karpenter-core/pkg/test" + . "github.com/aws/karpenter-core/pkg/test/expectations" + + corev1beta1 "github.com/aws/karpenter-core/pkg/apis/v1beta1" +) + +var ctx context.Context +var stop context.CancelFunc +var env *coretest.Environment +var azureEnv *test.Environment +var fakeClock *clock.FakeClock +var nodePool *corev1beta1.NodePool +var nodeClass *v1alpha2.AKSNodeClass +var cluster *state.Cluster +var cloudProvider *cloudprovider.CloudProvider +var garbageCollectionController controller.Controller +var linkedNodeClaimCache *cache.Cache + +func TestAPIs(t *testing.T) { + ctx = TestContextWithLogger(t) + RegisterFailHandler(Fail) + RunSpecs(t, "NodeClaim") +} + +var _ = BeforeSuite(func() { + ctx = coreoptions.ToContext(ctx, coretest.Options()) + // TODO v1beta1 we dont have options yet + // ctx = options.ToContext(ctx, test.Options()) + ctx = settings.ToContext(ctx, test.Settings()) + + env = coretest.NewEnvironment(scheme.Scheme, coretest.WithCRDs(apis.CRDs...)) + ctx, stop = context.WithCancel(ctx) + azureEnv = test.NewEnvironment(ctx, env) + cloudProvider = cloudprovider.New(azureEnv.InstanceTypesProvider, azureEnv.InstanceProvider, events.NewRecorder(&record.FakeRecorder{}), env.Client, azureEnv.ImageProvider) + linkedNodeClaimCache = cache.New(time.Minute*10, time.Second*10) + linkController := &link.Controller{ + Cache: linkedNodeClaimCache, + } + garbageCollectionController = garbagecollection.NewController(env.Client, cloudProvider, linkController) + fakeClock = &clock.FakeClock{} + cluster = state.NewCluster(fakeClock, env.Client, cloudProvider) +}) + +var _ = AfterSuite(func() { + stop() + Expect(env.Stop()).To(Succeed(), "Failed to stop environment") +}) + +var _ = BeforeEach(func() { + nodeClass = test.AKSNodeClass() + nodePool = coretest.NodePool(corev1beta1.NodePool{ + Spec: corev1beta1.NodePoolSpec{ + Template: corev1beta1.NodeClaimTemplate{ + Spec: corev1beta1.NodeClaimSpec{ + NodeClassRef: &corev1beta1.NodeClassReference{ + Name: nodeClass.Name, + }, + }, + }, + }, + }) + + cluster.Reset() + azureEnv.Reset() +}) + +var _ = AfterEach(func() { + ExpectCleanedUp(ctx, env.Client) + linkedNodeClaimCache.Flush() +}) + +var _ = Describe("NodeClaimGarbageCollection", func() { + var vm armcompute.VirtualMachine + var providerID string + + BeforeEach(func() { + id := utils.MkVMID(azureEnv.AzureResourceGraphAPI.ResourceGroup, "vm-a") + vm = armcompute.VirtualMachine{ + ID: lo.ToPtr(id), + Name: lo.ToPtr("vm-a"), + Tags: map[string]*string{ + instance.NodePoolTagKey: lo.ToPtr("default"), + }, + } + providerID = utils.ResourceIDToProviderID(ctx, id) + }) + + It("should delete an instance if there is no NodeClaim owner", func() { + // Launch happened 10m ago + vm.Properties = &armcompute.VirtualMachineProperties{ + TimeCreated: lo.ToPtr(time.Now().Add(-time.Minute * 10)), + } + azureEnv.VirtualMachinesAPI.Instances.Store(lo.FromPtr(vm.ID), vm) + + ExpectReconcileSucceeded(ctx, garbageCollectionController, client.ObjectKey{}) + _, err := cloudProvider.Get(ctx, providerID) + Expect(err).To(HaveOccurred()) + Expect(corecloudprovider.IsNodeClaimNotFoundError(err)).To(BeTrue()) + }) + It("should not delete an instance if it was not launched by a NodeClaim", func() { + // Remove the "karpenter.sh/managed-by" tag (this isn't launched by a NodeClaim) + vm.Properties = &armcompute.VirtualMachineProperties{ + TimeCreated: lo.ToPtr(time.Now().Add(-time.Minute * 10)), + } + vm.Tags = lo.OmitBy(vm.Tags, func(key string, value *string) bool { + return key == instance.NodePoolTagKey + }) + azureEnv.VirtualMachinesAPI.Instances.Store(lo.FromPtr(vm.ID), vm) + + ExpectReconcileSucceeded(ctx, garbageCollectionController, client.ObjectKey{}) + _, err := cloudProvider.Get(ctx, providerID) + Expect(err).NotTo(HaveOccurred()) + }) + It("should delete an instance along with the node if there is no NodeClaim owner (to quicken scheduling)", func() { + // Launch happened 10m ago + vm.Properties = &armcompute.VirtualMachineProperties{ + TimeCreated: lo.ToPtr(time.Now().Add(-time.Minute * 10)), + } + azureEnv.VirtualMachinesAPI.Instances.Store(lo.FromPtr(vm.ID), vm) + + node := coretest.Node(coretest.NodeOptions{ + ProviderID: providerID, + }) + ExpectApplied(ctx, env.Client, node) + + ExpectReconcileSucceeded(ctx, garbageCollectionController, client.ObjectKey{}) + _, err := cloudProvider.Get(ctx, providerID) + Expect(err).To(HaveOccurred()) + Expect(corecloudprovider.IsNodeClaimNotFoundError(err)).To(BeTrue()) + + ExpectNotFound(ctx, env.Client, node) + }) + It("should delete many instances if they all don't have NodeClaim owners", func() { + // Generate 100 instances that have different vmIDs + var ids []string + var vmName string + var vmID string + for i := 0; i < 100; i++ { + vmName = fmt.Sprintf("vm-%d", i) + vmID = utils.MkVMID(azureEnv.AzureResourceGraphAPI.ResourceGroup, vmName) + azureEnv.VirtualMachinesAPI.Instances.Store( + vmID, + armcompute.VirtualMachine{ + ID: lo.ToPtr(utils.MkVMID(azureEnv.AzureResourceGraphAPI.ResourceGroup, vmName)), + Name: lo.ToPtr(vmName), + Properties: &armcompute.VirtualMachineProperties{ + TimeCreated: lo.ToPtr(time.Now().Add(-time.Minute * 10)), + }, + Tags: map[string]*string{ + instance.NodePoolTagKey: lo.ToPtr("default"), + }, + }) + ids = append(ids, vmID) + } + ExpectReconcileSucceeded(ctx, garbageCollectionController, client.ObjectKey{}) + + wg := sync.WaitGroup{} + for _, id := range ids { + wg.Add(1) + go func(id string) { + defer GinkgoRecover() + defer wg.Done() + + _, err := cloudProvider.Get(ctx, utils.ResourceIDToProviderID(ctx, id)) + Expect(err).To(HaveOccurred()) + Expect(corecloudprovider.IsNodeClaimNotFoundError(err)).To(BeTrue()) + }(id) + } + wg.Wait() + }) + It("should not delete all instances if they all have NodeClaim owners", func() { + // Generate 100 instances that have different instanceIDs + var ids []string + var nodeClaims []*corev1beta1.NodeClaim + var vmName string + var vmID string + for i := 0; i < 100; i++ { + vmName = fmt.Sprintf("vm-%d", i) + vmID = utils.MkVMID(azureEnv.AzureResourceGraphAPI.ResourceGroup, vmName) + azureEnv.VirtualMachinesAPI.Instances.Store( + vmID, + armcompute.VirtualMachine{ + ID: lo.ToPtr(utils.MkVMID(azureEnv.AzureResourceGraphAPI.ResourceGroup, vmName)), + Name: lo.ToPtr(vmName), + Properties: &armcompute.VirtualMachineProperties{ + TimeCreated: lo.ToPtr(time.Now().Add(-time.Minute * 10)), + }, + Tags: map[string]*string{ + instance.NodePoolTagKey: lo.ToPtr("default"), + }, + }, + ) + nodeClaim := coretest.NodeClaim(corev1beta1.NodeClaim{ + Status: corev1beta1.NodeClaimStatus{ + ProviderID: utils.ResourceIDToProviderID(ctx, vmID), + }, + }) + ExpectApplied(ctx, env.Client, nodeClaim) + nodeClaims = append(nodeClaims, nodeClaim) + ids = append(ids, vmID) + } + ExpectReconcileSucceeded(ctx, garbageCollectionController, client.ObjectKey{}) + + wg := sync.WaitGroup{} + for _, id := range ids { + wg.Add(1) + go func(id string) { + defer GinkgoRecover() + defer wg.Done() + + _, err := cloudProvider.Get(ctx, utils.ResourceIDToProviderID(ctx, id)) + Expect(err).ToNot(HaveOccurred()) + }(id) + } + wg.Wait() + + for _, nodeClaim := range nodeClaims { + ExpectExists(ctx, env.Client, nodeClaim) + } + }) + It("should not delete an instance if it is within the nodeClaim resolution window (5m)", func() { + // Launch time just happened + vm.Properties = &armcompute.VirtualMachineProperties{ + TimeCreated: lo.ToPtr(time.Now()), + } + azureEnv.VirtualMachinesAPI.Instances.Store(lo.FromPtr(vm.ID), vm) + + ExpectReconcileSucceeded(ctx, garbageCollectionController, client.ObjectKey{}) + _, err := cloudProvider.Get(ctx, providerID) + Expect(err).NotTo(HaveOccurred()) + }) + It("should not delete the instance or node if it already has a nodeClaim that matches it", func() { + // Launch time was 10m ago + vm.Properties = &armcompute.VirtualMachineProperties{ + TimeCreated: lo.ToPtr(time.Now().Add(-time.Minute * 10)), + } + azureEnv.VirtualMachinesAPI.Instances.Store(lo.FromPtr(vm.ID), vm) + + nodeClaim := coretest.NodeClaim(corev1beta1.NodeClaim{ + Status: corev1beta1.NodeClaimStatus{ + ProviderID: providerID, + }, + }) + node := coretest.Node(coretest.NodeOptions{ + ProviderID: providerID, + }) + ExpectApplied(ctx, env.Client, nodeClaim, node) + + ExpectReconcileSucceeded(ctx, garbageCollectionController, client.ObjectKey{}) + _, err := cloudProvider.Get(ctx, providerID) + Expect(err).ToNot(HaveOccurred()) + ExpectExists(ctx, env.Client, node) + }) + /* TODO v1beta1 is this gone with v1beta1? rachel + It("should not delete an instance if it is linked", func() { + // Launch time was 1m ago + vm.Properties = &armcompute.VirtualMachineProperties{ + TimeCreated: lo.ToPtr(time.Now().Add(-time.Minute)), + } + azureEnv.VirtualMachinesAPI.Instances.Store(lo.FromPtr(vm.ID), vm) + + // Create a nodeClaim that is actively linking + nodeClaim := coretest.NodeClaim(corev1beta1.NodeClaim{ + ObjectMeta: metav1.ObjectMeta{ + Annotations: map[string]string{ + corev1beta1.NodeClaimLinkedAnnotationKey: providerID, + }, + }, + }) + nodeClaim.Status.ProviderID = "" + ExpectApplied(ctx, env.Client, nodeClaim) + + ExpectReconcileSucceeded(ctx, garbageCollectionController, client.ObjectKey{}) + _, err := cloudProvider.Get(ctx, providerID) + Expect(err).NotTo(HaveOccurred()) + }) + */ + It("should not delete an instance if it is recently linked but the nodeClaim doesn't exist", func() { + // Launch time was 1m ago + vm.Properties = &armcompute.VirtualMachineProperties{ + TimeCreated: lo.ToPtr(time.Now().Add(-time.Minute)), + } + azureEnv.VirtualMachinesAPI.Instances.Store(lo.FromPtr(vm.ID), vm) + + // Add a provider id to the recently linked cache + linkedNodeClaimCache.SetDefault(providerID, nil) + + ExpectReconcileSucceeded(ctx, garbageCollectionController, client.ObjectKey{}) + _, err := cloudProvider.Get(ctx, providerID) + Expect(err).NotTo(HaveOccurred()) + }) +}) diff --git a/pkg/controllers/nodeclaim/inplaceupdate/controller.go b/pkg/controllers/nodeclaim/inplaceupdate/controller.go new file mode 100644 index 000000000..3d1cc1cdf --- /dev/null +++ b/pkg/controllers/nodeclaim/inplaceupdate/controller.go @@ -0,0 +1,186 @@ +/* +Portions Copyright (c) Microsoft Corporation. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package inplaceupdate + +import ( + "context" + "encoding/json" + "fmt" + + "github.com/aws/karpenter-core/pkg/apis/v1beta1" + corecontroller "github.com/aws/karpenter-core/pkg/operator/controller" + "github.com/samber/lo" + "go.uber.org/zap/zapcore" + "knative.dev/pkg/logging" + controllerruntime "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/builder" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller" + "sigs.k8s.io/controller-runtime/pkg/manager" + "sigs.k8s.io/controller-runtime/pkg/predicate" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + + "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute" + "github.com/Azure/karpenter/pkg/apis/settings" + "github.com/Azure/karpenter/pkg/apis/v1alpha2" + "github.com/Azure/karpenter/pkg/providers/instance" + "github.com/Azure/karpenter/pkg/utils" +) + +type Controller struct { + kubeClient client.Client + instanceProvider *instance.Provider +} + +var _ corecontroller.TypedController[*v1beta1.NodeClaim] = &Controller{} + +func NewController( + kubeClient client.Client, + instanceProvider *instance.Provider, +) corecontroller.Controller { + controller := &Controller{ + kubeClient: kubeClient, + instanceProvider: instanceProvider, + } + + return corecontroller.Typed[*v1beta1.NodeClaim](kubeClient, controller) +} + +func (c *Controller) Name() string { + return "nodeclaim.inplaceupdate" +} + +func (c *Controller) Reconcile(ctx context.Context, nodeClaim *v1beta1.NodeClaim) (reconcile.Result, error) { + if !nodeClaim.DeletionTimestamp.IsZero() { + return reconcile.Result{}, nil + } + + // Node doesn't have provider ID yet + if nodeClaim.Status.ProviderID == "" { + return reconcile.Result{}, nil + } + + stored := nodeClaim.DeepCopy() + + // TODO: When we have sources of truth coming from NodePool we can do: + // nodePool, err := nodeclaimutil.Owner(ctx, c.kubeClient, nodeClaim) + // TODO: To look it up and use that as input to calculate the goal state as well + + // Compare the expected hash with the actual hash + settings := settings.FromContext(ctx) + goalHash, err := HashFromNodeClaim(settings, nodeClaim) + if err != nil { + return reconcile.Result{}, err + } + actualHash := nodeClaim.Annotations[v1alpha2.AnnotationInPlaceUpdateHash] + + logging.FromContext(ctx).Debugf("goal hash is: %q, actual hash is: %q", goalHash, actualHash) + + // If there's no difference from goal state, no need to do anything else + if goalHash == actualHash { + return reconcile.Result{}, nil + } + + vmName, err := utils.GetVMName(nodeClaim.Status.ProviderID) + if err != nil { + return reconcile.Result{}, err + } + + vm, err := c.instanceProvider.Get(ctx, vmName) + if err != nil { + return reconcile.Result{}, fmt.Errorf("getting azure VM for machine, %w", err) + } + + update := calculateVMPatch(settings, vm) + // This is safe only as long as we're not updating fields which we consider secret. + // If we do/are, we need to redact them. + logVMPatch(ctx, update) + + // Apply the update, if one is needed + if update != nil { + err = c.instanceProvider.Update(ctx, vmName, *update) + if err != nil { + return reconcile.Result{}, fmt.Errorf("failed to apply update to VM, %w", err) + } + } + + if nodeClaim.Annotations == nil { + nodeClaim.Annotations = make(map[string]string) + } + // Regardless of whether we actually changed anything in Azure, we have confirmed that + // the goal shape is in alignment with our expected shape, so update the annotation to reflect that + nodeClaim.Annotations[v1alpha2.AnnotationInPlaceUpdateHash] = goalHash + err = c.kubeClient.Patch(ctx, nodeClaim, client.MergeFrom(stored)) + if err != nil { + return reconcile.Result{}, client.IgnoreNotFound(err) + } + + return reconcile.Result{}, nil +} + +func calculateVMPatch( + settings *settings.Settings, + // TODO: Can pass and consider NodeClaim and/or NodePool here if we need to in the future + currentVM *armcompute.VirtualMachine, +) *armcompute.VirtualMachineUpdate { + // Determine the differences between the current state and the goal state + expectedIdentities := settings.NodeIdentities + var currentIdentities []string + if currentVM.Identity != nil { + currentIdentities = lo.Keys(currentVM.Identity.UserAssignedIdentities) + } + + toAdd, _ := lo.Difference(expectedIdentities, currentIdentities) + // It's not possible to PATCH identities away, so for now we never remove them even if they've been removed from + // the configmap. This matches the RPs behavior and also ensures that we don't remove identities which users have + // manually added. + + if len(toAdd) == 0 { + return nil // No update to perform + } + + identity := instance.ConvertToVirtualMachineIdentity(toAdd) + + return &armcompute.VirtualMachineUpdate{ + Identity: identity, + } +} + +func (c *Controller) Builder(_ context.Context, m manager.Manager) corecontroller.Builder { + return corecontroller.Adapt(controllerruntime.NewControllerManagedBy(m).For( + &v1beta1.NodeClaim{}, + builder.WithPredicates( + predicate.Or( + predicate.GenerationChangedPredicate{}, // Note that this will trigger on pod restart for all Machines. + ), + )).WithOptions(controller.Options{MaxConcurrentReconciles: 10}), + // TODO: Can add .Watches(&v1beta1.NodePool{}, nodeclaimutil.NodePoolEventHandler(c.kubeClient)) + // TODO: similar to https://github.com/aws/karpenter-core/blob/main/pkg/controllers/nodeclaim/disruption/controller.go#L214C3-L217C5 + // TODO: if/when we need to monitor provisoner changes and flow updates on the NodePool down to the underlying VMs. + ) +} + +func logVMPatch(ctx context.Context, update *armcompute.VirtualMachineUpdate) { + if logging.FromContext(ctx).Level().Enabled(zapcore.DebugLevel) { + rawStr := "" + if update != nil { + raw, _ := json.Marshal(update) + rawStr = string(raw) + } + logging.FromContext(ctx).Debugf("applying patch to Azure VM: %s", rawStr) + } +} diff --git a/pkg/controllers/nodeclaim/inplaceupdate/suite_test.go b/pkg/controllers/nodeclaim/inplaceupdate/suite_test.go new file mode 100644 index 000000000..42726ebc3 --- /dev/null +++ b/pkg/controllers/nodeclaim/inplaceupdate/suite_test.go @@ -0,0 +1,372 @@ +/* +Portions Copyright (c) Microsoft Corporation. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package inplaceupdate + +import ( + "context" + "testing" + + "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute" + corev1beta1 "github.com/aws/karpenter-core/pkg/apis/v1beta1" + corecontroller "github.com/aws/karpenter-core/pkg/operator/controller" + coreoptions "github.com/aws/karpenter-core/pkg/operator/options" + "github.com/aws/karpenter-core/pkg/operator/scheme" + coretest "github.com/aws/karpenter-core/pkg/test" + . "github.com/aws/karpenter-core/pkg/test/expectations" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "github.com/samber/lo" + . "knative.dev/pkg/logging/testing" + "sigs.k8s.io/controller-runtime/pkg/client" + + "github.com/Azure/karpenter/pkg/apis" + "github.com/Azure/karpenter/pkg/apis/settings" + "github.com/Azure/karpenter/pkg/apis/v1alpha2" + "github.com/Azure/karpenter/pkg/test" + "github.com/Azure/karpenter/pkg/utils" +) + +var ctx context.Context +var stop context.CancelFunc +var env *coretest.Environment +var azureEnv *test.Environment +var inPlaceUpdateController corecontroller.Controller + +func TestInPlaceUpdate(t *testing.T) { + ctx = TestContextWithLogger(t) + RegisterFailHandler(Fail) + RunSpecs(t, "Controllers/InPlaceUpdate") +} + +var _ = BeforeSuite(func() { + ctx = coreoptions.ToContext(ctx, coretest.Options()) + + env = coretest.NewEnvironment(scheme.Scheme, coretest.WithCRDs(apis.CRDs...)) + + ctx, stop = context.WithCancel(ctx) + azureEnv = test.NewEnvironment(ctx, env) + + inPlaceUpdateController = NewController(env.Client, azureEnv.InstanceProvider) +}) + +var _ = AfterSuite(func() { + stop() + Expect(env.Stop()).To(Succeed(), "Failed to stop environment") +}) + +var _ = Describe("Unit tests", func() { + Context("HashFromVM", func() { + It("should not depend on identity ordering", func() { + vm1 := &armcompute.VirtualMachine{ + Identity: &armcompute.VirtualMachineIdentity{ + UserAssignedIdentities: map[string]*armcompute.UserAssignedIdentitiesValue{ + "/subscriptions/1234/resourceGroups/mcrg/providers/Microsoft.ManagedIdentity/userAssignedIdentities/myid1": {}, + "/subscriptions/1234/resourceGroups/mcrg/providers/Microsoft.ManagedIdentity/userAssignedIdentities/myid2": {}, + "/subscriptions/1234/resourceGroups/mcrg/providers/Microsoft.ManagedIdentity/userAssignedIdentities/myid3": {}, + }, + }, + } + + vm2 := &armcompute.VirtualMachine{ + Identity: &armcompute.VirtualMachineIdentity{ + UserAssignedIdentities: map[string]*armcompute.UserAssignedIdentitiesValue{ + "/subscriptions/1234/resourceGroups/mcrg/providers/Microsoft.ManagedIdentity/userAssignedIdentities/myid2": {}, + "/subscriptions/1234/resourceGroups/mcrg/providers/Microsoft.ManagedIdentity/userAssignedIdentities/myid1": {}, + "/subscriptions/1234/resourceGroups/mcrg/providers/Microsoft.ManagedIdentity/userAssignedIdentities/myid3": {}, + }, + }, + } + + vm3 := &armcompute.VirtualMachine{ + Identity: &armcompute.VirtualMachineIdentity{ + UserAssignedIdentities: map[string]*armcompute.UserAssignedIdentitiesValue{ + "/subscriptions/1234/resourceGroups/mcrg/providers/Microsoft.ManagedIdentity/userAssignedIdentities/myid3": {}, + "/subscriptions/1234/resourceGroups/mcrg/providers/Microsoft.ManagedIdentity/userAssignedIdentities/myid2": {}, + "/subscriptions/1234/resourceGroups/mcrg/providers/Microsoft.ManagedIdentity/userAssignedIdentities/myid1": {}, + }, + }, + } + + hash1, err := HashFromVM(vm1) + Expect(err).ToNot(HaveOccurred()) + + hash2, err := HashFromVM(vm2) + Expect(err).ToNot(HaveOccurred()) + + hash3, err := HashFromVM(vm3) + Expect(err).ToNot(HaveOccurred()) + + Expect(hash1).To(Equal(hash2)) + Expect(hash2).To(Equal(hash3)) + }) + }) + + Context("HashFromNodeClaim", func() { + It("should not depend on identity ordering", func() { + settings := test.Settings() + settings.NodeIdentities = []string{ + "/subscriptions/1234/resourceGroups/mcrg/providers/Microsoft.ManagedIdentity/userAssignedIdentities/myid1", + "/subscriptions/1234/resourceGroups/mcrg/providers/Microsoft.ManagedIdentity/userAssignedIdentities/myid2", + "/subscriptions/1234/resourceGroups/mcrg/providers/Microsoft.ManagedIdentity/userAssignedIdentities/myid3", + } + + hash1, err := HashFromNodeClaim(settings, nil) + Expect(err).ToNot(HaveOccurred()) + + settings.NodeIdentities = []string{ + "/subscriptions/1234/resourceGroups/mcrg/providers/Microsoft.ManagedIdentity/userAssignedIdentities/myid2", + "/subscriptions/1234/resourceGroups/mcrg/providers/Microsoft.ManagedIdentity/userAssignedIdentities/myid1", + "/subscriptions/1234/resourceGroups/mcrg/providers/Microsoft.ManagedIdentity/userAssignedIdentities/myid3", + } + hash2, err := HashFromNodeClaim(settings, nil) + Expect(err).ToNot(HaveOccurred()) + + settings.NodeIdentities = []string{ + "/subscriptions/1234/resourceGroups/mcrg/providers/Microsoft.ManagedIdentity/userAssignedIdentities/myid3", + "/subscriptions/1234/resourceGroups/mcrg/providers/Microsoft.ManagedIdentity/userAssignedIdentities/myid2", + "/subscriptions/1234/resourceGroups/mcrg/providers/Microsoft.ManagedIdentity/userAssignedIdentities/myid1", + } + hash3, err := HashFromNodeClaim(settings, nil) + Expect(err).ToNot(HaveOccurred()) + + Expect(hash1).To(Equal(hash2)) + Expect(hash2).To(Equal(hash3)) + }) + }) + + Context("calculateVMPatch", func() { + It("should add missing identities when there are no existing identities", func() { + currentVM := &armcompute.VirtualMachine{} + + settings := test.Settings() + settings.NodeIdentities = []string{ + "/subscriptions/1234/resourceGroups/mcrg/providers/Microsoft.ManagedIdentity/userAssignedIdentities/myid1", + } + update := calculateVMPatch(settings, currentVM) + + Expect(update).ToNot(BeNil()) + Expect(update.Identity).ToNot(BeNil()) + Expect(update.Identity.UserAssignedIdentities).To(HaveLen(1)) + Expect(update.Identity.UserAssignedIdentities).To(HaveKey("/subscriptions/1234/resourceGroups/mcrg/providers/Microsoft.ManagedIdentity/userAssignedIdentities/myid1")) + }) + + It("should add missing identities when there are existing identities", func() { + currentVM := &armcompute.VirtualMachine{ + Identity: &armcompute.VirtualMachineIdentity{ + UserAssignedIdentities: map[string]*armcompute.UserAssignedIdentitiesValue{ + "/subscriptions/1234/resourceGroups/mcrg/providers/Microsoft.ManagedIdentity/userAssignedIdentities/myid1": {}, + }, + }, + } + + settings := test.Settings() + settings.NodeIdentities = []string{ + "/subscriptions/1234/resourceGroups/mcrg/providers/Microsoft.ManagedIdentity/userAssignedIdentities/myid2", + } + update := calculateVMPatch(settings, currentVM) + + Expect(update).ToNot(BeNil()) + Expect(update.Identity).ToNot(BeNil()) + Expect(update.Identity.UserAssignedIdentities).To(HaveLen(1)) + Expect(update.Identity.UserAssignedIdentities).To(HaveKey("/subscriptions/1234/resourceGroups/mcrg/providers/Microsoft.ManagedIdentity/userAssignedIdentities/myid2")) + }) + + It("should add no identities when identities already exist", func() { + currentVM := &armcompute.VirtualMachine{ + Identity: &armcompute.VirtualMachineIdentity{ + UserAssignedIdentities: map[string]*armcompute.UserAssignedIdentitiesValue{ + "/subscriptions/1234/resourceGroups/mcrg/providers/Microsoft.ManagedIdentity/userAssignedIdentities/myid1": {}, + "/subscriptions/1234/resourceGroups/mcrg/providers/Microsoft.ManagedIdentity/userAssignedIdentities/myid2": {}, + }, + }, + } + + settings := test.Settings() + settings.NodeIdentities = []string{ + "/subscriptions/1234/resourceGroups/mcrg/providers/Microsoft.ManagedIdentity/userAssignedIdentities/myid2", + "/subscriptions/1234/resourceGroups/mcrg/providers/Microsoft.ManagedIdentity/userAssignedIdentities/myid1", + } + update := calculateVMPatch(settings, currentVM) + + Expect(update).To(BeNil()) + }) + + It("should not remove identities", func() { + currentVM := &armcompute.VirtualMachine{ + Identity: &armcompute.VirtualMachineIdentity{ + UserAssignedIdentities: map[string]*armcompute.UserAssignedIdentitiesValue{ + "/subscriptions/1234/resourceGroups/mcrg/providers/Microsoft.ManagedIdentity/userAssignedIdentities/myid1": {}, + "/subscriptions/1234/resourceGroups/mcrg/providers/Microsoft.ManagedIdentity/userAssignedIdentities/myotheridentity": {}, + }, + }, + } + + settings := test.Settings() + settings.NodeIdentities = []string{ + "/subscriptions/1234/resourceGroups/mcrg/providers/Microsoft.ManagedIdentity/userAssignedIdentities/myid1", + } + update := calculateVMPatch(settings, currentVM) + + Expect(update).To(BeNil()) + }) + }) +}) + +var _ = Describe("In Place Update Controller", func() { + var vmName string + var vm *armcompute.VirtualMachine + var nodeClaim *corev1beta1.NodeClaim + + BeforeEach(func() { + vmName = "vm-a" + vm = &armcompute.VirtualMachine{ + ID: lo.ToPtr(utils.MkVMID(azureEnv.AzureResourceGraphAPI.ResourceGroup, vmName)), + Name: lo.ToPtr(vmName), + } + + nodeClaim = coretest.NodeClaim(corev1beta1.NodeClaim{ + Status: corev1beta1.NodeClaimStatus{ + ProviderID: utils.ResourceIDToProviderID(ctx, *vm.ID), + }, + }) + + ctx = settings.ToContext(ctx, test.Settings()) + + azureEnv.Reset() + }) + + AfterEach(func() { + ExpectCleanedUp(ctx, env.Client) + }) + + Context("Basic tests", func() { + It("should not call Azure if the hash matches", func() { + azureEnv.VirtualMachinesAPI.Instances.Store(lo.FromPtr(vm.ID), *vm) + hash, err := HashFromNodeClaim(settings.FromContext(ctx), nodeClaim) + Expect(err).ToNot(HaveOccurred()) + + // Force the goal hash into annotations here, which should prevent the reconciler from doing anything on Azure + nodeClaim.Annotations = map[string]string{v1alpha2.AnnotationInPlaceUpdateHash: hash} + + ExpectApplied(ctx, env.Client, nodeClaim) + ExpectReconcileSucceeded(ctx, inPlaceUpdateController, client.ObjectKeyFromObject(nodeClaim)) + + Expect(azureEnv.VirtualMachinesAPI.VirtualMachineUpdateBehavior.Calls()).To(Equal(0)) + + nodeClaim = ExpectExists(ctx, env.Client, nodeClaim) + Expect(nodeClaim.Annotations).To(HaveKey(v1alpha2.AnnotationInPlaceUpdateHash)) + Expect(nodeClaim.Annotations[v1alpha2.AnnotationInPlaceUpdateHash]).ToNot(BeEmpty()) + }) + }) + + Context("Identity tests", func() { + It("should add a hash annotation to NodeClaim if there are no identities", func() { + azureEnv.VirtualMachinesAPI.Instances.Store(lo.FromPtr(vm.ID), *vm) + + ExpectApplied(ctx, env.Client, nodeClaim) + ExpectReconcileSucceeded(ctx, inPlaceUpdateController, client.ObjectKeyFromObject(nodeClaim)) + + updatedVM, err := azureEnv.InstanceProvider.Get(ctx, vmName) + Expect(err).ToNot(HaveOccurred()) + + Expect(updatedVM).To(Equal(vm)) // No change expected + + // The nodeClaim should have the InPlaceUpdateHash annotation + nodeClaim = ExpectExists(ctx, env.Client, nodeClaim) + Expect(nodeClaim.Annotations).To(HaveKey(v1alpha2.AnnotationInPlaceUpdateHash)) + Expect(nodeClaim.Annotations[v1alpha2.AnnotationInPlaceUpdateHash]).ToNot(BeEmpty()) + }) + + It("should add a hash annotation to NodeClaim and update VM if there are missing identities", func() { + azureEnv.VirtualMachinesAPI.Instances.Store(lo.FromPtr(vm.ID), *vm) + + ctx = settings.ToContext( + ctx, + test.Settings( + test.SettingOptions{ + NodeIdentities: []string{ + "/subscriptions/1234/resourceGroups/mcrg/providers/Microsoft.ManagedIdentity/userAssignedIdentities/myid1", + }, + })) + + ExpectApplied(ctx, env.Client, nodeClaim) + ExpectReconcileSucceeded(ctx, inPlaceUpdateController, client.ObjectKeyFromObject(nodeClaim)) + + updatedVM, err := azureEnv.InstanceProvider.Get(ctx, vmName) + Expect(err).ToNot(HaveOccurred()) + + Expect(updatedVM).ToNot(Equal(vm)) + Expect(updatedVM.Identity).ToNot(BeNil()) + Expect(updatedVM.Identity.UserAssignedIdentities).To(HaveKey("/subscriptions/1234/resourceGroups/mcrg/providers/Microsoft.ManagedIdentity/userAssignedIdentities/myid1")) + + nodeClaim = ExpectExists(ctx, env.Client, nodeClaim) + Expect(nodeClaim.Annotations).To(HaveKey(v1alpha2.AnnotationInPlaceUpdateHash)) + Expect(nodeClaim.Annotations[v1alpha2.AnnotationInPlaceUpdateHash]).ToNot(BeEmpty()) + }) + + It("should not call Azure if the expected identities already exist on the VM", func() { + vm.Identity = &armcompute.VirtualMachineIdentity{ + UserAssignedIdentities: map[string]*armcompute.UserAssignedIdentitiesValue{ + "/subscriptions/1234/resourceGroups/mcrg/providers/Microsoft.ManagedIdentity/userAssignedIdentities/myid1": {}, + }, + } + azureEnv.VirtualMachinesAPI.Instances.Store(lo.FromPtr(vm.ID), *vm) + + ExpectApplied(ctx, env.Client, nodeClaim) + ExpectReconcileSucceeded(ctx, inPlaceUpdateController, client.ObjectKeyFromObject(nodeClaim)) + + Expect(azureEnv.VirtualMachinesAPI.VirtualMachineUpdateBehavior.Calls()).To(Equal(0)) + + nodeClaim = ExpectExists(ctx, env.Client, nodeClaim) + Expect(nodeClaim.Annotations).To(HaveKey(v1alpha2.AnnotationInPlaceUpdateHash)) + Expect(nodeClaim.Annotations[v1alpha2.AnnotationInPlaceUpdateHash]).ToNot(BeEmpty()) + }) + + It("should not clear existing identities on VM", func() { + vm.Identity = &armcompute.VirtualMachineIdentity{ + UserAssignedIdentities: map[string]*armcompute.UserAssignedIdentitiesValue{ + "/subscriptions/1234/resourceGroups/mcrg/providers/Microsoft.ManagedIdentity/userAssignedIdentities/myotheridentity": {}, + }, + } + + ctx = settings.ToContext( + ctx, + test.Settings( + test.SettingOptions{ + NodeIdentities: []string{ + "/subscriptions/1234/resourceGroups/mcrg/providers/Microsoft.ManagedIdentity/userAssignedIdentities/myid1", + }, + })) + + azureEnv.VirtualMachinesAPI.Instances.Store(lo.FromPtr(vm.ID), *vm) + + ExpectApplied(ctx, env.Client, nodeClaim) + ExpectReconcileSucceeded(ctx, inPlaceUpdateController, client.ObjectKeyFromObject(nodeClaim)) + + updatedVM, err := azureEnv.InstanceProvider.Get(ctx, vmName) + Expect(err).ToNot(HaveOccurred()) + + Expect(updatedVM.Identity).ToNot(BeNil()) + Expect(updatedVM.Identity.UserAssignedIdentities).To(HaveKey("/subscriptions/1234/resourceGroups/mcrg/providers/Microsoft.ManagedIdentity/userAssignedIdentities/myid1")) + Expect(updatedVM.Identity.UserAssignedIdentities).To(HaveKey("/subscriptions/1234/resourceGroups/mcrg/providers/Microsoft.ManagedIdentity/userAssignedIdentities/myotheridentity")) + + nodeClaim = ExpectExists(ctx, env.Client, nodeClaim) + Expect(nodeClaim.Annotations).To(HaveKey(v1alpha2.AnnotationInPlaceUpdateHash)) + Expect(nodeClaim.Annotations[v1alpha2.AnnotationInPlaceUpdateHash]).ToNot(BeEmpty()) + }) + }) +}) diff --git a/pkg/controllers/nodeclaim/inplaceupdate/utils.go b/pkg/controllers/nodeclaim/inplaceupdate/utils.go new file mode 100644 index 000000000..7ec85d211 --- /dev/null +++ b/pkg/controllers/nodeclaim/inplaceupdate/utils.go @@ -0,0 +1,73 @@ +/* +Portions Copyright (c) Microsoft Corporation. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package inplaceupdate + +import ( + "encoding/json" + "hash/fnv" + "strconv" + + "github.com/aws/karpenter-core/pkg/apis/v1beta1" + "k8s.io/apimachinery/pkg/util/sets" + + "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute" + "github.com/Azure/karpenter/pkg/apis/settings" +) + +// According to https://pkg.go.dev/encoding/json#Marshal, it's safe to use map-types (and encoding/json in general) to produce +// strings deterministically. +type inPlaceUpdateFields struct { + Identities sets.Set[string] `json:"identities,omitempty"` +} + +func (i *inPlaceUpdateFields) CalculateHash() (string, error) { + encoded, err := json.Marshal(i) + if err != nil { + return "", err + } + + h := fnv.New32a() + _, err = h.Write(encoded) + if err != nil { + return "", err + } + return strconv.FormatUint(uint64(h.Sum32()), 10), nil +} + +func HashFromVM(vm *armcompute.VirtualMachine) (string, error) { + identities := sets.Set[string]{} + if vm.Identity != nil { + for ident := range vm.Identity.UserAssignedIdentities { + identities.Insert(ident) + } + } + + hashStruct := &inPlaceUpdateFields{ + Identities: identities, + } + + return hashStruct.CalculateHash() +} + +// HashFromNodeClaim calculates an inplace update hash from the specified machine and settings +func HashFromNodeClaim(settings *settings.Settings, _ *v1beta1.NodeClaim) (string, error) { + hashStruct := &inPlaceUpdateFields{ + Identities: sets.New(settings.NodeIdentities...), + } + + return hashStruct.CalculateHash() +} diff --git a/pkg/controllers/nodeclaim/link/controller.go b/pkg/controllers/nodeclaim/link/controller.go new file mode 100644 index 000000000..5d1939a25 --- /dev/null +++ b/pkg/controllers/nodeclaim/link/controller.go @@ -0,0 +1,155 @@ +/* +Portions Copyright (c) Microsoft Corporation. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package link + +import ( + "context" + "fmt" + "math" + "time" + + "github.com/patrickmn/go-cache" + "github.com/prometheus/client_golang/prometheus" + "github.com/samber/lo" + "go.uber.org/multierr" + v1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/sets" + "k8s.io/client-go/util/workqueue" + "knative.dev/pkg/logging" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/manager" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + + "github.com/Azure/karpenter/pkg/cloudprovider" + "github.com/aws/karpenter-core/pkg/apis/v1alpha5" + corev1beta1 "github.com/aws/karpenter-core/pkg/apis/v1beta1" + + corecloudprovider "github.com/aws/karpenter-core/pkg/cloudprovider" + "github.com/aws/karpenter-core/pkg/metrics" + "github.com/aws/karpenter-core/pkg/operator/controller" + nodeclaimutil "github.com/aws/karpenter-core/pkg/utils/nodeclaim" +) + +const creationReasonLabel = "linking" +const NodeClaimLinkedAnnotationKey = v1alpha5.MachineLinkedAnnotationKey // still using the one from v1alpha5 + +type Controller struct { + kubeClient client.Client + cloudProvider *cloudprovider.CloudProvider + Cache *cache.Cache // exists due to eventual consistency on the controller-runtime cache +} + +func NewController(kubeClient client.Client, cloudProvider *cloudprovider.CloudProvider) *Controller { + return &Controller{ + kubeClient: kubeClient, + cloudProvider: cloudProvider, + Cache: cache.New(time.Minute, time.Second*10), + } +} + +func (c *Controller) Name() string { + return "nodeclaim.link" +} + +func (c *Controller) Reconcile(ctx context.Context, _ reconcile.Request) (reconcile.Result, error) { + // We LIST VMs on the CloudProvider BEFORE we grab NodeClaims/Nodes on the cluster so that we make sure that, if + // LISTing instances takes a long time, our information is more updated by the time we get to NodeClaim and Node LIST + retrieved, err := c.cloudProvider.List(ctx) + if err != nil { + return reconcile.Result{}, fmt.Errorf("listing cloudprovider VMs, %w", err) + } + nodeClaims := &corev1beta1.NodeClaimList{} + if err := c.kubeClient.List(ctx, nodeClaims); err != nil { + return reconcile.Result{}, err + } + nodeList := &v1.NodeList{} + if err := c.kubeClient.List(ctx, nodeList, client.HasLabels{corev1beta1.NodePoolLabelKey}); err != nil { + return reconcile.Result{}, err + } + retrievedIDs := sets.New(lo.Map(retrieved, func(m *corev1beta1.NodeClaim, _ int) string { return m.Status.ProviderID })...) + // Inject any nodes that are re-owned using karpenter.sh/nodepool but aren't found from the cloudprovider.List() call + for i := range nodeList.Items { + if _, ok := lo.Find(retrieved, func(r *corev1beta1.NodeClaim) bool { + return retrievedIDs.Has(nodeList.Items[i].Spec.ProviderID) + }); !ok { + retrieved = append(retrieved, nodeclaimutil.NewFromNode(&nodeList.Items[i])) + } + } + // Filter out any machines that shouldn't be linked + retrieved = lo.Filter(retrieved, func(m *corev1beta1.NodeClaim, _ int) bool { + return m.DeletionTimestamp.IsZero() && m.Labels[corev1beta1.NodePoolLabelKey] != "" + }) + errs := make([]error, len(retrieved)) + workqueue.ParallelizeUntil(ctx, 100, len(retrieved), func(i int) { + errs[i] = c.link(ctx, retrieved[i], nodeClaims.Items) + }) + // Effectively, don't requeue this again once it succeeds + return reconcile.Result{RequeueAfter: math.MaxInt64}, multierr.Combine(errs...) +} + +func (c *Controller) link(ctx context.Context, retrieved *corev1beta1.NodeClaim, existingNodeClaims []corev1beta1.NodeClaim) error { + ctx = logging.WithLogger(ctx, logging.FromContext(ctx).With("provider-id", retrieved.Status.ProviderID, "nodepool", retrieved.Labels[corev1beta1.NodePoolLabelKey])) + nodePool := &corev1beta1.NodePool{} + if err := c.kubeClient.Get(ctx, types.NamespacedName{Name: retrieved.Labels[corev1beta1.NodePoolLabelKey]}, nodePool); err != nil { + return client.IgnoreNotFound(err) + } + if c.shouldCreateLinkedNodeClaim(retrieved, existingNodeClaims) { + // TODO v1beta1 there does not seem to be v1beta1 equivalent to this: + // machine := machineutil.NewFromNode(machineutil.New(&v1.Node{}, provisioner)) + // For now creating manually + nodeClaim := &corev1beta1.NodeClaim{ + Spec: nodePool.Spec.Template.Spec, + } + + nodeClaim.GenerateName = fmt.Sprintf("%s-", nodePool.Name) + // This annotation communicates to the nodeclaim controller that this is a nodeclaim linking scenario, not + // a case where we want to provision a new VM + nodeClaim.Annotations = lo.Assign(nodeClaim.Annotations, map[string]string{ + NodeClaimLinkedAnnotationKey: retrieved.Status.ProviderID, + }) + if err := c.kubeClient.Create(ctx, nodeClaim); err != nil { + return err + } + logging.FromContext(ctx).With("nodeclaim", nodeClaim.Name).Debugf("generated nodeclaim from cloudprovider") + metrics.NodeClaimsCreatedCounter.With(prometheus.Labels{ + metrics.ReasonLabel: creationReasonLabel, + metrics.NodePoolLabel: nodeClaim.Labels[corev1beta1.NodePoolLabelKey], + }).Inc() + c.Cache.SetDefault(retrieved.Status.ProviderID, nil) + } + return corecloudprovider.IgnoreNodeClaimNotFoundError(c.cloudProvider.Link(ctx, retrieved)) +} + +func (c *Controller) shouldCreateLinkedNodeClaim(retrieved *corev1beta1.NodeClaim, existingNodeClaims []corev1beta1.NodeClaim) bool { + // VM was already created but controller-runtime cache didn't update + if _, ok := c.Cache.Get(retrieved.Status.ProviderID); ok { + return false + } + // We have a nodeclaim registered for this, so no need to hydrate it + if _, ok := lo.Find(existingNodeClaims, func(m corev1beta1.NodeClaim) bool { + return m.Annotations[NodeClaimLinkedAnnotationKey] == retrieved.Status.ProviderID || + m.Status.ProviderID == retrieved.Status.ProviderID + }); ok { + return false + } + return true +} + +func (c *Controller) Builder(_ context.Context, m manager.Manager) controller.Builder { + return controller.NewSingletonManagedBy(m) +} diff --git a/pkg/controllers/nodeclaim/link/suite_test.go b/pkg/controllers/nodeclaim/link/suite_test.go new file mode 100644 index 000000000..0389ab1b4 --- /dev/null +++ b/pkg/controllers/nodeclaim/link/suite_test.go @@ -0,0 +1,399 @@ +/* +Portions Copyright (c) Microsoft Corporation. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package link_test + +import ( + "context" + "fmt" + "testing" + "time" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "github.com/samber/lo" + v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/sets" + "k8s.io/client-go/tools/record" + . "knative.dev/pkg/logging/testing" + "sigs.k8s.io/controller-runtime/pkg/client" + + "github.com/aws/karpenter-core/pkg/apis/v1alpha5" + + corev1beta1 "github.com/aws/karpenter-core/pkg/apis/v1beta1" + "github.com/aws/karpenter-core/pkg/events" + coreoptions "github.com/aws/karpenter-core/pkg/operator/options" + "github.com/aws/karpenter-core/pkg/operator/scheme" + coretest "github.com/aws/karpenter-core/pkg/test" + . "github.com/aws/karpenter-core/pkg/test/expectations" + + "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute" + "github.com/Azure/karpenter/pkg/apis" + "github.com/Azure/karpenter/pkg/apis/settings" + "github.com/Azure/karpenter/pkg/apis/v1alpha2" + "github.com/Azure/karpenter/pkg/cloudprovider" + "github.com/Azure/karpenter/pkg/controllers/nodeclaim/link" + "github.com/Azure/karpenter/pkg/fake" + "github.com/Azure/karpenter/pkg/providers/instance" + "github.com/Azure/karpenter/pkg/test" + "github.com/Azure/karpenter/pkg/utils" +) + +var ctx context.Context +var stop context.CancelFunc +var env *coretest.Environment +var azureEnv *test.Environment +var cloudProvider *cloudprovider.CloudProvider +var linkController *link.Controller + +const nodeClaimLinkedAnnotationKey = v1alpha5.MachineLinkedAnnotationKey // still using the one from v1alpha5 + +func TestAPIs(t *testing.T) { + ctx = TestContextWithLogger(t) + RegisterFailHandler(Fail) + RunSpecs(t, "NodeClaim") +} + +var _ = BeforeSuite(func() { + ctx = coreoptions.ToContext(ctx, coretest.Options()) + // ctx = options.ToContext(ctx, test.Options()) + ctx = settings.ToContext(ctx, test.Settings()) + + env = coretest.NewEnvironment(scheme.Scheme, coretest.WithCRDs(apis.CRDs...)) + + ctx, stop = context.WithCancel(ctx) + azureEnv = test.NewEnvironment(ctx, env) + cloudProvider = cloudprovider.New(azureEnv.InstanceTypesProvider, azureEnv.InstanceProvider, events.NewRecorder(&record.FakeRecorder{}), env.Client, azureEnv.ImageProvider) + + linkController = link.NewController(env.Client, cloudProvider) +}) + +var _ = AfterSuite(func() { + stop() + Expect(env.Stop()).To(Succeed(), "Failed to stop environment") +}) + +var _ = BeforeEach(func() { + azureEnv.Reset() +}) + +var _ = AfterEach(func() { + ExpectCleanedUp(ctx, env.Client) + linkController.Cache.Flush() +}) + +var _ = Describe("NodeClaimLink", func() { + var vm armcompute.VirtualMachine + var providerID string + + var nodeClass *v1alpha2.AKSNodeClass + var nodePool *corev1beta1.NodePool + + BeforeEach(func() { + id := utils.MkVMID(azureEnv.AzureResourceGraphAPI.ResourceGroup, "vm-a") + providerID = utils.ResourceIDToProviderID(ctx, id) + nodeClass = test.AKSNodeClass() + nodePool = coretest.NodePool(corev1beta1.NodePool{ + Spec: corev1beta1.NodePoolSpec{ + Template: corev1beta1.NodeClaimTemplate{ + Spec: corev1beta1.NodeClaimSpec{ + NodeClassRef: &corev1beta1.NodeClassReference{ + Name: nodeClass.Name, + }, + }, + }, + }, + }) + vm = armcompute.VirtualMachine{ + ID: lo.ToPtr(id), + Name: lo.ToPtr("vm-a"), + Tags: map[string]*string{instance.NodePoolTagKey: lo.ToPtr(nodePool.Name)}, + Zones: []*string{lo.ToPtr(fmt.Sprintf("%s-1a", fake.Region))}, + Location: lo.ToPtr(fake.Region), + Properties: lo.ToPtr(armcompute.VirtualMachineProperties{ + TimeCreated: lo.ToPtr(time.Now()), + }), + } + azureEnv.VirtualMachinesAPI.Instances.Store(*vm.ID, vm) + }) + + It("should link an instance with basic spec set", func() { + nodePool.Spec.Template.Spec.Taints = []v1.Taint{ + { + Key: "testkey", + Value: "testvalue", + Effect: v1.TaintEffectNoSchedule, + }, + } + nodePool.Spec.Template.Spec.StartupTaints = []v1.Taint{ + { + Key: "othertestkey", + Value: "othertestvalue", + Effect: v1.TaintEffectNoExecute, + }, + } + ExpectApplied(ctx, env.Client, nodePool) + ExpectVMExists(*vm.Name) + ExpectReconcileSucceeded(ctx, linkController, client.ObjectKey{}) + + nodeClaims := &corev1beta1.NodeClaimList{} + Expect(env.Client.List(ctx, nodeClaims)).To(Succeed()) + Expect(nodeClaims.Items).To(HaveLen(1)) + nodeClaim := nodeClaims.Items[0] + + // Expect NodeClaim to have populated fields from the node + Expect(nodeClaim.Spec.Taints).To(Equal(nodePool.Spec.Template.Spec.Taints)) + Expect(nodeClaim.Spec.StartupTaints).To(Equal(nodePool.Spec.Template.Spec.StartupTaints)) + + // Expect NodeClaim has linking annotation to get NodeClaim details + Expect(nodeClaim.Annotations).To(HaveKeyWithValue(nodeClaimLinkedAnnotationKey, providerID)) + vm = ExpectVMExists(*vm.Name) + ExpectProvisionerNameTagExists(vm) + }) + It("should link an instance with expected requirements and labels", func() { + nodePool.Spec.Template.Spec.Requirements = []v1.NodeSelectorRequirement{ + { + Key: v1.LabelTopologyZone, + Operator: v1.NodeSelectorOpIn, + Values: []string{"test-zone-1a", "test-zone-1b", "test-zone-1c"}, + }, + { + Key: v1.LabelOSStable, + Operator: v1.NodeSelectorOpIn, + Values: []string{string(v1.Linux), string(v1.Windows)}, + }, + { + Key: v1.LabelArchStable, + Operator: v1.NodeSelectorOpIn, + Values: []string{corev1beta1.ArchitectureAmd64}, + }, + } + ExpectApplied(ctx, env.Client, nodePool) + ExpectVMExists(*vm.Name) + ExpectReconcileSucceeded(ctx, linkController, client.ObjectKey{}) + + nodeClaims := &corev1beta1.NodeClaimList{} + Expect(env.Client.List(ctx, nodeClaims)).To(Succeed()) + Expect(nodeClaims.Items).To(HaveLen(1)) + nodeClaim := nodeClaims.Items[0] + + Expect(nodeClaim.Spec.Requirements).To(HaveLen(3)) + Expect(nodeClaim.Spec.Requirements).To(ContainElements( + v1.NodeSelectorRequirement{ + Key: v1.LabelTopologyZone, + Operator: v1.NodeSelectorOpIn, + Values: []string{"test-zone-1a", "test-zone-1b", "test-zone-1c"}, + }, + v1.NodeSelectorRequirement{ + Key: v1.LabelOSStable, + Operator: v1.NodeSelectorOpIn, + Values: []string{string(v1.Linux), string(v1.Windows)}, + }, + v1.NodeSelectorRequirement{ + Key: v1.LabelArchStable, + Operator: v1.NodeSelectorOpIn, + Values: []string{corev1beta1.ArchitectureAmd64}, + }, + )) + + // Expect NodeClaim has linking annotation to get NodeClaim details + Expect(nodeClaim.Annotations).To(HaveKeyWithValue(nodeClaimLinkedAnnotationKey, providerID)) + ExpectVMExists(*vm.Name) + }) + It("should link an instance with expected kubelet from provisioner kubelet configuration", func() { + nodePool.Spec.Template.Spec.Kubelet = &corev1beta1.KubeletConfiguration{ + ClusterDNS: []string{"10.0.0.1"}, + MaxPods: lo.ToPtr[int32](10), + } + ExpectApplied(ctx, env.Client, nodePool) + ExpectReconcileSucceeded(ctx, linkController, client.ObjectKey{}) + + nodeClaims := &corev1beta1.NodeClaimList{} + Expect(env.Client.List(ctx, nodeClaims)).To(Succeed()) + Expect(nodeClaims.Items).To(HaveLen(1)) + nodeClaim := nodeClaims.Items[0] + + Expect(nodeClaim.Spec.Kubelet).ToNot(BeNil()) + Expect(nodeClaim.Spec.Kubelet.ClusterDNS[0]).To(Equal("10.0.0.1")) + Expect(lo.FromPtr(nodeClaim.Spec.Kubelet.MaxPods)).To(BeNumerically("==", 10)) + + // Expect NodeClaim has linking annotation to get NodeClaim details + Expect(nodeClaim.Annotations).To(HaveKeyWithValue(nodeClaimLinkedAnnotationKey, providerID)) + vm = ExpectVMExists(*vm.Name) + ExpectProvisionerNameTagExists(vm) + }) + It("should link many instances to many machines", func() { + azureEnv.VirtualMachinesAPI.Reset() // Reset so we don't store the extra VM from BeforeEach() + ExpectApplied(ctx, env.Client, nodePool) + // Generate 100 instances that have different vmIDs + var vmNames []string + var vmName string + var vmID string + for i := 0; i < 100; i++ { + vmName = fmt.Sprintf("vm-%d", i) + vmID = utils.MkVMID(azureEnv.AzureResourceGraphAPI.ResourceGroup, vmName) + azureEnv.VirtualMachinesAPI.Instances.Store( + vmID, + armcompute.VirtualMachine{ + ID: lo.ToPtr(utils.MkVMID(azureEnv.AzureResourceGraphAPI.ResourceGroup, vmName)), + Name: lo.ToPtr(vmName), + Properties: &armcompute.VirtualMachineProperties{ + TimeCreated: lo.ToPtr(time.Now().Add(-time.Minute)), + }, + Tags: map[string]*string{ + instance.NodePoolTagKey: lo.ToPtr(nodePool.Name), + }, + Zones: []*string{lo.ToPtr(fmt.Sprintf("%s-1a", fake.Region))}, + Location: lo.ToPtr(fake.Region), + }) + vmNames = append(vmNames, vmName) + } + + // Generate a reconcile loop to link the machines + ExpectReconcileSucceeded(ctx, linkController, client.ObjectKey{}) + + nodeClaims := &corev1beta1.NodeClaimList{} + Expect(env.Client.List(ctx, nodeClaims)).To(Succeed()) + Expect(nodeClaims.Items).To(HaveLen(100)) + + nodeClaimInstanceIDs := sets.New(lo.Map(nodeClaims.Items, func(m corev1beta1.NodeClaim, _ int) string { + return lo.Must(utils.GetVMName(m.Annotations[nodeClaimLinkedAnnotationKey])) + })...) + + Expect(nodeClaimInstanceIDs).To(HaveLen(len(vmNames))) + for _, name := range vmNames { + Expect(nodeClaimInstanceIDs.Has(name)).To(BeTrue()) + vm = ExpectVMExists(name) + ExpectProvisionerNameTagExists(vm) + } + }) + It("should link an instance without node template existence", func() { + // No node template has been applied here + ExpectApplied(ctx, env.Client, nodePool) + ExpectReconcileSucceeded(ctx, linkController, client.ObjectKey{}) + + nodeClaims := &corev1beta1.NodeClaimList{} + Expect(env.Client.List(ctx, nodeClaims)).To(Succeed()) + Expect(nodeClaims.Items).To(HaveLen(1)) + nodeClaim := nodeClaims.Items[0] + + // Expect NodeClaim has linking annotation to get NodeClaim details + Expect(nodeClaim.Annotations).To(HaveKeyWithValue(nodeClaimLinkedAnnotationKey, providerID)) + vm = ExpectVMExists(*vm.Name) + ExpectProvisionerNameTagExists(vm) + }) + It("should link an instance that was re-owned with a provisioner-name label", func() { + azureEnv.VirtualMachinesAPI.Reset() // Reset so we don't store the extra instance + + // Don't include the provisioner-name tag + vm.Tags = lo.OmitBy(vm.Tags, func(key string, value *string) bool { + return key == instance.NodePoolTagKey + }) + azureEnv.VirtualMachinesAPI.Instances.Store(lo.FromPtr(vm.ID), vm) + node := coretest.Node(coretest.NodeOptions{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{ + corev1beta1.NodePoolLabelKey: nodePool.Name, + }, + }, + ProviderID: providerID, + }) + ExpectApplied(ctx, env.Client, node, nodePool) + ExpectReconcileSucceeded(ctx, linkController, client.ObjectKey{}) + + nodeClaims := &corev1beta1.NodeClaimList{} + Expect(env.Client.List(ctx, nodeClaims)).To(Succeed()) + Expect(nodeClaims.Items).To(HaveLen(1)) + nodeClaim := nodeClaims.Items[0] + Expect(nodeClaim.Annotations).To(HaveKeyWithValue(nodeClaimLinkedAnnotationKey, providerID)) + }) + It("should not link an instance without a provisioner tag", func() { + v := ExpectVMExists(*vm.Name) + v.Tags = lo.OmitBy(v.Tags, func(key string, value *string) bool { + return key == instance.NodePoolTagKey + }) + azureEnv.VirtualMachinesAPI.Instances.Store(*v.ID, v) + + ExpectApplied(ctx, env.Client, nodePool) + ExpectReconcileSucceeded(ctx, linkController, client.ObjectKey{}) + + nodeClaims := &corev1beta1.NodeClaimList{} + Expect(env.Client.List(ctx, nodeClaims)).To(Succeed()) + Expect(nodeClaims.Items).To(HaveLen(0)) + }) + It("should not link an instance without a provisioner that exists on the cluster", func() { + // No provisioner has been applied here + ExpectApplied(ctx, env.Client) + ExpectReconcileSucceeded(ctx, linkController, client.ObjectKey{}) + + nodeClaims := &corev1beta1.NodeClaimList{} + Expect(env.Client.List(ctx, nodeClaims)).To(Succeed()) + Expect(nodeClaims.Items).To(HaveLen(0)) + + // Expect that the instance was left alone if the provisioner wasn't found + ExpectVMExists(*vm.Name) + }) + It("should not link an instance for an instance that is already linked", func() { + m := coretest.NodeClaim(corev1beta1.NodeClaim{ + Status: corev1beta1.NodeClaimStatus{ + ProviderID: providerID, + }, + }) + ExpectApplied(ctx, env.Client, nodePool, m) + + nodeClaims := &corev1beta1.NodeClaimList{} + Expect(env.Client.List(ctx, nodeClaims)).To(Succeed()) + Expect(nodeClaims.Items).To(HaveLen(1)) + + // Expect that we go to link machines, and we don't add extra machines from the existing one + ExpectReconcileSucceeded(ctx, linkController, client.ObjectKey{}) + Expect(env.Client.List(ctx, nodeClaims)).To(Succeed()) + Expect(nodeClaims.Items).To(HaveLen(1)) + }) + It("should not remove existing tags when linking", func() { + v := ExpectVMExists(*vm.Name) + v.Tags["testKey"] = lo.ToPtr("testVal") + azureEnv.VirtualMachinesAPI.Instances.Store(*v.ID, v) + + ExpectApplied(ctx, env.Client, nodePool) + ExpectReconcileSucceeded(ctx, linkController, client.ObjectKey{}) + + nodeClaims := &corev1beta1.NodeClaimList{} + Expect(env.Client.List(ctx, nodeClaims)).To(Succeed()) + Expect(nodeClaims.Items).To(HaveLen(1)) + nodeClaim := nodeClaims.Items[0] + + // Expect NodeClaim has linking annotation to get NodeClaim details + Expect(nodeClaim.Annotations).To(HaveKeyWithValue(nodeClaimLinkedAnnotationKey, providerID)) + vm = ExpectVMExists(*vm.Name) + ExpectProvisionerNameTagExists(vm) + Expect(*v.Tags["testKey"]).To(Equal("testVal")) + }) +}) + +func ExpectVMExists(vmName string) armcompute.VirtualMachine { + resp, err := azureEnv.VirtualMachinesAPI.Get(ctx, azureEnv.AzureResourceGraphAPI.ResourceGroup, vmName, nil) + Expect(err).NotTo(HaveOccurred()) + return resp.VirtualMachine +} + +func ExpectProvisionerNameTagExists(vm armcompute.VirtualMachine) { + _, ok := lo.FindKeyBy(vm.Tags, func(key string, value *string) bool { + return key == instance.NodePoolTagKey + }) + Expect(ok).To(BeTrue()) +} diff --git a/pkg/fake/atomic.go b/pkg/fake/atomic.go new file mode 100644 index 000000000..6ae18e388 --- /dev/null +++ b/pkg/fake/atomic.go @@ -0,0 +1,198 @@ +/* +Portions Copyright (c) Microsoft Corporation. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package fake + +import ( + "bytes" + "encoding/json" + "log" + "math" + "sync" +) + +// AtomicPtr is intended for use in mocks to easily expose variables for use in testing. It makes setting and retrieving +// the values race free by wrapping the pointer itself in a mutex. There is no Get() method, but instead a Clone() method +// deep copies the object being stored by serializing/de-serializing it from JSON. This pattern shouldn't be followed +// anywhere else but is an easy way to eliminate races in our tests. +type AtomicPtr[T any] struct { + mu sync.Mutex + value *T +} + +func (a *AtomicPtr[T]) Set(v *T) { + a.mu.Lock() + defer a.mu.Unlock() + a.value = v +} + +func (a *AtomicPtr[T]) IsNil() bool { + a.mu.Lock() + defer a.mu.Unlock() + return a.value == nil +} + +func (a *AtomicPtr[T]) Clone() *T { + a.mu.Lock() + defer a.mu.Unlock() + return clone(a.value) +} + +func clone[T any](v *T) *T { + var buf bytes.Buffer + enc := json.NewEncoder(&buf) + if err := enc.Encode(v); err != nil { + log.Fatalf("encoding %T, %s", v, err) + } + dec := json.NewDecoder(&buf) + var cp T + if err := dec.Decode(&cp); err != nil { + log.Fatalf("encoding %T, %s", v, err) + } + return &cp +} + +func (a *AtomicPtr[T]) Reset() { + a.mu.Lock() + defer a.mu.Unlock() + a.value = nil +} + +type AtomicError struct { + mu sync.Mutex + err error + + calls int + maxCalls int +} + +func (e *AtomicError) Reset() { + e.mu.Lock() + defer e.mu.Unlock() + e.err = nil + e.calls = 0 + e.maxCalls = 0 +} + +func (e *AtomicError) IsNil() bool { + e.mu.Lock() + defer e.mu.Unlock() + return e.err == nil +} + +// Get is equivalent to the error being called, so we increase +// number of calls in this function +func (e *AtomicError) Get() error { + e.mu.Lock() + defer e.mu.Unlock() + if e.calls >= e.maxCalls { + return nil + } + e.calls++ + return e.err +} + +func (e *AtomicError) Set(err error, opts ...AtomicErrorOption) { + e.mu.Lock() + defer e.mu.Unlock() + e.err = err + for _, opt := range opts { + opt(e) + } + if e.maxCalls == 0 { + e.maxCalls = 1 + } +} + +type AtomicErrorOption func(atomicError *AtomicError) + +func MaxCalls(maxCalls int) AtomicErrorOption { + // Setting to 0 is equivalent to allowing infinite errors to API + if maxCalls <= 0 { + maxCalls = math.MaxInt + } + return func(e *AtomicError) { + e.maxCalls = maxCalls + } +} + +// AtomicPtrStack exposes a slice of a pointer type in a race-free manner. The interface is just enough to replace the +// set.Set usage in our previous tests. +type AtomicPtrStack[T any] struct { + mu sync.Mutex + values []*T +} + +func (a *AtomicPtrStack[T]) Reset() { + a.mu.Lock() + defer a.mu.Unlock() + a.values = nil +} + +func (a *AtomicPtrStack[T]) Add(input *T) { + a.mu.Lock() + defer a.mu.Unlock() + a.values = append(a.values, clone(input)) +} + +func (a *AtomicPtrStack[T]) Len() int { + a.mu.Lock() + defer a.mu.Unlock() + return len(a.values) +} + +func (a *AtomicPtrStack[T]) Pop() *T { + a.mu.Lock() + defer a.mu.Unlock() + last := a.values[len(a.values)-1] + a.values = a.values[0 : len(a.values)-1] + return last +} + +// AtomicPtrSlice exposes a slice of a pointer type in a race-free manner. +type AtomicPtrSlice[T any] struct { + mu sync.Mutex + values []*T +} + +func (a *AtomicPtrSlice[T]) Reset() { + a.mu.Lock() + defer a.mu.Unlock() + a.values = nil +} + +func (a *AtomicPtrSlice[T]) Append(input ...*T) { + a.mu.Lock() + defer a.mu.Unlock() + + for _, item := range input { + a.values = append(a.values, clone(item)) + } +} +func (a *AtomicPtrSlice[T]) Len() int { + a.mu.Lock() + defer a.mu.Unlock() + return len(a.values) +} + +func (a *AtomicPtrSlice[T]) Get(index int) *T { + a.mu.Lock() + defer a.mu.Unlock() + if index < 0 || index >= len(a.values) { + return nil + } + return clone(a.values[index]) +} diff --git a/pkg/fake/azureresourcegraphapi.go b/pkg/fake/azureresourcegraphapi.go new file mode 100644 index 000000000..e3fa4d07f --- /dev/null +++ b/pkg/fake/azureresourcegraphapi.go @@ -0,0 +1,96 @@ +/* +Portions Copyright (c) Microsoft Corporation. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package fake + +import ( + "context" + "encoding/json" + + "github.com/samber/lo" + + "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute" + "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resourcegraph/armresourcegraph" + "github.com/Azure/karpenter/pkg/providers/instance" +) + +type AzureResourceGraphResourcesInput struct { + Query armresourcegraph.QueryRequest + Options *armresourcegraph.ClientResourcesOptions +} + +type AzureResourceGraphBehavior struct { + AzureResourceGraphResourcesBehavior MockedFunction[AzureResourceGraphResourcesInput, armresourcegraph.ClientResourcesResponse] + VirtualMachinesAPI *VirtualMachinesAPI + ResourceGroup string +} + +// assert that the fake implements the interface +var _ instance.AzureResourceGraphAPI = (*AzureResourceGraphAPI)(nil) + +type AzureResourceGraphAPI struct { + AzureResourceGraphBehavior +} + +// Reset must be called between tests otherwise tests will pollute each other. +func (c *AzureResourceGraphAPI) Reset() {} + +func (c *AzureResourceGraphAPI) Resources(_ context.Context, query armresourcegraph.QueryRequest, options *armresourcegraph.ClientResourcesOptions) (armresourcegraph.ClientResourcesResponse, error) { + input := &AzureResourceGraphResourcesInput{ + Query: query, + Options: options, + } + resourceList := c.getResourceList(*query.Query) + + return c.AzureResourceGraphResourcesBehavior.Invoke(input, func(input *AzureResourceGraphResourcesInput) (armresourcegraph.ClientResourcesResponse, error) { + return armresourcegraph.ClientResourcesResponse{ + QueryResponse: armresourcegraph.QueryResponse{ + Data: resourceList, + }, + }, nil + }) +} + +func (c *AzureResourceGraphAPI) getResourceList(query string) []interface{} { + switch query { + case instance.GetListQueryBuilder(c.ResourceGroup).String(): + vmList := lo.Filter(c.loadVMObjects(), func(vm armcompute.VirtualMachine, _ int) bool { + return vm.Tags != nil && vm.Tags[instance.NodePoolTagKey] != nil + }) + resourceList := lo.Map(vmList, func(vm armcompute.VirtualMachine, _ int) interface{} { + b, _ := json.Marshal(vm) + return convertBytesToInterface(b) + }) + return resourceList + } + return nil +} + +func (c *AzureResourceGraphAPI) loadVMObjects() []armcompute.VirtualMachine { + vmList := []armcompute.VirtualMachine{} + c.VirtualMachinesAPI.Instances.Range(func(k, v any) bool { + vm, _ := c.VirtualMachinesAPI.Instances.Load(k) + vmList = append(vmList, vm.(armcompute.VirtualMachine)) + return true + }) + return vmList +} + +func convertBytesToInterface(b []byte) interface{} { + jsonObj := instance.Resource{} + _ = json.Unmarshal(b, &jsonObj) + return interface{}(jsonObj) +} diff --git a/pkg/fake/azureresourcegraphapi_test.go b/pkg/fake/azureresourcegraphapi_test.go new file mode 100644 index 000000000..45eccdfde --- /dev/null +++ b/pkg/fake/azureresourcegraphapi_test.go @@ -0,0 +1,117 @@ +/* +Portions Copyright (c) Microsoft Corporation. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package fake + +import ( + "context" + "fmt" + "testing" + + "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute" + "github.com/Azure/karpenter/pkg/providers/instance" + "github.com/Azure/karpenter/pkg/utils" + "github.com/samber/lo" + "github.com/stretchr/testify/assert" +) + +func TestAzureResourceGraphAPI_Resources_VM(t *testing.T) { + resourceGroup := "test_managed_cluster_rg" + subscriptionID := "test_sub" + virtualMachinesAPI := &VirtualMachinesAPI{} + azureResourceGraphAPI := &AzureResourceGraphAPI{AzureResourceGraphBehavior{VirtualMachinesAPI: virtualMachinesAPI, ResourceGroup: resourceGroup}} + cases := []struct { + testName string + vmNames []string + tags map[string]*string + expectedError string + }{ + { + testName: "happy case", + vmNames: []string{"A", "B", "C"}, + tags: map[string]*string{instance.NodePoolTagKey: lo.ToPtr("default")}, + expectedError: "", + }, + { + testName: "no tags", + vmNames: []string{"A", "B", "C"}, + tags: nil, + expectedError: "Unexpected nil resource data", + }, + { + testName: "wrong tags", + vmNames: []string{"A", "B", "C"}, + tags: map[string]*string{"dummy tag": lo.ToPtr("default")}, + expectedError: "Unexpected nil resource data", + }, + } + for _, c := range cases { + t.Run(c.testName, func(t *testing.T) { + for _, name := range c.vmNames { + _, err := instance.CreateVirtualMachine(context.Background(), virtualMachinesAPI, resourceGroup, name, armcompute.VirtualMachine{Tags: c.tags}) + if err != nil { + t.Errorf("Unexpected error %v", err) + return + } + } + queryRequest := instance.NewQueryRequest(&subscriptionID, instance.GetListQueryBuilder(resourceGroup).String()) + data, err := instance.GetResourceData(context.Background(), azureResourceGraphAPI, *queryRequest) + if err != nil { + t.Errorf("Unexpected error %v", err) + return + } + if data == nil { + assert.Equal(t, c.expectedError, "Unexpected nil resource data") + } + if c.expectedError == "" { + if len(data) != len(c.vmNames) { + t.Errorf("Not all VMs were returned") + return + } + for _, vm := range data { + err := checkVM(vm, resourceGroup) + if err != nil { + t.Error(err) + } + } + } + }) + } +} + +func checkVM(vm instance.Resource, rg string) error { + name, ok := vm["name"] + if !ok { + return fmt.Errorf("VM is missing name") + } + expectedID := utils.MkVMID(rg, name.(string)) + id, ok := vm["id"] + if !ok { + return fmt.Errorf("VM is missing id") + } + if expectedID != id { + return fmt.Errorf("VM id is incorrect") + } + properties, ok := vm["properties"] + if !ok { + return fmt.Errorf("VM is missing properties") + } + _, ok = properties.(instance.Resource)["timeCreated"] + if !ok { + return fmt.Errorf("VM is missing timeCreated property") + } + return nil +} diff --git a/pkg/fake/communityimageversionsapi.go b/pkg/fake/communityimageversionsapi.go new file mode 100644 index 000000000..c9d8f265c --- /dev/null +++ b/pkg/fake/communityimageversionsapi.go @@ -0,0 +1,59 @@ +/* +Portions Copyright (c) Microsoft Corporation. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package fake + +import ( + "context" + + "github.com/Azure/azure-sdk-for-go/sdk/azcore/runtime" + "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute/v5" + + "github.com/Azure/karpenter/pkg/providers/imagefamily" +) + +type CommunityGalleryImageVersionsAPI struct { + ImageVersions AtomicPtrSlice[armcompute.CommunityGalleryImageVersion] +} + +// assert that the fake implements the interface +var _ imagefamily.CommunityGalleryImageVersionsAPI = (*CommunityGalleryImageVersionsAPI)(nil) + +// NewListPager returns a new pager to return the next page of CommunityGalleryImageVersionsClientListResponse +func (c *CommunityGalleryImageVersionsAPI) NewListPager(_ string, _ string, _ string, _ *armcompute.CommunityGalleryImageVersionsClientListOptions) *runtime.Pager[armcompute.CommunityGalleryImageVersionsClientListResponse] { + pagingHandler := runtime.PagingHandler[armcompute.CommunityGalleryImageVersionsClientListResponse]{ + More: func(page armcompute.CommunityGalleryImageVersionsClientListResponse) bool { + return false + }, + Fetcher: func(ctx context.Context, _ *armcompute.CommunityGalleryImageVersionsClientListResponse) (armcompute.CommunityGalleryImageVersionsClientListResponse, error) { + output := armcompute.CommunityGalleryImageVersionList{ + Value: []*armcompute.CommunityGalleryImageVersion{}, + } + output.Value = append(output.Value, c.ImageVersions.values...) + return armcompute.CommunityGalleryImageVersionsClientListResponse{ + CommunityGalleryImageVersionList: output, + }, nil + }, + } + return runtime.NewPager(pagingHandler) +} + +func (c *CommunityGalleryImageVersionsAPI) Reset() { + if c == nil { + return + } + c.ImageVersions.Reset() +} diff --git a/pkg/fake/loadbalancerapi.go b/pkg/fake/loadbalancerapi.go new file mode 100644 index 000000000..5796dbed6 --- /dev/null +++ b/pkg/fake/loadbalancerapi.go @@ -0,0 +1,105 @@ +/* +Portions Copyright (c) Microsoft Corporation. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package fake + +import ( + "context" + "fmt" + "sort" + "sync" + + "github.com/Azure/azure-sdk-for-go/sdk/azcore/runtime" + "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/network/armnetwork" + "github.com/Azure/karpenter/pkg/providers/loadbalancer" + "github.com/samber/lo" +) + +type LoadBalancersBehavior struct { + LoadBalancers sync.Map +} + +// assert that the fake implements the interface +var _ loadbalancer.LoadBalancersAPI = &LoadBalancersAPI{} + +type LoadBalancersAPI struct { + LoadBalancersBehavior +} + +// Reset must be called between tests otherwise tests will pollute each other. +func (api *LoadBalancersAPI) Reset() { + api.LoadBalancers.Range(func(k, v any) bool { + api.LoadBalancers.Delete(k) + return true + }) +} + +func (api *LoadBalancersAPI) Get(_ context.Context, resourceGroupName string, loadBalancerName string, _ *armnetwork.LoadBalancersClientGetOptions) (armnetwork.LoadBalancersClientGetResponse, error) { + id := MakeLoadBalancerID(resourceGroupName, loadBalancerName) + lb, ok := api.LoadBalancers.Load(id) + if !ok { + return armnetwork.LoadBalancersClientGetResponse{}, fmt.Errorf("not found") + } + return armnetwork.LoadBalancersClientGetResponse{ + LoadBalancer: lb.(armnetwork.LoadBalancer), + }, nil +} + +func (api *LoadBalancersAPI) NewListPager(_ string, _ *armnetwork.LoadBalancersClientListOptions) *runtime.Pager[armnetwork.LoadBalancersClientListResponse] { + pagingHandler := runtime.PagingHandler[armnetwork.LoadBalancersClientListResponse]{ + More: func(page armnetwork.LoadBalancersClientListResponse) bool { + return false // TODO: It might be ideal if we had a MockPager which sometimes simulated multiple pages of results to ensure we handle that correctly + }, + Fetcher: func(ctx context.Context, _ *armnetwork.LoadBalancersClientListResponse) (armnetwork.LoadBalancersClientListResponse, error) { + output := armnetwork.LoadBalancerListResult{ + Value: []*armnetwork.LoadBalancer{}, + } + api.LoadBalancers.Range(func(key, value any) bool { + cast := value.(armnetwork.LoadBalancer) + output.Value = append(output.Value, &cast) + + return true + }) + + // Sort the result according to ID so that we have a stable base to write asserts upon + sort.Slice(output.Value, func(i, j int) bool { + l := output.Value[i] + r := output.Value[j] + + return lo.FromPtr(l.ID) < lo.FromPtr(r.ID) + }) + + return armnetwork.LoadBalancersClientListResponse{ + LoadBalancerListResult: output, + }, nil + }, + } + return runtime.NewPager(pagingHandler) +} + +func MakeLoadBalancerID(resourceGroupName, loadBalancerName string) string { + const subscriptionID = "subscriptionID" // not important for fake + const idFormat = "/subscriptions/%s/resourceGroups/%s/providers/Microsoft.Network/loadBalancers/%s" + + return fmt.Sprintf(idFormat, subscriptionID, resourceGroupName, loadBalancerName) +} + +func MakeBackendAddressPoolID(resourceGroupName, loadBalancerName string, backendAddressPoolName string) string { + const subscriptionID = "subscriptionID" // not important for fake + const idFormat = "/subscriptions/%s/resourceGroups/%s/providers/Microsoft.Network/loadBalancers/%s/backendAddressPools/%s" + + return fmt.Sprintf(idFormat, subscriptionID, resourceGroupName, loadBalancerName, backendAddressPoolName) +} diff --git a/pkg/fake/networkinterfaceapi.go b/pkg/fake/networkinterfaceapi.go new file mode 100644 index 000000000..16eb4402c --- /dev/null +++ b/pkg/fake/networkinterfaceapi.go @@ -0,0 +1,99 @@ +/* +Portions Copyright (c) Microsoft Corporation. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package fake + +import ( + "context" + "fmt" + "sync" + + "github.com/Azure/azure-sdk-for-go/sdk/azcore/runtime" + "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/network/armnetwork" + "github.com/Azure/go-autorest/autorest/to" + "github.com/Azure/karpenter/pkg/providers/instance" +) + +type NetworkInterfaceCreateOrUpdateInput struct { + ResourceGroupName string + InterfaceName string + Interface armnetwork.Interface + Options *armnetwork.InterfacesClientBeginCreateOrUpdateOptions +} + +type NetworkInterfacesBehavior struct { + NetworkInterfacesCreateOrUpdateBehavior MockedLRO[NetworkInterfaceCreateOrUpdateInput, armnetwork.InterfacesClientCreateOrUpdateResponse] + NetworkInterfaces sync.Map +} + +// assert that the fake implements the interface +var _ instance.NetworkInterfacesAPI = (*NetworkInterfacesAPI)(nil) + +type NetworkInterfacesAPI struct { + // instance.NetworkInterfacesAPI + NetworkInterfacesBehavior +} + +// Reset must be called between tests otherwise tests will pollute each other. +func (c *NetworkInterfacesAPI) Reset() { + c.NetworkInterfacesCreateOrUpdateBehavior.Reset() + c.NetworkInterfaces.Range(func(k, v any) bool { + c.NetworkInterfaces.Delete(k) + return true + }) +} + +func (c *NetworkInterfacesAPI) BeginCreateOrUpdate(_ context.Context, resourceGroupName string, interfaceName string, iface armnetwork.Interface, options *armnetwork.InterfacesClientBeginCreateOrUpdateOptions) (*runtime.Poller[armnetwork.InterfacesClientCreateOrUpdateResponse], error) { + input := &NetworkInterfaceCreateOrUpdateInput{ + ResourceGroupName: resourceGroupName, + InterfaceName: interfaceName, + Interface: iface, + Options: options, + } + + return c.NetworkInterfacesCreateOrUpdateBehavior.Invoke(input, func(input *NetworkInterfaceCreateOrUpdateInput) (*armnetwork.InterfacesClientCreateOrUpdateResponse, error) { + iface := input.Interface + id := mkNetworkInterfaceID(input.ResourceGroupName, input.InterfaceName) + iface.ID = to.StringPtr(id) + c.NetworkInterfaces.Store(id, iface) + return &armnetwork.InterfacesClientCreateOrUpdateResponse{ + Interface: iface, + }, nil + }) +} + +func (c *NetworkInterfacesAPI) Get(_ context.Context, resourceGroupName string, interfaceName string, _ *armnetwork.InterfacesClientGetOptions) (armnetwork.InterfacesClientGetResponse, error) { + id := mkNetworkInterfaceID(resourceGroupName, interfaceName) + iface, ok := c.NetworkInterfaces.Load(id) + if !ok { + return armnetwork.InterfacesClientGetResponse{}, fmt.Errorf("not found") + } + return armnetwork.InterfacesClientGetResponse{ + Interface: iface.(armnetwork.Interface), + }, nil +} + +func (c *NetworkInterfacesAPI) BeginDelete(_ context.Context, resourceGroupName string, interfaceName string, _ *armnetwork.InterfacesClientBeginDeleteOptions) (*runtime.Poller[armnetwork.InterfacesClientDeleteResponse], error) { + id := mkNetworkInterfaceID(resourceGroupName, interfaceName) + c.NetworkInterfaces.Delete(id) + return nil, nil +} + +func mkNetworkInterfaceID(resourceGroupName, interfaceName string) string { + const subscriptionID = "subscriptionID" // not important for fake + const idFormat = "/subscriptions/%s/resourceGroups/%s/providers/Microsoft.Network/networkInterfaces/%s" + return fmt.Sprintf(idFormat, subscriptionID, resourceGroupName, interfaceName) +} diff --git a/pkg/fake/pricingapi.go b/pkg/fake/pricingapi.go new file mode 100644 index 000000000..3afbbab93 --- /dev/null +++ b/pkg/fake/pricingapi.go @@ -0,0 +1,68 @@ +/* +Portions Copyright (c) Microsoft Corporation. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package fake + +import ( + "context" + "errors" + "fmt" + + "github.com/Azure/karpenter/pkg/providers/pricing/client" +) + +const Region = "eastus" + +type PricingAPI struct { + client.PricingAPI + PricingBehavior +} +type PricingBehavior struct { + NextError AtomicError + ProductsPricePage AtomicPtr[client.ProductsPricePage] +} + +func (p *PricingAPI) Reset() { + p.NextError.Reset() + p.ProductsPricePage.Reset() +} + +func (p *PricingAPI) GetProductsPricePages(_ context.Context, _ []*client.Filter, fn func(output *client.ProductsPricePage)) error { + if !p.NextError.IsNil() { + return p.NextError.Get() + } + if !p.ProductsPricePage.IsNil() { + fn(p.ProductsPricePage.Clone()) + return nil + } + // fail if the test doesn't provide specific data which causes our pricing provider to use its static price list + return errors.New("no pricing data provided") +} + +func NewProductPrice(instanceType string, price float64) client.Item { + return client.Item{ + ArmSkuName: instanceType, + RetailPrice: price, + } +} + +func NewSpotProductPrice(instanceType string, price float64) client.Item { + return client.Item{ + SkuName: fmt.Sprintf("%s %s", instanceType, "Spot"), + ArmSkuName: instanceType, + RetailPrice: price, + } +} diff --git a/pkg/fake/resourceskuapi.go b/pkg/fake/resourceskuapi.go new file mode 100644 index 000000000..30c589d6f --- /dev/null +++ b/pkg/fake/resourceskuapi.go @@ -0,0 +1,69 @@ +/* +Portions Copyright (c) Microsoft Corporation. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package fake + +import ( + "context" + + //nolint SA1019 - deprecated package + "github.com/Azure/azure-sdk-for-go/services/compute/mgmt/2022-08-01/compute" + "github.com/Azure/skewer" +) + +// TODO: consider using fakes from skewer itself + +type MockSkuClientSingleton struct { + SKUClient *ResourceSKUsAPI +} + +func (sc *MockSkuClientSingleton) GetInstance() skewer.ResourceClient { + return sc.SKUClient +} + +func (sc *MockSkuClientSingleton) Reset() { + sc.SKUClient.Reset() +} + +// assert that the fake implements the interface +var _ skewer.ResourceClient = &ResourceSKUsAPI{} + +type ResourceSKUsAPI struct { + // skewer.ResourceClient +} + +// Reset must be called between tests otherwise tests will pollute each other. +func (s *ResourceSKUsAPI) Reset() { + //c.ResourceSKUsBehavior.Reset() +} + +func (s *ResourceSKUsAPI) ListComplete(_ context.Context, _, _ string) (compute.ResourceSkusResultIterator, error) { + return compute.NewResourceSkusResultIterator( + compute.NewResourceSkusResultPage( + // cur + compute.ResourceSkusResult{ + Value: &ResourceSkus, + }, + // fn + func(ctx context.Context, result compute.ResourceSkusResult) (compute.ResourceSkusResult, error) { + return compute.ResourceSkusResult{ + Value: nil, // end of iterator + NextLink: nil, + }, nil + }, + ), + ), nil +} diff --git a/pkg/fake/types.go b/pkg/fake/types.go new file mode 100644 index 000000000..f364dbfef --- /dev/null +++ b/pkg/fake/types.go @@ -0,0 +1,172 @@ +/* +Portions Copyright (c) Microsoft Corporation. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package fake + +import ( + "context" + "net/http" + "sync/atomic" + + "github.com/Azure/azure-sdk-for-go/sdk/azcore/runtime" +) + +type MockedFunction[I any, O any] struct { + Output AtomicPtr[O] // Output to return on call to this function + CalledWithInput AtomicPtrStack[I] // Stack used to keep track of passed input to this function + Error AtomicError // Error to return a certain number of times defined by custom error options + + successfulCalls atomic.Int32 // Internal construct to keep track of the number of times this function has successfully been called + failedCalls atomic.Int32 // Internal construct to keep track of the number of times this function has failed (with error) +} + +// Reset must be called between tests otherwise tests will pollute +// each other. +func (m *MockedFunction[I, O]) Reset() { + m.Output.Reset() + m.CalledWithInput.Reset() + m.Error.Reset() + + m.successfulCalls.Store(0) + m.failedCalls.Store(0) +} + +func (m *MockedFunction[I, O]) Invoke(input *I, defaultTransformer func(*I) (O, error)) (O, error) { + err := m.Error.Get() + if err != nil { + m.failedCalls.Add(1) + return *new(O), err + } + m.CalledWithInput.Add(input) + + if !m.Output.IsNil() { + m.successfulCalls.Add(1) + return *m.Output.Clone(), nil + } + out, err := defaultTransformer(input) + if err != nil { + m.failedCalls.Add(1) + } else { + m.successfulCalls.Add(1) + } + return out, err +} + +func (m *MockedFunction[I, O]) Calls() int { + return m.SuccessfulCalls() + m.FailedCalls() +} + +func (m *MockedFunction[I, O]) SuccessfulCalls() int { + return int(m.successfulCalls.Load()) +} + +func (m *MockedFunction[I, O]) FailedCalls() int { + return int(m.failedCalls.Load()) +} + +type MockedLRO[I any, O any] struct { + MockedFunction[I, O] + BeginError AtomicError // Error to return a certain number of times defined by custom error options (for Begin) +} + +// Reset must be called between tests otherwise tests will pollute each other. +func (m *MockedLRO[I, O]) Reset() { + m.Output.Reset() + m.CalledWithInput.Reset() + m.BeginError.Reset() + m.Error.Reset() + + m.successfulCalls.Store(0) + m.failedCalls.Store(0) +} + +func (m *MockedLRO[I, O]) Invoke(input *I, defaultTransformer func(*I) (*O, error)) (*runtime.Poller[O], error) { + if err := m.BeginError.Get(); err != nil { + m.failedCalls.Add(1) + return nil, err + } + if err := m.Error.Get(); err != nil { + m.failedCalls.Add(1) + return newMockPoller[O](nil, err) + } + + m.CalledWithInput.Add(input) + + if !m.Output.IsNil() { + m.successfulCalls.Add(1) + return newMockPoller(m.Output.Clone(), nil) + } + out, err := defaultTransformer(input) + if err != nil { + m.failedCalls.Add(1) + } else { + m.successfulCalls.Add(1) + } + return newMockPoller(out, err) +} + +func (m *MockedLRO[I, O]) Calls() int { + return m.SuccessfulCalls() + m.FailedCalls() +} + +func (m *MockedLRO[I, O]) SuccessfulCalls() int { + return int(m.successfulCalls.Load()) +} + +func (m *MockedLRO[I, O]) FailedCalls() int { + return int(m.failedCalls.Load()) +} + +// MockHandler returns a pre-defined result or error. +type MockHandler[T any] struct { + result *T + err error +} + +// Done returns true if the LRO has reached a terminal state. TrivialHanlder is always done. +func (h MockHandler[T]) Done() bool { + return true +} + +// Poll fetches the latest state of the LRO. +func (h MockHandler[T]) Poll(context.Context) (*http.Response, error) { + if h.err != nil { + return nil, h.err + } + return nil, nil +} + +// Result is called once the LRO has reached a terminal state. It populates the out parameter +// with the result of the operation. +func (h MockHandler[T]) Result(_ context.Context, result *T) error { + if h.err != nil { + return h.err + } + *result = *h.result // TODO: may need to deep copy + return nil +} + +// newMockPoller returns a poller with a mock handler that returns the given result and error. +func newMockPoller[T any](result *T, err error) (*runtime.Poller[T], error) { + // http.Response and Pipeline are not used + return runtime.NewPoller(nil, runtime.Pipeline{}, &runtime.NewPollerOptions[T]{ + Handler: MockHandler[T]{ + result: result, + err: err, + }, + // Response: &result at the poller level is not needed, result from handler is always used + }) +} diff --git a/pkg/fake/virtualmachineextensionsapi.go b/pkg/fake/virtualmachineextensionsapi.go new file mode 100644 index 000000000..e118117d9 --- /dev/null +++ b/pkg/fake/virtualmachineextensionsapi.go @@ -0,0 +1,76 @@ +/* +Portions Copyright (c) Microsoft Corporation. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package fake + +import ( + "context" + "fmt" + + "github.com/Azure/azure-sdk-for-go/sdk/azcore/runtime" + "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute" + "github.com/Azure/go-autorest/autorest/to" + "github.com/Azure/karpenter/pkg/providers/instance" +) + +type VirtualMachineExtensionCreateOrUpdateInput struct { + ResourceGroupName string + VirtualMachineName string + VirtualMachineExtensionName string + VirtualMachineExtension armcompute.VirtualMachineExtension + Options *armcompute.VirtualMachineExtensionsClientBeginCreateOrUpdateOptions +} + +type VirtualMachineExtensionsBehavior struct { + VirtualMachineExtensionsCreateOrUpdateBehavior MockedLRO[VirtualMachineExtensionCreateOrUpdateInput, armcompute.VirtualMachineExtensionsClientCreateOrUpdateResponse] + // not keeping track of extensions +} + +// assert that ComputeAPI implements ARMComputeAPI +var _ instance.VirtualMachineExtensionsAPI = (*VirtualMachineExtensionsAPI)(nil) + +type VirtualMachineExtensionsAPI struct { + // instance.VirtualMachineExtensionsAPI + VirtualMachineExtensionsBehavior +} + +// Reset must be called between tests otherwise tests will pollute each other. +func (c *VirtualMachineExtensionsAPI) Reset() { + c.VirtualMachineExtensionsCreateOrUpdateBehavior.Reset() +} + +func (c *VirtualMachineExtensionsAPI) BeginCreateOrUpdate(_ context.Context, resourceGroupName, vmName, extensionName string, extension armcompute.VirtualMachineExtension, options *armcompute.VirtualMachineExtensionsClientBeginCreateOrUpdateOptions) (*runtime.Poller[armcompute.VirtualMachineExtensionsClientCreateOrUpdateResponse], error) { + input := &VirtualMachineExtensionCreateOrUpdateInput{ + ResourceGroupName: resourceGroupName, + VirtualMachineName: vmName, + VirtualMachineExtensionName: extensionName, + VirtualMachineExtension: extension, + Options: options, + } + + return c.VirtualMachineExtensionsCreateOrUpdateBehavior.Invoke(input, func(input *VirtualMachineExtensionCreateOrUpdateInput) (*armcompute.VirtualMachineExtensionsClientCreateOrUpdateResponse, error) { + result := input.VirtualMachineExtension + result.ID = to.StringPtr(mkVMExtensionID(input.ResourceGroupName, input.VirtualMachineName, input.VirtualMachineExtensionName)) + return &armcompute.VirtualMachineExtensionsClientCreateOrUpdateResponse{ + VirtualMachineExtension: result, + }, nil + }) +} + +func mkVMExtensionID(resourceGroupName, vmName, extensionName string) string { + const idFormat = "/subscriptions/subscriptionID/resourceGroups/%s/providers/Microsoft.Compute/virtualMachines/%s/extensions/%s" + return fmt.Sprintf(idFormat, resourceGroupName, vmName, extensionName) +} diff --git a/pkg/fake/virtualmachinesapi.go b/pkg/fake/virtualmachinesapi.go new file mode 100644 index 000000000..2066f20de --- /dev/null +++ b/pkg/fake/virtualmachinesapi.go @@ -0,0 +1,193 @@ +/* +Portions Copyright (c) Microsoft Corporation. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package fake + +import ( + "context" + "sync" + "time" + + "github.com/Azure/azure-sdk-for-go-extensions/pkg/errors" + "github.com/Azure/azure-sdk-for-go/sdk/azcore" + "github.com/Azure/azure-sdk-for-go/sdk/azcore/runtime" + "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute" + "github.com/Azure/go-autorest/autorest/to" + "github.com/Azure/karpenter/pkg/providers/instance" + "github.com/Azure/karpenter/pkg/utils" + "github.com/samber/lo" +) + +type VirtualMachineCreateOrUpdateInput struct { + ResourceGroupName string + VMName string + VM armcompute.VirtualMachine + Options *armcompute.VirtualMachinesClientBeginCreateOrUpdateOptions +} + +type VirtualMachineUpdateInput struct { + ResourceGroupName string + VMName string + Updates armcompute.VirtualMachineUpdate + Options *armcompute.VirtualMachinesClientBeginUpdateOptions +} + +type VirtualMachineDeleteInput struct { + ResourceGroupName string + VMName string + Options *armcompute.VirtualMachinesClientBeginDeleteOptions +} + +type VirtualMachineGetInput struct { + ResourceGroupName string + VMName string + Options *armcompute.VirtualMachinesClientGetOptions +} + +type VirtualMachinesBehavior struct { + VirtualMachineCreateOrUpdateBehavior MockedLRO[VirtualMachineCreateOrUpdateInput, armcompute.VirtualMachinesClientCreateOrUpdateResponse] + VirtualMachineUpdateBehavior MockedLRO[VirtualMachineUpdateInput, armcompute.VirtualMachinesClientUpdateResponse] + VirtualMachineDeleteBehavior MockedLRO[VirtualMachineDeleteInput, armcompute.VirtualMachinesClientDeleteResponse] + VirtualMachineGetBehavior MockedFunction[VirtualMachineGetInput, armcompute.VirtualMachinesClientGetResponse] + Instances sync.Map +} + +// assert that the fake implements the interface +var _ instance.VirtualMachinesAPI = (*VirtualMachinesAPI)(nil) + +type VirtualMachinesAPI struct { + // TODO: document the implications of embedding vs. not embedding the interface here + // instance.VirtualMachinesAPI // - this is the interface we are mocking. + VirtualMachinesBehavior +} + +// Reset must be called between tests otherwise tests will pollute each other. +func (c *VirtualMachinesAPI) Reset() { + c.VirtualMachineCreateOrUpdateBehavior.Reset() + c.VirtualMachineDeleteBehavior.Reset() + c.VirtualMachineGetBehavior.Reset() + c.VirtualMachineUpdateBehavior.Reset() + c.Instances.Range(func(k, v any) bool { + c.Instances.Delete(k) + return true + }) +} + +func (c *VirtualMachinesAPI) BeginCreateOrUpdate(_ context.Context, resourceGroupName string, vmName string, parameters armcompute.VirtualMachine, options *armcompute.VirtualMachinesClientBeginCreateOrUpdateOptions) (*runtime.Poller[armcompute.VirtualMachinesClientCreateOrUpdateResponse], error) { + // gather input parameters (may get rid of this with multiple mocked function signatures to reflect common patterns) + input := &VirtualMachineCreateOrUpdateInput{ + ResourceGroupName: resourceGroupName, + VMName: vmName, + VM: parameters, + Options: options, + } + + return c.VirtualMachineCreateOrUpdateBehavior.Invoke(input, func(input *VirtualMachineCreateOrUpdateInput) (*armcompute.VirtualMachinesClientCreateOrUpdateResponse, error) { + // example of input validation + //if input.ResourceGroupName == "" { + // return nil, errors.New("ResourceGroupName is required") + //} + // TODO: may have to clone ... + // TODO: subscription ID? + vm := input.VM + id := utils.MkVMID(input.ResourceGroupName, input.VMName) + vm.ID = to.StringPtr(id) + vm.Name = to.StringPtr(input.VMName) + if vm.Properties == nil { + vm.Properties = &armcompute.VirtualMachineProperties{} + } + if vm.Properties.TimeCreated == nil { + vm.Properties.TimeCreated = lo.ToPtr(time.Now()) // TODO: use simulated time? + } + c.Instances.Store(id, vm) + return &armcompute.VirtualMachinesClientCreateOrUpdateResponse{ + VirtualMachine: vm, + }, nil + }) +} + +func (c *VirtualMachinesAPI) BeginUpdate(_ context.Context, resourceGroupName string, vmName string, updates armcompute.VirtualMachineUpdate, options *armcompute.VirtualMachinesClientBeginUpdateOptions) (*runtime.Poller[armcompute.VirtualMachinesClientUpdateResponse], error) { + input := &VirtualMachineUpdateInput{ + ResourceGroupName: resourceGroupName, + VMName: vmName, + Updates: updates, + Options: options, + } + return c.VirtualMachineUpdateBehavior.Invoke(input, func(input *VirtualMachineUpdateInput) (*armcompute.VirtualMachinesClientUpdateResponse, error) { + id := utils.MkVMID(input.ResourceGroupName, input.VMName) + + instance, ok := c.Instances.Load(id) + if !ok { + return nil, &azcore.ResponseError{ErrorCode: errors.ResourceNotFound} + } + vm := instance.(armcompute.VirtualMachine) + + // If other fields need to be updated in the future, you can similarly + // update the VM object by merging with updates.. + vm.Tags = updates.Tags + if updates.Identity != nil { + if vm.Identity == nil { + vm.Identity = &armcompute.VirtualMachineIdentity{} + } + + if updates.Identity.Type != nil { + vm.Identity.Type = updates.Identity.Type + } + if len(updates.Identity.UserAssignedIdentities) > 0 { + if vm.Identity.UserAssignedIdentities == nil { + vm.Identity.UserAssignedIdentities = make(map[string]*armcompute.UserAssignedIdentitiesValue) + } + for id, val := range updates.Identity.UserAssignedIdentities { + vm.Identity.UserAssignedIdentities[id] = val + } + } + } + + // Update the stored shape + c.Instances.Store(id, vm) + + return &armcompute.VirtualMachinesClientUpdateResponse{VirtualMachine: vm}, nil + }) +} + +func (c *VirtualMachinesAPI) Get(_ context.Context, resourceGroupName string, vmName string, options *armcompute.VirtualMachinesClientGetOptions) (armcompute.VirtualMachinesClientGetResponse, error) { + input := &VirtualMachineGetInput{ + ResourceGroupName: resourceGroupName, + VMName: vmName, + Options: options, + } + return c.VirtualMachineGetBehavior.Invoke(input, func(input *VirtualMachineGetInput) (armcompute.VirtualMachinesClientGetResponse, error) { + instance, ok := c.Instances.Load(utils.MkVMID(input.ResourceGroupName, input.VMName)) + if !ok { + return armcompute.VirtualMachinesClientGetResponse{}, &azcore.ResponseError{ErrorCode: errors.ResourceNotFound} + } + return armcompute.VirtualMachinesClientGetResponse{ + VirtualMachine: instance.(armcompute.VirtualMachine), + }, nil + }) +} + +func (c *VirtualMachinesAPI) BeginDelete(_ context.Context, resourceGroupName string, vmName string, options *armcompute.VirtualMachinesClientBeginDeleteOptions) (*runtime.Poller[armcompute.VirtualMachinesClientDeleteResponse], error) { + input := &VirtualMachineDeleteInput{ + ResourceGroupName: resourceGroupName, + VMName: vmName, + Options: options, + } + return c.VirtualMachineDeleteBehavior.Invoke(input, func(input *VirtualMachineDeleteInput) (*armcompute.VirtualMachinesClientDeleteResponse, error) { + c.Instances.Delete(utils.MkVMID(input.ResourceGroupName, input.VMName)) + return &armcompute.VirtualMachinesClientDeleteResponse{}, nil + }) +} diff --git a/pkg/fake/virtualmachinesapi_test.go b/pkg/fake/virtualmachinesapi_test.go new file mode 100644 index 000000000..9c432b90c --- /dev/null +++ b/pkg/fake/virtualmachinesapi_test.go @@ -0,0 +1,48 @@ +/* +Portions Copyright (c) Microsoft Corporation. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package fake + +import ( + "context" + "testing" + + "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute" + "github.com/Azure/karpenter/pkg/providers/instance" +) + +// test BeginCreateOrUpdate +func TestComputeAPI_BeginCreateOrUpdate(t *testing.T) { + // setup + computeAPI := &VirtualMachinesAPI{} + //computeAPI.VirtualMachineCreateOrUpdateBehavior.Set(func(input *VirtualMachineCreateOrUpdateInput) (*runtime.Poller[armcompute.VirtualMachinesClientCreateOrUpdateResponse], error) { + // return nil, nil + //}) + // test + vm, err := instance.CreateVirtualMachine(context.Background(), computeAPI, "resourceGroupName", "vmName", armcompute.VirtualMachine{}) + // verify + if err != nil { + t.Errorf("Unexpected error %v", err) + return + } + if vm == nil { + t.Errorf("Unexpected nil vm") + return + } + if vm.ID == nil || *(vm.ID) != "/subscriptions/subscriptionID/resourceGroups/resourceGroupName/providers/Microsoft.Compute/virtualMachines/vmName" { + t.Errorf("Unexpected vm.ID %v", vm.ID) + } +} diff --git a/pkg/fake/zz_generated.sku.go b/pkg/fake/zz_generated.sku.go new file mode 100644 index 000000000..62a8ef09b --- /dev/null +++ b/pkg/fake/zz_generated.sku.go @@ -0,0 +1,582 @@ +//go:build !ignore_autogenerated + +/* +Portions Copyright (c) Microsoft Corporation. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package fake + +import ( + "github.com/samber/lo" + // nolint SA1019 - deprecated package + "github.com/Azure/azure-sdk-for-go/services/compute/mgmt/2022-08-01/compute" +) + +// generated at 2023-11-04T06:56:34Z + +// ResourceSkus is a list of all available skus from azure +var ResourceSkus = []compute.ResourceSku{ + { + Name: lo.ToPtr("Standard_A0"), + Tier: lo.ToPtr("Standard"), + Kind: lo.ToPtr(""), + Size: lo.ToPtr("A0"), + Family: lo.ToPtr("standardA0_A7Family"), + ResourceType: lo.ToPtr("virtualMachines"), + APIVersions: &[]string{}, + Costs: &[]compute.ResourceSkuCosts{}, + Restrictions: &[]compute.ResourceSkuRestrictions{ + { + Values: &[]string{"eastus"}}, + }, + Capabilities: &[]compute.ResourceSkuCapabilities{ + {Name: lo.ToPtr("MaxResourceVolumeMB"), Value: lo.ToPtr("20480")}, + {Name: lo.ToPtr("OSVhdSizeMB"), Value: lo.ToPtr("1047552")}, + {Name: lo.ToPtr("vCPUs"), Value: lo.ToPtr("1")}, + {Name: lo.ToPtr("MemoryPreservingMaintenanceSupported"), Value: lo.ToPtr("True")}, + {Name: lo.ToPtr("HyperVGenerations"), Value: lo.ToPtr("V1")}, + {Name: lo.ToPtr("MemoryGB"), Value: lo.ToPtr("0.75")}, + {Name: lo.ToPtr("MaxDataDiskCount"), Value: lo.ToPtr("1")}, + {Name: lo.ToPtr("CpuArchitectureType"), Value: lo.ToPtr("x64")}, + {Name: lo.ToPtr("LowPriorityCapable"), Value: lo.ToPtr("False")}, + {Name: lo.ToPtr("PremiumIO"), Value: lo.ToPtr("False")}, + {Name: lo.ToPtr("VMDeploymentTypes"), Value: lo.ToPtr("IaaS,PaaS")}, + {Name: lo.ToPtr("vCPUsAvailable"), Value: lo.ToPtr("1")}, + {Name: lo.ToPtr("ACUs"), Value: lo.ToPtr("50")}, + {Name: lo.ToPtr("vCPUsPerCore"), Value: lo.ToPtr("1")}, + {Name: lo.ToPtr("EphemeralOSDiskSupported"), Value: lo.ToPtr("False")}, + {Name: lo.ToPtr("EncryptionAtHostSupported"), Value: lo.ToPtr("False")}, + {Name: lo.ToPtr("CapacityReservationSupported"), Value: lo.ToPtr("False")}, + {Name: lo.ToPtr("AcceleratedNetworkingEnabled"), Value: lo.ToPtr("False")}, + {Name: lo.ToPtr("RdmaEnabled"), Value: lo.ToPtr("False")}, + {Name: lo.ToPtr("MaxNetworkInterfaces"), Value: lo.ToPtr("2")}, + }, + Locations: &[]string{"eastus"}, + LocationInfo: &[]compute.ResourceSkuLocationInfo{{Location: lo.ToPtr("eastus"), Zones: &[]string{}}}, + }, + { + Name: lo.ToPtr("Standard_B1s"), + Tier: lo.ToPtr("Standard"), + Kind: lo.ToPtr(""), + Size: lo.ToPtr("B1s"), + Family: lo.ToPtr("standardBSFamily"), + ResourceType: lo.ToPtr("virtualMachines"), + APIVersions: &[]string{}, + Costs: &[]compute.ResourceSkuCosts{}, + Restrictions: &[]compute.ResourceSkuRestrictions{}, + Capabilities: &[]compute.ResourceSkuCapabilities{ + {Name: lo.ToPtr("MaxResourceVolumeMB"), Value: lo.ToPtr("4096")}, + {Name: lo.ToPtr("OSVhdSizeMB"), Value: lo.ToPtr("1047552")}, + {Name: lo.ToPtr("vCPUs"), Value: lo.ToPtr("1")}, + {Name: lo.ToPtr("MemoryPreservingMaintenanceSupported"), Value: lo.ToPtr("True")}, + {Name: lo.ToPtr("HyperVGenerations"), Value: lo.ToPtr("V1,V2")}, + {Name: lo.ToPtr("MemoryGB"), Value: lo.ToPtr("1")}, + {Name: lo.ToPtr("MaxDataDiskCount"), Value: lo.ToPtr("2")}, + {Name: lo.ToPtr("CpuArchitectureType"), Value: lo.ToPtr("x64")}, + {Name: lo.ToPtr("LowPriorityCapable"), Value: lo.ToPtr("False")}, + {Name: lo.ToPtr("PremiumIO"), Value: lo.ToPtr("True")}, + {Name: lo.ToPtr("VMDeploymentTypes"), Value: lo.ToPtr("IaaS")}, + {Name: lo.ToPtr("vCPUsAvailable"), Value: lo.ToPtr("1")}, + {Name: lo.ToPtr("vCPUsPerCore"), Value: lo.ToPtr("1")}, + {Name: lo.ToPtr("CombinedTempDiskAndCachedIOPS"), Value: lo.ToPtr("400")}, + {Name: lo.ToPtr("CombinedTempDiskAndCachedReadBytesPerSecond"), Value: lo.ToPtr("22528000")}, + {Name: lo.ToPtr("CombinedTempDiskAndCachedWriteBytesPerSecond"), Value: lo.ToPtr("22528000")}, + {Name: lo.ToPtr("CachedDiskBytes"), Value: lo.ToPtr("32212254720")}, + {Name: lo.ToPtr("UncachedDiskIOPS"), Value: lo.ToPtr("320")}, + {Name: lo.ToPtr("UncachedDiskBytesPerSecond"), Value: lo.ToPtr("22500000")}, + {Name: lo.ToPtr("EphemeralOSDiskSupported"), Value: lo.ToPtr("True")}, + {Name: lo.ToPtr("EncryptionAtHostSupported"), Value: lo.ToPtr("True")}, + {Name: lo.ToPtr("CapacityReservationSupported"), Value: lo.ToPtr("False")}, + {Name: lo.ToPtr("AcceleratedNetworkingEnabled"), Value: lo.ToPtr("False")}, + {Name: lo.ToPtr("RdmaEnabled"), Value: lo.ToPtr("False")}, + {Name: lo.ToPtr("MaxNetworkInterfaces"), Value: lo.ToPtr("2")}, + }, + Locations: &[]string{"eastus"}, + LocationInfo: &[]compute.ResourceSkuLocationInfo{{Location: lo.ToPtr("eastus"), Zones: &[]string{ + "1", + "3", + "2", + }, + }, + }, + }, + { + Name: lo.ToPtr("Standard_D2s_v3"), + Tier: lo.ToPtr("Standard"), + Kind: lo.ToPtr(""), + Size: lo.ToPtr("D2s_v3"), + Family: lo.ToPtr("standardDSv3Family"), + ResourceType: lo.ToPtr("virtualMachines"), + APIVersions: &[]string{}, + Costs: &[]compute.ResourceSkuCosts{}, + Restrictions: &[]compute.ResourceSkuRestrictions{}, + Capabilities: &[]compute.ResourceSkuCapabilities{ + {Name: lo.ToPtr("MaxResourceVolumeMB"), Value: lo.ToPtr("16384")}, + {Name: lo.ToPtr("OSVhdSizeMB"), Value: lo.ToPtr("1047552")}, + {Name: lo.ToPtr("vCPUs"), Value: lo.ToPtr("2")}, + {Name: lo.ToPtr("MemoryPreservingMaintenanceSupported"), Value: lo.ToPtr("True")}, + {Name: lo.ToPtr("HyperVGenerations"), Value: lo.ToPtr("V1,V2")}, + {Name: lo.ToPtr("MemoryGB"), Value: lo.ToPtr("8")}, + {Name: lo.ToPtr("MaxDataDiskCount"), Value: lo.ToPtr("4")}, + {Name: lo.ToPtr("CpuArchitectureType"), Value: lo.ToPtr("x64")}, + {Name: lo.ToPtr("LowPriorityCapable"), Value: lo.ToPtr("True")}, + {Name: lo.ToPtr("PremiumIO"), Value: lo.ToPtr("True")}, + {Name: lo.ToPtr("VMDeploymentTypes"), Value: lo.ToPtr("IaaS")}, + {Name: lo.ToPtr("vCPUsAvailable"), Value: lo.ToPtr("2")}, + {Name: lo.ToPtr("ACUs"), Value: lo.ToPtr("160")}, + {Name: lo.ToPtr("vCPUsPerCore"), Value: lo.ToPtr("2")}, + {Name: lo.ToPtr("CombinedTempDiskAndCachedIOPS"), Value: lo.ToPtr("4000")}, + {Name: lo.ToPtr("CombinedTempDiskAndCachedReadBytesPerSecond"), Value: lo.ToPtr("33554432")}, + {Name: lo.ToPtr("CombinedTempDiskAndCachedWriteBytesPerSecond"), Value: lo.ToPtr("33554432")}, + {Name: lo.ToPtr("CachedDiskBytes"), Value: lo.ToPtr("53687091200")}, + {Name: lo.ToPtr("UncachedDiskIOPS"), Value: lo.ToPtr("3200")}, + {Name: lo.ToPtr("UncachedDiskBytesPerSecond"), Value: lo.ToPtr("50331648")}, + {Name: lo.ToPtr("EphemeralOSDiskSupported"), Value: lo.ToPtr("True")}, + {Name: lo.ToPtr("EncryptionAtHostSupported"), Value: lo.ToPtr("True")}, + {Name: lo.ToPtr("CapacityReservationSupported"), Value: lo.ToPtr("False")}, + {Name: lo.ToPtr("AcceleratedNetworkingEnabled"), Value: lo.ToPtr("True")}, + {Name: lo.ToPtr("RdmaEnabled"), Value: lo.ToPtr("False")}, + {Name: lo.ToPtr("MaxNetworkInterfaces"), Value: lo.ToPtr("2")}, + }, + Locations: &[]string{"eastus"}, + LocationInfo: &[]compute.ResourceSkuLocationInfo{{Location: lo.ToPtr("eastus"), Zones: &[]string{ + "1", + "3", + "2", + }, + }, + }, + }, + { + Name: lo.ToPtr("Standard_D2_v2"), + Tier: lo.ToPtr("Standard"), + Kind: lo.ToPtr(""), + Size: lo.ToPtr("D2_v2"), + Family: lo.ToPtr("standardDv2Family"), + ResourceType: lo.ToPtr("virtualMachines"), + APIVersions: &[]string{}, + Costs: &[]compute.ResourceSkuCosts{}, + Restrictions: &[]compute.ResourceSkuRestrictions{}, + Capabilities: &[]compute.ResourceSkuCapabilities{ + {Name: lo.ToPtr("MaxResourceVolumeMB"), Value: lo.ToPtr("102400")}, + {Name: lo.ToPtr("OSVhdSizeMB"), Value: lo.ToPtr("1047552")}, + {Name: lo.ToPtr("vCPUs"), Value: lo.ToPtr("2")}, + {Name: lo.ToPtr("MemoryPreservingMaintenanceSupported"), Value: lo.ToPtr("True")}, + {Name: lo.ToPtr("HyperVGenerations"), Value: lo.ToPtr("V1")}, + {Name: lo.ToPtr("MemoryGB"), Value: lo.ToPtr("7")}, + {Name: lo.ToPtr("MaxDataDiskCount"), Value: lo.ToPtr("8")}, + {Name: lo.ToPtr("CpuArchitectureType"), Value: lo.ToPtr("x64")}, + {Name: lo.ToPtr("LowPriorityCapable"), Value: lo.ToPtr("True")}, + {Name: lo.ToPtr("PremiumIO"), Value: lo.ToPtr("False")}, + {Name: lo.ToPtr("VMDeploymentTypes"), Value: lo.ToPtr("IaaS,PaaS")}, + {Name: lo.ToPtr("vCPUsAvailable"), Value: lo.ToPtr("2")}, + {Name: lo.ToPtr("ACUs"), Value: lo.ToPtr("210")}, + {Name: lo.ToPtr("vCPUsPerCore"), Value: lo.ToPtr("1")}, + {Name: lo.ToPtr("CombinedTempDiskAndCachedIOPS"), Value: lo.ToPtr("6000")}, + {Name: lo.ToPtr("CombinedTempDiskAndCachedReadBytesPerSecond"), Value: lo.ToPtr("97517568")}, + {Name: lo.ToPtr("CombinedTempDiskAndCachedWriteBytesPerSecond"), Value: lo.ToPtr("48234496")}, + {Name: lo.ToPtr("EphemeralOSDiskSupported"), Value: lo.ToPtr("False")}, + {Name: lo.ToPtr("EncryptionAtHostSupported"), Value: lo.ToPtr("False")}, + {Name: lo.ToPtr("CapacityReservationSupported"), Value: lo.ToPtr("False")}, + {Name: lo.ToPtr("AcceleratedNetworkingEnabled"), Value: lo.ToPtr("True")}, + {Name: lo.ToPtr("RdmaEnabled"), Value: lo.ToPtr("False")}, + {Name: lo.ToPtr("MaxNetworkInterfaces"), Value: lo.ToPtr("2")}, + }, + Locations: &[]string{"eastus"}, + LocationInfo: &[]compute.ResourceSkuLocationInfo{{Location: lo.ToPtr("eastus"), Zones: &[]string{ + "1", + "3", + "2", + }, + }, + }, + }, + { + Name: lo.ToPtr("Standard_D2_v3"), + Tier: lo.ToPtr("Standard"), + Kind: lo.ToPtr(""), + Size: lo.ToPtr("D2_v3"), + Family: lo.ToPtr("standardDv3Family"), + ResourceType: lo.ToPtr("virtualMachines"), + APIVersions: &[]string{}, + Costs: &[]compute.ResourceSkuCosts{}, + Restrictions: &[]compute.ResourceSkuRestrictions{}, + Capabilities: &[]compute.ResourceSkuCapabilities{ + {Name: lo.ToPtr("MaxResourceVolumeMB"), Value: lo.ToPtr("51200")}, + {Name: lo.ToPtr("OSVhdSizeMB"), Value: lo.ToPtr("1047552")}, + {Name: lo.ToPtr("vCPUs"), Value: lo.ToPtr("2")}, + {Name: lo.ToPtr("MemoryPreservingMaintenanceSupported"), Value: lo.ToPtr("True")}, + {Name: lo.ToPtr("HyperVGenerations"), Value: lo.ToPtr("V1")}, + {Name: lo.ToPtr("MemoryGB"), Value: lo.ToPtr("8")}, + {Name: lo.ToPtr("MaxDataDiskCount"), Value: lo.ToPtr("4")}, + {Name: lo.ToPtr("CpuArchitectureType"), Value: lo.ToPtr("x64")}, + {Name: lo.ToPtr("LowPriorityCapable"), Value: lo.ToPtr("True")}, + {Name: lo.ToPtr("PremiumIO"), Value: lo.ToPtr("False")}, + {Name: lo.ToPtr("VMDeploymentTypes"), Value: lo.ToPtr("IaaS,PaaS")}, + {Name: lo.ToPtr("vCPUsAvailable"), Value: lo.ToPtr("2")}, + {Name: lo.ToPtr("ACUs"), Value: lo.ToPtr("160")}, + {Name: lo.ToPtr("vCPUsPerCore"), Value: lo.ToPtr("2")}, + {Name: lo.ToPtr("CombinedTempDiskAndCachedIOPS"), Value: lo.ToPtr("3000")}, + {Name: lo.ToPtr("CombinedTempDiskAndCachedReadBytesPerSecond"), Value: lo.ToPtr("48234496")}, + {Name: lo.ToPtr("CombinedTempDiskAndCachedWriteBytesPerSecond"), Value: lo.ToPtr("24117248")}, + {Name: lo.ToPtr("EphemeralOSDiskSupported"), Value: lo.ToPtr("False")}, + {Name: lo.ToPtr("EncryptionAtHostSupported"), Value: lo.ToPtr("False")}, + {Name: lo.ToPtr("CapacityReservationSupported"), Value: lo.ToPtr("False")}, + {Name: lo.ToPtr("AcceleratedNetworkingEnabled"), Value: lo.ToPtr("True")}, + {Name: lo.ToPtr("RdmaEnabled"), Value: lo.ToPtr("False")}, + {Name: lo.ToPtr("MaxNetworkInterfaces"), Value: lo.ToPtr("2")}, + }, + Locations: &[]string{"eastus"}, + LocationInfo: &[]compute.ResourceSkuLocationInfo{{Location: lo.ToPtr("eastus"), Zones: &[]string{ + "1", + "3", + "2", + }, + }, + }, + }, + { + Name: lo.ToPtr("Standard_D2_v5"), + Tier: lo.ToPtr("Standard"), + Kind: lo.ToPtr(""), + Size: lo.ToPtr("D2_v5"), + Family: lo.ToPtr("standardDv5Family"), + ResourceType: lo.ToPtr("virtualMachines"), + APIVersions: &[]string{}, + Costs: &[]compute.ResourceSkuCosts{}, + Restrictions: &[]compute.ResourceSkuRestrictions{}, + Capabilities: &[]compute.ResourceSkuCapabilities{ + {Name: lo.ToPtr("MaxResourceVolumeMB"), Value: lo.ToPtr("0")}, + {Name: lo.ToPtr("OSVhdSizeMB"), Value: lo.ToPtr("1047552")}, + {Name: lo.ToPtr("vCPUs"), Value: lo.ToPtr("2")}, + {Name: lo.ToPtr("MemoryPreservingMaintenanceSupported"), Value: lo.ToPtr("True")}, + {Name: lo.ToPtr("HyperVGenerations"), Value: lo.ToPtr("V1,V2")}, + {Name: lo.ToPtr("MemoryGB"), Value: lo.ToPtr("8")}, + {Name: lo.ToPtr("MaxDataDiskCount"), Value: lo.ToPtr("4")}, + {Name: lo.ToPtr("CpuArchitectureType"), Value: lo.ToPtr("x64")}, + {Name: lo.ToPtr("LowPriorityCapable"), Value: lo.ToPtr("True")}, + {Name: lo.ToPtr("PremiumIO"), Value: lo.ToPtr("False")}, + {Name: lo.ToPtr("VMDeploymentTypes"), Value: lo.ToPtr("IaaS")}, + {Name: lo.ToPtr("vCPUsAvailable"), Value: lo.ToPtr("2")}, + {Name: lo.ToPtr("vCPUsPerCore"), Value: lo.ToPtr("2")}, + {Name: lo.ToPtr("CombinedTempDiskAndCachedIOPS"), Value: lo.ToPtr("19000")}, + {Name: lo.ToPtr("CombinedTempDiskAndCachedReadBytesPerSecond"), Value: lo.ToPtr("131072000")}, + {Name: lo.ToPtr("CombinedTempDiskAndCachedWriteBytesPerSecond"), Value: lo.ToPtr("131072000")}, + {Name: lo.ToPtr("UncachedDiskIOPS"), Value: lo.ToPtr("3750")}, + {Name: lo.ToPtr("UncachedDiskBytesPerSecond"), Value: lo.ToPtr("89128960")}, + {Name: lo.ToPtr("EphemeralOSDiskSupported"), Value: lo.ToPtr("False")}, + {Name: lo.ToPtr("EncryptionAtHostSupported"), Value: lo.ToPtr("True")}, + {Name: lo.ToPtr("CapacityReservationSupported"), Value: lo.ToPtr("False")}, + {Name: lo.ToPtr("AcceleratedNetworkingEnabled"), Value: lo.ToPtr("True")}, + {Name: lo.ToPtr("RdmaEnabled"), Value: lo.ToPtr("False")}, + {Name: lo.ToPtr("MaxNetworkInterfaces"), Value: lo.ToPtr("2")}, + }, + Locations: &[]string{"eastus"}, + LocationInfo: &[]compute.ResourceSkuLocationInfo{{Location: lo.ToPtr("eastus"), Zones: &[]string{ + "1", + "3", + "2", + }, + }, + }, + }, + { + Name: lo.ToPtr("Standard_D4s_v3"), + Tier: lo.ToPtr("Standard"), + Kind: lo.ToPtr(""), + Size: lo.ToPtr("D4s_v3"), + Family: lo.ToPtr("standardDSv3Family"), + ResourceType: lo.ToPtr("virtualMachines"), + APIVersions: &[]string{}, + Costs: &[]compute.ResourceSkuCosts{}, + Restrictions: &[]compute.ResourceSkuRestrictions{}, + Capabilities: &[]compute.ResourceSkuCapabilities{ + {Name: lo.ToPtr("MaxResourceVolumeMB"), Value: lo.ToPtr("32768")}, + {Name: lo.ToPtr("OSVhdSizeMB"), Value: lo.ToPtr("1047552")}, + {Name: lo.ToPtr("vCPUs"), Value: lo.ToPtr("4")}, + {Name: lo.ToPtr("MemoryPreservingMaintenanceSupported"), Value: lo.ToPtr("True")}, + {Name: lo.ToPtr("HyperVGenerations"), Value: lo.ToPtr("V1,V2")}, + {Name: lo.ToPtr("MemoryGB"), Value: lo.ToPtr("16")}, + {Name: lo.ToPtr("MaxDataDiskCount"), Value: lo.ToPtr("8")}, + {Name: lo.ToPtr("CpuArchitectureType"), Value: lo.ToPtr("x64")}, + {Name: lo.ToPtr("LowPriorityCapable"), Value: lo.ToPtr("True")}, + {Name: lo.ToPtr("PremiumIO"), Value: lo.ToPtr("True")}, + {Name: lo.ToPtr("VMDeploymentTypes"), Value: lo.ToPtr("IaaS")}, + {Name: lo.ToPtr("vCPUsAvailable"), Value: lo.ToPtr("4")}, + {Name: lo.ToPtr("ACUs"), Value: lo.ToPtr("160")}, + {Name: lo.ToPtr("vCPUsPerCore"), Value: lo.ToPtr("2")}, + {Name: lo.ToPtr("CombinedTempDiskAndCachedIOPS"), Value: lo.ToPtr("8000")}, + {Name: lo.ToPtr("CombinedTempDiskAndCachedReadBytesPerSecond"), Value: lo.ToPtr("67108864")}, + {Name: lo.ToPtr("CombinedTempDiskAndCachedWriteBytesPerSecond"), Value: lo.ToPtr("67108864")}, + {Name: lo.ToPtr("CachedDiskBytes"), Value: lo.ToPtr("107374182400")}, + {Name: lo.ToPtr("UncachedDiskIOPS"), Value: lo.ToPtr("6400")}, + {Name: lo.ToPtr("UncachedDiskBytesPerSecond"), Value: lo.ToPtr("100663296")}, + {Name: lo.ToPtr("EphemeralOSDiskSupported"), Value: lo.ToPtr("True")}, + {Name: lo.ToPtr("EncryptionAtHostSupported"), Value: lo.ToPtr("True")}, + {Name: lo.ToPtr("CapacityReservationSupported"), Value: lo.ToPtr("False")}, + {Name: lo.ToPtr("AcceleratedNetworkingEnabled"), Value: lo.ToPtr("True")}, + {Name: lo.ToPtr("RdmaEnabled"), Value: lo.ToPtr("False")}, + {Name: lo.ToPtr("MaxNetworkInterfaces"), Value: lo.ToPtr("2")}, + }, + Locations: &[]string{"eastus"}, + LocationInfo: &[]compute.ResourceSkuLocationInfo{{Location: lo.ToPtr("eastus"), Zones: &[]string{ + "1", + "3", + "2", + }, + }, + }, + }, + { + Name: lo.ToPtr("Standard_D64s_v3"), + Tier: lo.ToPtr("Standard"), + Kind: lo.ToPtr(""), + Size: lo.ToPtr("D64s_v3"), + Family: lo.ToPtr("standardDSv3Family"), + ResourceType: lo.ToPtr("virtualMachines"), + APIVersions: &[]string{}, + Costs: &[]compute.ResourceSkuCosts{}, + Restrictions: &[]compute.ResourceSkuRestrictions{}, + Capabilities: &[]compute.ResourceSkuCapabilities{ + {Name: lo.ToPtr("MaxResourceVolumeMB"), Value: lo.ToPtr("524288")}, + {Name: lo.ToPtr("OSVhdSizeMB"), Value: lo.ToPtr("1047552")}, + {Name: lo.ToPtr("vCPUs"), Value: lo.ToPtr("64")}, + {Name: lo.ToPtr("MemoryPreservingMaintenanceSupported"), Value: lo.ToPtr("True")}, + {Name: lo.ToPtr("HyperVGenerations"), Value: lo.ToPtr("V1,V2")}, + {Name: lo.ToPtr("MemoryGB"), Value: lo.ToPtr("256")}, + {Name: lo.ToPtr("MaxDataDiskCount"), Value: lo.ToPtr("32")}, + {Name: lo.ToPtr("CpuArchitectureType"), Value: lo.ToPtr("x64")}, + {Name: lo.ToPtr("LowPriorityCapable"), Value: lo.ToPtr("True")}, + {Name: lo.ToPtr("PremiumIO"), Value: lo.ToPtr("True")}, + {Name: lo.ToPtr("VMDeploymentTypes"), Value: lo.ToPtr("IaaS")}, + {Name: lo.ToPtr("vCPUsAvailable"), Value: lo.ToPtr("64")}, + {Name: lo.ToPtr("ACUs"), Value: lo.ToPtr("160")}, + {Name: lo.ToPtr("vCPUsPerCore"), Value: lo.ToPtr("2")}, + {Name: lo.ToPtr("CombinedTempDiskAndCachedIOPS"), Value: lo.ToPtr("128000")}, + {Name: lo.ToPtr("CombinedTempDiskAndCachedReadBytesPerSecond"), Value: lo.ToPtr("1073741824")}, + {Name: lo.ToPtr("CombinedTempDiskAndCachedWriteBytesPerSecond"), Value: lo.ToPtr("1073741824")}, + {Name: lo.ToPtr("CachedDiskBytes"), Value: lo.ToPtr("1717986918400")}, + {Name: lo.ToPtr("UncachedDiskIOPS"), Value: lo.ToPtr("80000")}, + {Name: lo.ToPtr("UncachedDiskBytesPerSecond"), Value: lo.ToPtr("1258291200")}, + {Name: lo.ToPtr("EphemeralOSDiskSupported"), Value: lo.ToPtr("True")}, + {Name: lo.ToPtr("EncryptionAtHostSupported"), Value: lo.ToPtr("True")}, + {Name: lo.ToPtr("CapacityReservationSupported"), Value: lo.ToPtr("False")}, + {Name: lo.ToPtr("AcceleratedNetworkingEnabled"), Value: lo.ToPtr("True")}, + {Name: lo.ToPtr("RdmaEnabled"), Value: lo.ToPtr("False")}, + {Name: lo.ToPtr("MaxNetworkInterfaces"), Value: lo.ToPtr("8")}, + }, + Locations: &[]string{"eastus"}, + LocationInfo: &[]compute.ResourceSkuLocationInfo{{Location: lo.ToPtr("eastus"), Zones: &[]string{ + "1", + "3", + "2", + }, + }, + }, + }, + { + Name: lo.ToPtr("Standard_DS2_v2"), + Tier: lo.ToPtr("Standard"), + Kind: lo.ToPtr(""), + Size: lo.ToPtr("DS2_v2"), + Family: lo.ToPtr("standardDSv2Family"), + ResourceType: lo.ToPtr("virtualMachines"), + APIVersions: &[]string{}, + Costs: &[]compute.ResourceSkuCosts{}, + Restrictions: &[]compute.ResourceSkuRestrictions{}, + Capabilities: &[]compute.ResourceSkuCapabilities{ + {Name: lo.ToPtr("MaxResourceVolumeMB"), Value: lo.ToPtr("14336")}, + {Name: lo.ToPtr("OSVhdSizeMB"), Value: lo.ToPtr("1047552")}, + {Name: lo.ToPtr("vCPUs"), Value: lo.ToPtr("2")}, + {Name: lo.ToPtr("MemoryPreservingMaintenanceSupported"), Value: lo.ToPtr("True")}, + {Name: lo.ToPtr("HyperVGenerations"), Value: lo.ToPtr("V1,V2")}, + {Name: lo.ToPtr("MemoryGB"), Value: lo.ToPtr("7")}, + {Name: lo.ToPtr("MaxDataDiskCount"), Value: lo.ToPtr("8")}, + {Name: lo.ToPtr("CpuArchitectureType"), Value: lo.ToPtr("x64")}, + {Name: lo.ToPtr("LowPriorityCapable"), Value: lo.ToPtr("True")}, + {Name: lo.ToPtr("PremiumIO"), Value: lo.ToPtr("True")}, + {Name: lo.ToPtr("VMDeploymentTypes"), Value: lo.ToPtr("IaaS")}, + {Name: lo.ToPtr("vCPUsAvailable"), Value: lo.ToPtr("2")}, + {Name: lo.ToPtr("ACUs"), Value: lo.ToPtr("210")}, + {Name: lo.ToPtr("vCPUsPerCore"), Value: lo.ToPtr("1")}, + {Name: lo.ToPtr("CombinedTempDiskAndCachedIOPS"), Value: lo.ToPtr("8000")}, + {Name: lo.ToPtr("CombinedTempDiskAndCachedReadBytesPerSecond"), Value: lo.ToPtr("65536000")}, + {Name: lo.ToPtr("CombinedTempDiskAndCachedWriteBytesPerSecond"), Value: lo.ToPtr("65536000")}, + {Name: lo.ToPtr("CachedDiskBytes"), Value: lo.ToPtr("92341796864")}, + {Name: lo.ToPtr("UncachedDiskIOPS"), Value: lo.ToPtr("6400")}, + {Name: lo.ToPtr("UncachedDiskBytesPerSecond"), Value: lo.ToPtr("96000000")}, + {Name: lo.ToPtr("EphemeralOSDiskSupported"), Value: lo.ToPtr("True")}, + {Name: lo.ToPtr("EncryptionAtHostSupported"), Value: lo.ToPtr("True")}, + {Name: lo.ToPtr("CapacityReservationSupported"), Value: lo.ToPtr("False")}, + {Name: lo.ToPtr("AcceleratedNetworkingEnabled"), Value: lo.ToPtr("True")}, + {Name: lo.ToPtr("RdmaEnabled"), Value: lo.ToPtr("False")}, + {Name: lo.ToPtr("MaxNetworkInterfaces"), Value: lo.ToPtr("2")}, + }, + Locations: &[]string{"eastus"}, + LocationInfo: &[]compute.ResourceSkuLocationInfo{{Location: lo.ToPtr("eastus"), Zones: &[]string{ + "1", + "3", + "2", + }, + }, + }, + }, + { + Name: lo.ToPtr("Standard_F16s_v2"), + Tier: lo.ToPtr("Standard"), + Kind: lo.ToPtr(""), + Size: lo.ToPtr("F16s_v2"), + Family: lo.ToPtr("standardFSv2Family"), + ResourceType: lo.ToPtr("virtualMachines"), + APIVersions: &[]string{}, + Costs: &[]compute.ResourceSkuCosts{}, + Restrictions: &[]compute.ResourceSkuRestrictions{}, + Capabilities: &[]compute.ResourceSkuCapabilities{ + {Name: lo.ToPtr("MaxResourceVolumeMB"), Value: lo.ToPtr("131072")}, + {Name: lo.ToPtr("OSVhdSizeMB"), Value: lo.ToPtr("1047552")}, + {Name: lo.ToPtr("vCPUs"), Value: lo.ToPtr("16")}, + {Name: lo.ToPtr("MemoryPreservingMaintenanceSupported"), Value: lo.ToPtr("True")}, + {Name: lo.ToPtr("HyperVGenerations"), Value: lo.ToPtr("V1,V2")}, + {Name: lo.ToPtr("MemoryGB"), Value: lo.ToPtr("32")}, + {Name: lo.ToPtr("MaxDataDiskCount"), Value: lo.ToPtr("32")}, + {Name: lo.ToPtr("CpuArchitectureType"), Value: lo.ToPtr("x64")}, + {Name: lo.ToPtr("LowPriorityCapable"), Value: lo.ToPtr("True")}, + {Name: lo.ToPtr("PremiumIO"), Value: lo.ToPtr("True")}, + {Name: lo.ToPtr("VMDeploymentTypes"), Value: lo.ToPtr("IaaS")}, + {Name: lo.ToPtr("vCPUsAvailable"), Value: lo.ToPtr("16")}, + {Name: lo.ToPtr("ACUs"), Value: lo.ToPtr("195")}, + {Name: lo.ToPtr("vCPUsPerCore"), Value: lo.ToPtr("2")}, + {Name: lo.ToPtr("CombinedTempDiskAndCachedIOPS"), Value: lo.ToPtr("32000")}, + {Name: lo.ToPtr("CombinedTempDiskAndCachedReadBytesPerSecond"), Value: lo.ToPtr("262144000")}, + {Name: lo.ToPtr("CombinedTempDiskAndCachedWriteBytesPerSecond"), Value: lo.ToPtr("262144000")}, + {Name: lo.ToPtr("CachedDiskBytes"), Value: lo.ToPtr("274877906944")}, + {Name: lo.ToPtr("UncachedDiskIOPS"), Value: lo.ToPtr("25600")}, + {Name: lo.ToPtr("UncachedDiskBytesPerSecond"), Value: lo.ToPtr("384000000")}, + {Name: lo.ToPtr("EphemeralOSDiskSupported"), Value: lo.ToPtr("True")}, + {Name: lo.ToPtr("EncryptionAtHostSupported"), Value: lo.ToPtr("True")}, + {Name: lo.ToPtr("CapacityReservationSupported"), Value: lo.ToPtr("False")}, + {Name: lo.ToPtr("AcceleratedNetworkingEnabled"), Value: lo.ToPtr("True")}, + {Name: lo.ToPtr("RdmaEnabled"), Value: lo.ToPtr("False")}, + {Name: lo.ToPtr("MaxNetworkInterfaces"), Value: lo.ToPtr("4")}, + }, + Locations: &[]string{"eastus"}, + LocationInfo: &[]compute.ResourceSkuLocationInfo{{Location: lo.ToPtr("eastus"), Zones: &[]string{ + "1", + "2", + "3", + }, + }, + }, + }, + { + Name: lo.ToPtr("Standard_M8-2ms"), + Tier: lo.ToPtr("Standard"), + Kind: lo.ToPtr(""), + Size: lo.ToPtr("M8-2ms"), + Family: lo.ToPtr("standardMSFamily"), + ResourceType: lo.ToPtr("virtualMachines"), + APIVersions: &[]string{}, + Costs: &[]compute.ResourceSkuCosts{}, + Restrictions: &[]compute.ResourceSkuRestrictions{ + { + Values: &[]string{"eastus"}}, + }, + Capabilities: &[]compute.ResourceSkuCapabilities{ + {Name: lo.ToPtr("MaxResourceVolumeMB"), Value: lo.ToPtr("256000")}, + {Name: lo.ToPtr("OSVhdSizeMB"), Value: lo.ToPtr("1047552")}, + {Name: lo.ToPtr("vCPUs"), Value: lo.ToPtr("8")}, + {Name: lo.ToPtr("MemoryPreservingMaintenanceSupported"), Value: lo.ToPtr("False")}, + {Name: lo.ToPtr("HyperVGenerations"), Value: lo.ToPtr("V1,V2")}, + {Name: lo.ToPtr("MemoryGB"), Value: lo.ToPtr("218.75")}, + {Name: lo.ToPtr("MaxDataDiskCount"), Value: lo.ToPtr("8")}, + {Name: lo.ToPtr("CpuArchitectureType"), Value: lo.ToPtr("x64")}, + {Name: lo.ToPtr("MaxWriteAcceleratorDisksAllowed"), Value: lo.ToPtr("1")}, + {Name: lo.ToPtr("LowPriorityCapable"), Value: lo.ToPtr("True")}, + {Name: lo.ToPtr("PremiumIO"), Value: lo.ToPtr("True")}, + {Name: lo.ToPtr("VMDeploymentTypes"), Value: lo.ToPtr("IaaS")}, + {Name: lo.ToPtr("vCPUsAvailable"), Value: lo.ToPtr("2")}, + {Name: lo.ToPtr("ACUs"), Value: lo.ToPtr("160")}, + {Name: lo.ToPtr("ParentSize"), Value: lo.ToPtr("Standard_M8ms")}, + {Name: lo.ToPtr("vCPUsPerCore"), Value: lo.ToPtr("2")}, + {Name: lo.ToPtr("CombinedTempDiskAndCachedIOPS"), Value: lo.ToPtr("10000")}, + {Name: lo.ToPtr("CombinedTempDiskAndCachedReadBytesPerSecond"), Value: lo.ToPtr("104857600")}, + {Name: lo.ToPtr("CombinedTempDiskAndCachedWriteBytesPerSecond"), Value: lo.ToPtr("104857600")}, + {Name: lo.ToPtr("CachedDiskBytes"), Value: lo.ToPtr("851477266432")}, + {Name: lo.ToPtr("UncachedDiskIOPS"), Value: lo.ToPtr("5000")}, + {Name: lo.ToPtr("UncachedDiskBytesPerSecond"), Value: lo.ToPtr("131072000")}, + {Name: lo.ToPtr("EphemeralOSDiskSupported"), Value: lo.ToPtr("True")}, + {Name: lo.ToPtr("EncryptionAtHostSupported"), Value: lo.ToPtr("True")}, + {Name: lo.ToPtr("CapacityReservationSupported"), Value: lo.ToPtr("False")}, + {Name: lo.ToPtr("TrustedLaunchDisabled"), Value: lo.ToPtr("True")}, + {Name: lo.ToPtr("AcceleratedNetworkingEnabled"), Value: lo.ToPtr("True")}, + {Name: lo.ToPtr("RdmaEnabled"), Value: lo.ToPtr("False")}, + {Name: lo.ToPtr("MaxNetworkInterfaces"), Value: lo.ToPtr("2")}, + }, + Locations: &[]string{"eastus"}, + LocationInfo: &[]compute.ResourceSkuLocationInfo{{Location: lo.ToPtr("eastus"), Zones: &[]string{ + "3", + "1", + "2", + }, + }, + }, + }, + { + Name: lo.ToPtr("Standard_NC24ads_A100_v4"), + Tier: lo.ToPtr("Standard"), + Kind: lo.ToPtr(""), + Size: lo.ToPtr("NC24ads_A100_v4"), + Family: lo.ToPtr("StandardNCADSA100v4Family"), + ResourceType: lo.ToPtr("virtualMachines"), + APIVersions: &[]string{}, + Costs: &[]compute.ResourceSkuCosts{}, + Restrictions: &[]compute.ResourceSkuRestrictions{}, + Capabilities: &[]compute.ResourceSkuCapabilities{ + {Name: lo.ToPtr("MaxResourceVolumeMB"), Value: lo.ToPtr("65536")}, + {Name: lo.ToPtr("OSVhdSizeMB"), Value: lo.ToPtr("1047552")}, + {Name: lo.ToPtr("vCPUs"), Value: lo.ToPtr("24")}, + {Name: lo.ToPtr("MemoryPreservingMaintenanceSupported"), Value: lo.ToPtr("False")}, + {Name: lo.ToPtr("HyperVGenerations"), Value: lo.ToPtr("V2")}, + {Name: lo.ToPtr("MemoryGB"), Value: lo.ToPtr("220")}, + {Name: lo.ToPtr("MaxDataDiskCount"), Value: lo.ToPtr("8")}, + {Name: lo.ToPtr("CpuArchitectureType"), Value: lo.ToPtr("x64")}, + {Name: lo.ToPtr("LowPriorityCapable"), Value: lo.ToPtr("True")}, + {Name: lo.ToPtr("PremiumIO"), Value: lo.ToPtr("True")}, + {Name: lo.ToPtr("VMDeploymentTypes"), Value: lo.ToPtr("IaaS")}, + {Name: lo.ToPtr("vCPUsAvailable"), Value: lo.ToPtr("24")}, + {Name: lo.ToPtr("GPUs"), Value: lo.ToPtr("1")}, + {Name: lo.ToPtr("vCPUsPerCore"), Value: lo.ToPtr("1")}, + {Name: lo.ToPtr("CombinedTempDiskAndCachedIOPS"), Value: lo.ToPtr("75000")}, + {Name: lo.ToPtr("CombinedTempDiskAndCachedReadBytesPerSecond"), Value: lo.ToPtr("1000000000")}, + {Name: lo.ToPtr("CombinedTempDiskAndCachedWriteBytesPerSecond"), Value: lo.ToPtr("1000000000")}, + {Name: lo.ToPtr("CachedDiskBytes"), Value: lo.ToPtr("274877906944")}, + {Name: lo.ToPtr("UncachedDiskIOPS"), Value: lo.ToPtr("25600")}, + {Name: lo.ToPtr("UncachedDiskBytesPerSecond"), Value: lo.ToPtr("384000000")}, + {Name: lo.ToPtr("NvmeDiskSizeInMiB"), Value: lo.ToPtr("915456")}, + {Name: lo.ToPtr("EphemeralOSDiskSupported"), Value: lo.ToPtr("True")}, + {Name: lo.ToPtr("EncryptionAtHostSupported"), Value: lo.ToPtr("True")}, + {Name: lo.ToPtr("CapacityReservationSupported"), Value: lo.ToPtr("False")}, + {Name: lo.ToPtr("AcceleratedNetworkingEnabled"), Value: lo.ToPtr("True")}, + {Name: lo.ToPtr("RdmaEnabled"), Value: lo.ToPtr("False")}, + {Name: lo.ToPtr("MaxNetworkInterfaces"), Value: lo.ToPtr("2")}, + }, + Locations: &[]string{"eastus"}, + LocationInfo: &[]compute.ResourceSkuLocationInfo{{Location: lo.ToPtr("eastus"), Zones: &[]string{ + "1", + }, + }, + }, + }, +} diff --git a/pkg/metrics/constants.go b/pkg/metrics/constants.go new file mode 100644 index 000000000..a05011edc --- /dev/null +++ b/pkg/metrics/constants.go @@ -0,0 +1,25 @@ +/* +Portions Copyright (c) Microsoft Corporation. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package metrics + +const ( + // Namespace(s). + Namespace = "karpenter" + + // Subsystem(s). + imageFamilySubsystem = "image" +) diff --git a/pkg/metrics/metrics.go b/pkg/metrics/metrics.go new file mode 100644 index 000000000..025225291 --- /dev/null +++ b/pkg/metrics/metrics.go @@ -0,0 +1,40 @@ +/* +Portions Copyright (c) Microsoft Corporation. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package metrics + +import ( + "github.com/prometheus/client_golang/prometheus" + crmetrics "sigs.k8s.io/controller-runtime/pkg/metrics" +) + +var ( + ImageSelectionErrorCount = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Namespace: Namespace, + Subsystem: imageFamilySubsystem, + Name: "selection_error_count", + Help: "The number of errors encountered while selecting an image.", + }, + []string{"family"}, + ) +) + +func init() { + crmetrics.Registry.MustRegister( + ImageSelectionErrorCount, + ) +} diff --git a/pkg/metrics/metrics_test.go b/pkg/metrics/metrics_test.go new file mode 100644 index 000000000..75ac81cc4 --- /dev/null +++ b/pkg/metrics/metrics_test.go @@ -0,0 +1,48 @@ +/* +Portions Copyright (c) Microsoft Corporation. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package metrics_test + +import ( + "testing" + + "github.com/Azure/karpenter/pkg/metrics" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "github.com/prometheus/client_golang/prometheus/testutil" +) + +func TestAzure(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "Metrics Suite") +} + +var _ = Describe("Image Selection Error Metrics", func() { + BeforeEach(func() { + metrics.ImageSelectionErrorCount.Reset() + }) + + Describe("ImageSelectionErrorCount", func() { + It("should have no errors initially", func() { + Expect(testutil.CollectAndCount(metrics.ImageSelectionErrorCount)).To(Equal(0)) + }) + + It("should increment the error count for a family", func() { + metrics.ImageSelectionErrorCount.WithLabelValues("Ubuntu2204").Inc() + Expect(testutil.CollectAndCount(metrics.ImageSelectionErrorCount)).To(Equal(1)) + }) + }) +}) diff --git a/pkg/operator/operator.go b/pkg/operator/operator.go new file mode 100644 index 000000000..866c6c02c --- /dev/null +++ b/pkg/operator/operator.go @@ -0,0 +1,150 @@ +/* +Portions Copyright (c) Microsoft Corporation. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package operator + +import ( + "context" + "encoding/base64" + "fmt" + + "github.com/patrickmn/go-cache" + "github.com/samber/lo" + "k8s.io/client-go/rest" + "k8s.io/client-go/transport" + "knative.dev/pkg/ptr" + + "github.com/Azure/karpenter/pkg/apis/settings" + "github.com/Azure/karpenter/pkg/auth" + azurecache "github.com/Azure/karpenter/pkg/cache" + "github.com/Azure/karpenter/pkg/providers/imagefamily" + "github.com/Azure/karpenter/pkg/providers/instance" + "github.com/Azure/karpenter/pkg/providers/instancetype" + "github.com/Azure/karpenter/pkg/providers/launchtemplate" + "github.com/Azure/karpenter/pkg/providers/loadbalancer" + "github.com/Azure/karpenter/pkg/providers/pricing" + "github.com/aws/karpenter-core/pkg/operator" +) + +type Operator struct { + *operator.Operator + + UnavailableOfferingsCache *azurecache.UnavailableOfferings + + ImageProvider *imagefamily.Provider + ImageResolver *imagefamily.Resolver + LaunchTemplateProvider *launchtemplate.Provider + PricingProvider *pricing.Provider + InstanceTypesProvider *instancetype.Provider + InstanceProvider *instance.Provider + LoadBalancerProvider *loadbalancer.Provider +} + +func NewOperator(ctx context.Context, operator *operator.Operator) (context.Context, *Operator) { + azConfig, err := GetAzConfig() + lo.Must0(err, "creating Azure config") // TODO: I assume we prefer this over the cleaner azConfig := lo.Must(GetAzConfig()), as this has a helpful error message? + + azClient, err := instance.CreateAzClient(ctx, azConfig) + lo.Must0(err, "creating Azure client") + + unavailableOfferingsCache := azurecache.NewUnavailableOfferings() + pricingProvider := pricing.NewProvider( + ctx, + pricing.NewAPI(), + azConfig.Location, + operator.Elected(), + ) + imageProvider := imagefamily.NewProvider( + operator.KubernetesInterface, + cache.New(azurecache.KubernetesVersionTTL, + azurecache.DefaultCleanupInterval), + azClient.ImageVersionsClient, + azConfig.Location, + ) + imageResolver := imagefamily.New(operator.GetClient(), imageProvider) + launchTemplateProvider := launchtemplate.NewProvider( + ctx, + imageResolver, + imageProvider, + lo.Must(getCABundle(operator.GetConfig())), + settings.FromContext(ctx).ClusterEndpoint, + azConfig.TenantID, + azConfig.SubscriptionID, + azConfig.UserAssignedIdentityID, + azConfig.NodeResourceGroup, + azConfig.Location, + ) + instanceTypeProvider := instancetype.NewProvider( + azConfig.Location, + cache.New(instancetype.InstanceTypesCacheTTL, azurecache.DefaultCleanupInterval), + azClient.SKUClient, + pricingProvider, + unavailableOfferingsCache, + ) + loadBalancerProvider := loadbalancer.NewProvider( + azClient.LoadBalancersClient, + cache.New(loadbalancer.LoadBalancersCacheTTL, azurecache.DefaultCleanupInterval), + azConfig.NodeResourceGroup, + ) + instanceProvider := instance.NewProvider( + ctx, + azClient, + instanceTypeProvider, + launchTemplateProvider, + loadBalancerProvider, + unavailableOfferingsCache, + azConfig.Location, + azConfig.NodeResourceGroup, + azConfig.SubnetID, + azConfig.SubscriptionID, + ) + + return ctx, &Operator{ + Operator: operator, + UnavailableOfferingsCache: unavailableOfferingsCache, + ImageProvider: imageProvider, + ImageResolver: imageResolver, + LaunchTemplateProvider: launchTemplateProvider, + PricingProvider: pricingProvider, + InstanceTypesProvider: instanceTypeProvider, + InstanceProvider: instanceProvider, + LoadBalancerProvider: loadBalancerProvider, + } +} + +func GetAzConfig() (*auth.Config, error) { + cfg, err := auth.BuildAzureConfig() + if err != nil { + return nil, err + } + return cfg, nil +} + +func getCABundle(restConfig *rest.Config) (*string, error) { + // Discover CA Bundle from the REST client. We could alternatively + // have used the simpler client-go InClusterConfig() method. + // However, that only works when Karpenter is running as a Pod + // within the same cluster it's managing. + transportConfig, err := restConfig.TransportConfig() + if err != nil { + return nil, fmt.Errorf("discovering caBundle, loading transport config, %w", err) + } + _, err = transport.TLSConfigFor(transportConfig) // fills in CAData! + if err != nil { + return nil, fmt.Errorf("discovering caBundle, loading TLS config, %w", err) + } + return ptr.String(base64.StdEncoding.EncodeToString(transportConfig.TLS.CAData)), nil +} diff --git a/pkg/providers/imagefamily/bootstrap/aksbootstrap.go b/pkg/providers/imagefamily/bootstrap/aksbootstrap.go new file mode 100644 index 000000000..a2df0c87e --- /dev/null +++ b/pkg/providers/imagefamily/bootstrap/aksbootstrap.go @@ -0,0 +1,589 @@ +/* +Portions Copyright (c) Microsoft Corporation. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package bootstrap + +import ( + "bytes" + _ "embed" + "encoding/base64" + "fmt" + "os" + "strings" + "text/template" + + corev1beta1 "github.com/aws/karpenter-core/pkg/apis/v1beta1" + "github.com/aws/karpenter-core/pkg/utils/resources" + "github.com/samber/lo" + v1 "k8s.io/api/core/v1" + "knative.dev/pkg/ptr" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +type AKS struct { + Options + + Arch string + TenantID string + SubscriptionID string + UserAssignedIdentityID string + Location string + ResourceGroup string + ClusterID string + APIServerName string + KubeletClientTLSBootstrapToken string + NetworkPlugin string + NetworkPolicy string + KubernetesVersion string +} + +var _ Bootstrapper = (*AKS)(nil) // assert AKS implements Bootstrapper + +func (a AKS) Script() (string, error) { + bootstrapScript, err := a.aksBootstrapScript() + if err != nil { + return "", fmt.Errorf("error getting AKS bootstrap script: %w", err) + } + + return base64.StdEncoding.EncodeToString([]byte(bootstrapScript)), nil +} + +// Config item types classified by code: +// +// - : known unnecessary or unused - (empty) value set in code, until dropped from template +// n : not (yet?) supported, set to empty or something reasonable in code +// s : static/constant (or very slow changing), value set in code; +// also the choice for something that does not have to be exposed for customization yet +// +// a : known argument/parameter, passed in (usually from environment) +// x : unique per cluster, extracted or specified. (Candidates for exposure/accessibility via API) +// X : unique per nodepool, extracted or specified. (Candidates for exposure/accessibility via API) +// c : user input, global Karpenter config (provider-specific) +// p : user input, part of standard Provisioner (NodePool) CR spec. Example: custom labels, kubelet config +// t : user input, NodeTemplate (potentially per node) +// k : computed (at runtime) by Karpenter (e.g. based on VM SKU, extra labels, etc.) +// (xk - computed from per cluster data, such as cluster id) +// +// ? : needs more investigation +// +// multiple codes: combined from several sources + +// Config sources for types: +// +// Hardcoded (this file) : unused (-), static (s) and unsupported (n), as well as selected defaults (s) +// Computed at runtime : computed (k) +// Karpenter global ConfigMap (provider-specific): cluster-level user input (c) - ALL DEFAULTED FOR NOW +// : as well as unique per cluster (x) - until we have a better place for these +// (TBD) : unique per nodepool. extracted or specified (X) +// NodeTemplate : user input that could be per-node (t) - ALL DEFAULTED FOR NOW +// Provisioner spec : selected nodepool-level user input (p) + +// NodeBootstrapVariables carries all variables needed to bootstrap a node +// It is used as input rendering the bootstrap script Go template (customDataTemplate) +type NodeBootstrapVariables struct { + IsAKSCustomCloud bool // n (false) + InitAKSCustomCloudFilepath string // n (static) + AKSCustomCloudRepoDepotEndpoint string // n derived from custom cloud env? + AdminUsername string // t typically azureuser but can be user input + MobyVersion string // - unnecessary + TenantID string // p environment derived, unnecessary? + KubernetesVersion string // ? cluster/node pool specific, derived from user input + HyperkubeURL string // - should be unnecessary + KubeBinaryURL string // - necessary only for non-cached versions / static-ish + CustomKubeBinaryURL string // - unnecessary + KubeproxyURL string // - should be unnecessary or bug + APIServerPublicKey string // - unique per cluster, actually not sure best way to extract? [should not be needed on agent nodes] + SubscriptionID string // a can be derived from environment/imds + ResourceGroup string // a can be derived from environment/imds + Location string // a can be derived from environment/imds + VMType string // xd derived from cluster but unnecessary (?) only used by CCM [will default to "vmss" for now] + Subnet string // xd derived from cluster but unnecessary (?) only used by CCM [will default to "aks-subnet for now] + NetworkSecurityGroup string // xk derived from cluster but unnecessary (?) only used by CCM [= "aks-agentpool--nsg" for now] + VirtualNetwork string // xk derived from cluster but unnecessary (?) only used by CCM [= "aks-vnet-" for now] + VirtualNetworkResourceGroup string // xd derived from cluster but unnecessary (?) only used by CCM [default to empty, looks like unused] + RouteTable string // xk derived from cluster but unnecessary (?) only used by CCM [= "aks-agentpool--routetable" for now] + PrimaryAvailabilitySet string // - derived from cluster but unnecessary (?) only used by CCM + PrimaryScaleSet string // - derived from cluster but unnecessary (?) only used by CCM + ServicePrincipalClientID string // ad user input + NetworkPlugin string // x user input (? actually derived from cluster, right?) + NetworkPolicy string // x user input / unique per cluster. user-specified. + VNETCNILinuxPluginsURL string // - unnecessary [actually, currently required] + CNIPluginsURL string // - unnecessary [actually, currently required] + CloudProviderBackoff bool // s BEGIN CLOUD CONFIG for azure stuff, static/derived from user inputs + CloudProviderBackoffMode string // s [static until has to be exposed; could propagate Karpenter RL config, but won't] + CloudProviderBackoffRetries string // s + CloudProviderBackoffExponent string // s + CloudProviderBackoffDuration string // s + CloudProviderBackoffJitter string // s + CloudProviderRatelimit bool // s + CloudProviderRatelimitQPS string // s + CloudProviderRatelimitQPSWrite string // s + CloudProviderRatelimitBucket string // s + CloudProviderRatelimitBucketWrite string // s + LoadBalancerDisableOutboundSNAT bool // xd [= false for now] + UseManagedIdentityExtension bool // s [always false?] + UseInstanceMetadata bool // s [always true?] + LoadBalancerSKU string // xd [= "Standard" for now] + ExcludeMasterFromStandardLB bool // s [always true?] + MaximumLoadbalancerRuleCount int // xd END CLOUD CONFIG [will default to 250 for now] + ContainerRuntime string // s always containerd + CLITool string // s static/unnecessary + ContainerdDownloadURLBase string // - unnecessary + NetworkMode string // c user input + UserAssignedIdentityID string // a user input + APIServerName string // x unique per cluster + IsVHD bool // s static-ish + GPUNode bool // k derived from VM size + SGXNode bool // - unused + MIGNode bool // t user input + ConfigGPUDriverIfNeeded bool // s depends on hardware, unnecessary for oss, but aks provisions gpu drivers + EnableGPUDevicePluginIfNeeded bool // - deprecated/preview only, don't do this for OSS + TeleportdPluginDownloadURL string // - user input, don't do this for OSS + ContainerdVersion string // - unused + ContainerdPackageURL string // - only for testing + RuncVersion string // - unused + RuncPackageURL string // - testing only + EnableHostsConfigAgent bool // n derived from private cluster user input...I think? + DisableSSH bool // t user input + NeedsContainerd bool // s static true + TeleportEnabled bool // t user input + ShouldConfigureHTTPProxy bool // c user input + ShouldConfigureHTTPProxyCA bool // c user input [secret] + HTTPProxyTrustedCA string // c user input [secret] + ShouldConfigureCustomCATrust bool // c user input + CustomCATrustConfigCerts []string // c user input [secret] + IsKrustlet bool // t user input + GPUNeedsFabricManager bool // v determined by GPU hardware type + NeedsDockerLogin bool // t user input [still needed?] + IPv6DualStackEnabled bool // t user input + OutboundCommand string // s mostly static/can be + EnableUnattendedUpgrades bool // c user input [presumably cluster level, correct?] + EnsureNoDupePromiscuousBridge bool // k derived {{ and NeedsContainerd IsKubenet (not HasCalicoNetworkPolicy) }} [could be computed by template ...] + ShouldConfigSwapFile bool // t user input + ShouldConfigTransparentHugePage bool // t user input + TargetCloud string // n derive from environment/user input + TargetEnvironment string // n derive from environment/user input + CustomEnvJSON string // n derive from environment/user input + IsCustomCloud bool // n derive from environment/user input + CSEHelpersFilepath string // s static + CSEDistroHelpersFilepath string // s static + CSEInstallFilepath string // s static + CSEDistroInstallFilepath string // s static + CSEConfigFilepath string // s static + AzurePrivateRegistryServer string // c user input + HasCustomSearchDomain bool // c user input + CustomSearchDomainFilepath string // s static + HTTPProxyURLs string // c user input [presumably cluster-level] + HTTPSProxyURLs string // c user input [presumably cluster-level] + NoProxyURLs string // c user input [presumably cluster-level] + TLSBootstrappingEnabled bool // s static true + SecureTLSBootstrappingEnabled bool // s static false + DHCPv6ServiceFilepath string // k derived from user input [how?] + DHCPv6ConfigFilepath string // k derived from user input [how?] + THPEnabled string // c user input [presumably cluster-level][should be bool?] + THPDefrag string // c user input [presumably cluster-level][should be bool?] + ServicePrincipalFileContent string // s only required for RP cluster [static: msi?] + KubeletClientContent string // - unnecessary [if using TLS bootstrapping] + KubeletClientCertContent string // - unnecessary + KubeletConfigFileEnabled bool // s can be static [should kubelet config be actually used/preferred instead of flags?] + KubeletConfigFileContent string // s mix of user/static/RP-generated. + SwapFileSizeMB int // t user input + GPUDriverVersion string // k determine by OS + GPU hardware requirements; can be determined automatically, but hard. suggest using GPU operator. + GPUInstanceProfile string // t user-specified + CustomSearchDomainName string // c user-specified [presumably cluster-level] + CustomSearchRealmUser string // c user-specified [presumably cluster-level] + CustomSearchRealmPassword string // c user-specified [presumably cluster-level] + MessageOfTheDay string // t user-specified [presumably node-level] + HasKubeletDiskType bool // t user-specified [presumably node-level] + NeedsCgroupV2 bool // k can be automatically determined + SysctlContent string // t user-specified + TLSBootstrapToken string // X nodepool or node specific. can be created automatically + KubeletFlags string // psX unique per nodepool. partially user-specified, static, and RP-generated + KubeletNodeLabels string // pk node-pool specific. user-specified. + AzureEnvironmentFilepath string // s can be made static [usually "/etc/kubernetes/azure.json", but my examples use ""?] + KubeCACrt string // x unique per cluster + KubenetTemplate string // s static + ContainerdConfigContent string // k determined by GPU VM size, WASM support, Kata support + IsKata bool // n user-specified +} + +var ( + //go:embed cse_cmd.sh.gtpl + customDataTemplateText string + customDataTemplate = template.Must(template.New("customdata").Parse(customDataTemplateText)) + + //go:embed containerd.toml.gtpl + containerdConfigTemplateText string + containerdConfigTemplate = template.Must(template.New("containerdconfig").Parse(containerdConfigTemplateText)) + + //go:embed sysctl.conf + sysctlContent []byte + //go:embed kubenet-cni.json.gtpl + kubenetTemplate []byte + + // source note: unique per nodepool. partially user-specified, static, and RP-generated + // removed --image-pull-progress-deadline=30m (not in 1.24?) + // removed --network-plugin=cni (not in 1.24?) + kubeletFlagsBase = map[string]string{ + "--address": "0.0.0.0", + "--anonymous-auth": "false", + "--authentication-token-webhook": "true", + "--authorization-mode": "Webhook", + "--azure-container-registry-config": "/etc/kubernetes/azure.json", + "--cgroups-per-qos": "true", + "--client-ca-file": "/etc/kubernetes/certs/ca.crt", + "--cloud-config": "/etc/kubernetes/azure.json", + "--cloud-provider": "external", + "--cluster-dns": "10.0.0.10", + "--cluster-domain": "cluster.local", + "--enforce-node-allocatable": "pods", + "--event-qps": "0", + "--eviction-hard": "memory.available<750Mi,nodefs.available<10%,nodefs.inodesFree<5%", + "--feature-gates": "CSIMigrationAzureFile=true,DelegateFSGroupToCSIDriver=true,RotateKubeletServerCertificate=true", + "--image-gc-high-threshold": "85", + "--image-gc-low-threshold": "80", + "--keep-terminated-pod-volumes": "false", + "--kubeconfig": "/var/lib/kubelet/kubeconfig", + "--max-pods": "110", + "--node-status-update-frequency": "10s", + "--pod-infra-container-image": "mcr.microsoft.com/oss/kubernetes/pause:3.6", + "--pod-manifest-path": "/etc/kubernetes/manifests", + "--pod-max-pids": "-1", + "--protect-kernel-defaults": "true", + "--read-only-port": "0", + "--resolv-conf": "/run/systemd/resolve/resolv.conf", + "--rotate-certificates": "true", + "--streaming-connection-idle-timeout": "4h", + "--tls-cert-file": "/etc/kubernetes/certs/kubeletserver.crt", + "--tls-cipher-suites": "TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256", + "--tls-private-key-file": "/etc/kubernetes/certs/kubeletserver.key", + } + + kubeletNodeLabelsBase = map[string]string{ + "kubernetes.azure.com/mode": "user", + } + vnetCNILinuxPluginsURL = fmt.Sprintf("%s/azure-cni/v1.4.32/binaries/azure-vnet-cni-linux-amd64-v1.4.32.tgz", globalAKSMirror) + cniPluginsURL = fmt.Sprintf("%s/cni-plugins/v1.1.1/binaries/cni-plugins-linux-amd64-v1.1.1.tgz", globalAKSMirror) +) + +var ( + + // baseline, covering unused (-), static (s), and unsupported (n) fields, + // as well as defaults, cluster/node level (cd/td/xd) + staticNodeBootstrapVars = NodeBootstrapVariables{ + IsAKSCustomCloud: false, // n + InitAKSCustomCloudFilepath: "", // n + AKSCustomCloudRepoDepotEndpoint: "", // n + AdminUsername: "azureuser", // td + MobyVersion: "", // - + HyperkubeURL: "", // - + KubeBinaryURL: "", // cd + CustomKubeBinaryURL: "", // - + KubeproxyURL: "", // - + VMType: "vmss", // xd + Subnet: "aks-subnet", // xd + VirtualNetworkResourceGroup: "", // xd + PrimaryAvailabilitySet: "", // - + PrimaryScaleSet: "", // - + ServicePrincipalClientID: "msi", // ad + VNETCNILinuxPluginsURL: vnetCNILinuxPluginsURL, // - [currently required, installCNI in provisioning scripts depends on CNI_PLUGINS_URL] + CNIPluginsURL: cniPluginsURL, // - [currently required, same] + CloudProviderBackoff: true, // s + CloudProviderBackoffMode: "v2", // s + CloudProviderBackoffRetries: "6", // s + CloudProviderBackoffExponent: "0", // s + CloudProviderBackoffDuration: "5", // s + CloudProviderBackoffJitter: "0", // s + CloudProviderRatelimit: true, // s + CloudProviderRatelimitQPS: "10", // s + CloudProviderRatelimitQPSWrite: "10", // s + CloudProviderRatelimitBucket: "100", // s + CloudProviderRatelimitBucketWrite: "100", // s + LoadBalancerDisableOutboundSNAT: false, // xd + UseManagedIdentityExtension: false, // s + UseInstanceMetadata: true, // s + LoadBalancerSKU: "Standard", // xd + ExcludeMasterFromStandardLB: true, // s + MaximumLoadbalancerRuleCount: 250, // xd + ContainerRuntime: "containerd", // s + CLITool: "ctr", // s + ContainerdDownloadURLBase: "", // - + NetworkMode: "", // cd + IsVHD: true, // s + SGXNode: false, // - + MIGNode: false, // td + ConfigGPUDriverIfNeeded: true, // s + EnableGPUDevicePluginIfNeeded: false, // - + TeleportdPluginDownloadURL: "", // - + ContainerdVersion: "", // - + ContainerdPackageURL: "", // - + RuncVersion: "", // - + RuncPackageURL: "", // - + DisableSSH: false, // td + EnableHostsConfigAgent: false, // n + NeedsContainerd: true, // s + TeleportEnabled: false, // td + ShouldConfigureHTTPProxy: false, // cd + ShouldConfigureHTTPProxyCA: false, // cd + HTTPProxyTrustedCA: "", // cd + ShouldConfigureCustomCATrust: false, // cd + CustomCATrustConfigCerts: []string{}, // cd + + OutboundCommand: "curl -v --insecure --proxy-insecure https://mcr.microsoft.com/v2/", // s + EnableUnattendedUpgrades: false, // cd + IsKrustlet: false, // td + ShouldConfigSwapFile: false, // td + ShouldConfigTransparentHugePage: false, // td + TargetCloud: "AzurePublicCloud", // n + TargetEnvironment: "AzurePublicCloud", // n + CustomEnvJSON: "", // n + IsCustomCloud: false, // n + CSEHelpersFilepath: "/opt/azure/containers/provision_source.sh", // s + CSEDistroHelpersFilepath: "/opt/azure/containers/provision_source_distro.sh", // s + CSEInstallFilepath: "/opt/azure/containers/provision_installs.sh", // s + CSEDistroInstallFilepath: "/opt/azure/containers/provision_installs_distro.sh", // s + CSEConfigFilepath: "/opt/azure/containers/provision_configs.sh", // s + AzurePrivateRegistryServer: "", // cd + HasCustomSearchDomain: false, // cd + CustomSearchDomainFilepath: "/opt/azure/containers/setup-custom-search-domains.sh", // s + HTTPProxyURLs: "", // cd + HTTPSProxyURLs: "", // cd + NoProxyURLs: "", // cd + TLSBootstrappingEnabled: true, // s + SecureTLSBootstrappingEnabled: false, // s + THPEnabled: "", // cd + THPDefrag: "", // cd + ServicePrincipalFileContent: base64.StdEncoding.EncodeToString([]byte("msi")), // s + KubeletClientContent: "", // - + KubeletClientCertContent: "", // - + KubeletConfigFileEnabled: false, // s + KubeletConfigFileContent: "", // s + SwapFileSizeMB: 0, // td + GPUInstanceProfile: "", // td + CustomSearchDomainName: "", // cd + CustomSearchRealmUser: "", // cd + CustomSearchRealmPassword: "", // cd + MessageOfTheDay: "", // td + HasKubeletDiskType: false, // td + SysctlContent: base64.StdEncoding.EncodeToString(sysctlContent), // td + KubeletFlags: "", // psX + AzureEnvironmentFilepath: "", // s + KubenetTemplate: base64.StdEncoding.EncodeToString(kubenetTemplate), // s + ContainerdConfigContent: "", // kd + IsKata: false, // n + + } +) + +// Node Labels for Vnet +const ( + vnetDataPlaneLabel = "kubernetes.azure.com/ebpf-dataplane" + vnetNetworkNameLabel = "kubernetes.azure.com/network-name" + vnetSubnetNameLabel = "kubernetes.azure.com/network-subnet" + vnetSubscriptionIDLabel = "kubernetes.azure.com/network-subscription" + vnetGUIDLabel = "kubernetes.azure.com/nodenetwork-vnetguid" + vnetPodNetworkTypeLabel = "kubernetes.azure.com/podnetwork-type" + ciliumDataPlane = "cilium" + overlayNetworkType = "overlay" + globalAKSMirror = "https://acs-mirror.azureedge.net" +) + +func (a AKS) aksBootstrapScript() (string, error) { + // use these as the base / defaults + nbv := staticNodeBootstrapVars // don't need deep copy (yet) + + // apply overrides from passed in options + a.applyOptions(&nbv) + + containerdConfigTemplate, err := containerdConfigFromNodeBootstrapVars(&nbv) + if err != nil { + return "", fmt.Errorf("error getting containerd config from node bootstrap variables: %w", err) + } + + nbv.ContainerdConfigContent = base64.StdEncoding.EncodeToString([]byte(containerdConfigTemplate)) + // generate script from template using the variables + customData, err := getCustomDataFromNodeBootstrapVars(&nbv) + if err != nil { + return "", fmt.Errorf("error getting custom data from node bootstrap variables: %w", err) + } + return customData, nil +} + +// Download URL for KUBE_BINARY_URL publishes each k8s version in the URL. +func kubeBinaryURL(kubernetesVersion, cpuArch string) string { + return fmt.Sprintf("%s/kubernetes/v%s/binaries/kubernetes-node-linux-%s.tar.gz", globalAKSMirror, kubernetesVersion, cpuArch) +} + +func (a AKS) applyOptions(nbv *NodeBootstrapVariables) { + nbv.KubeCACrt = *a.CABundle + nbv.APIServerName = a.APIServerName + nbv.TLSBootstrapToken = a.KubeletClientTLSBootstrapToken + + nbv.TenantID = a.TenantID + nbv.SubscriptionID = a.SubscriptionID + nbv.Location = a.Location + nbv.ResourceGroup = a.ResourceGroup + nbv.UserAssignedIdentityID = a.UserAssignedIdentityID + + nbv.NetworkPlugin = a.NetworkPlugin + nbv.NetworkPolicy = a.NetworkPolicy + nbv.KubernetesVersion = a.KubernetesVersion + + nbv.KubeBinaryURL = kubeBinaryURL(a.KubernetesVersion, a.Arch) + nbv.VNETCNILinuxPluginsURL = fmt.Sprintf("%s/azure-cni/v1.4.32/binaries/azure-vnet-cni-linux-%s-v1.4.32.tgz", globalAKSMirror, a.Arch) + nbv.CNIPluginsURL = fmt.Sprintf("%s/cni-plugins/v1.1.1/binaries/cni-plugins-linux-%s-v1.1.1.tgz", globalAKSMirror, a.Arch) + + // calculated values + nbv.EnsureNoDupePromiscuousBridge = nbv.NeedsContainerd && nbv.NetworkPlugin == "kubenet" && nbv.NetworkPolicy != "calico" + nbv.NetworkSecurityGroup = fmt.Sprintf("aks-agentpool-%s-nsg", a.ClusterID) + nbv.VirtualNetwork = fmt.Sprintf("aks-vnet-%s", a.ClusterID) + nbv.RouteTable = fmt.Sprintf("aks-agentpool-%s-routetable", a.ClusterID) + + if a.GPUNode { + nbv.GPUNode = true + nbv.ConfigGPUDriverIfNeeded = true + nbv.GPUDriverVersion = a.GPUDriverVersion + } + nbv.NeedsCgroupV2 = true + // merge and stringify labels + kubeletLabels := lo.Assign(kubeletNodeLabelsBase, a.Labels) + getAgentbakerGeneratedLabels(a.ResourceGroup, kubeletLabels) + + //Adding vnet-related labels to the nodeLabels. + azureVnetGUID := os.Getenv("AZURE_VNET_GUID") + azureVnetName := os.Getenv("AZURE_VNET_NAME") + azureSubnetName := os.Getenv("AZURE_SUBNET_NAME") + + vnetLabels := map[string]string{ + vnetDataPlaneLabel: ciliumDataPlane, + vnetNetworkNameLabel: azureVnetName, + vnetSubnetNameLabel: azureSubnetName, + vnetSubscriptionIDLabel: a.SubscriptionID, + vnetGUIDLabel: azureVnetGUID, + vnetPodNetworkTypeLabel: overlayNetworkType, + } + + kubeletLabels = lo.Assign(kubeletLabels, vnetLabels) + nbv.KubeletNodeLabels = strings.Join(lo.MapToSlice(kubeletLabels, func(k, v string) string { + return fmt.Sprintf("%s=%s", k, v) + }), ",") + + // merge and stringify taints + kubeletFlags := lo.Assign(kubeletFlagsBase) + if len(a.Taints) > 0 { + taintStrs := lo.Map(a.Taints, func(taint v1.Taint, _ int) string { return taint.ToString() }) + kubeletFlags = lo.Assign(kubeletFlags, map[string]string{"--register-with-taints": strings.Join(taintStrs, ",")}) + } + + machineKubeletConfig := KubeletConfigToMap(a.KubeletConfig) + kubeletFlags = lo.Assign(kubeletFlags, machineKubeletConfig) + + // striginify kubelet flags (including taints) + nbv.KubeletFlags = strings.Join(lo.MapToSlice(kubeletFlags, func(k, v string) string { + return fmt.Sprintf("%s=%s", k, v) + }), " ") +} + +func containerdConfigFromNodeBootstrapVars(nbv *NodeBootstrapVariables) (string, error) { + var buffer bytes.Buffer + if err := containerdConfigTemplate.Execute(&buffer, *nbv); err != nil { + return "", fmt.Errorf("error executing containerd config template: %w", err) + } + return buffer.String(), nil +} + +func getCustomDataFromNodeBootstrapVars(nbv *NodeBootstrapVariables) (string, error) { + var buffer bytes.Buffer + if err := customDataTemplate.Execute(&buffer, *nbv); err != nil { + return "", fmt.Errorf("error executing custom data template: %w", err) + } + return buffer.String(), nil +} + +func getAgentbakerGeneratedLabels(nodeResourceGroup string, nodeLabels map[string]string) { + nodeLabels["kubernetes.azure.com/role"] = "agent" + nodeLabels["kubernetes.azure.com/cluster"] = normalizeResourceGroupNameForLabel(nodeResourceGroup) +} + +func normalizeResourceGroupNameForLabel(resourceGroupName string) string { + truncated := resourceGroupName + truncated = strings.ReplaceAll(truncated, "(", "-") + truncated = strings.ReplaceAll(truncated, ")", "-") + const maxLen = 63 + if len(truncated) > maxLen { + truncated = truncated[0:maxLen] + } + + if strings.HasSuffix(truncated, "-") || + strings.HasSuffix(truncated, "_") || + strings.HasSuffix(truncated, ".") { + if len(truncated) > 62 { + return truncated[0:len(truncated)-1] + "z" + } + return truncated + "z" + } + return truncated +} + +func KubeletConfigToMap(kubeletConfig *corev1beta1.KubeletConfiguration) map[string]string { + args := make(map[string]string) + + if kubeletConfig == nil { + return args + } + if kubeletConfig.MaxPods != nil { + args["--max-pods"] = fmt.Sprintf("%d", ptr.Int32Value(kubeletConfig.MaxPods)) + } + if kubeletConfig.PodsPerCore != nil { + args["--pods-per-core"] = fmt.Sprintf("%d", ptr.Int32Value(kubeletConfig.PodsPerCore)) + } + JoinParameterArgsToMap(args, "--system-reserved", resources.StringMap(kubeletConfig.SystemReserved), "=") + JoinParameterArgsToMap(args, "--kube-reserved", resources.StringMap(kubeletConfig.KubeReserved), "=") + JoinParameterArgsToMap(args, "--eviction-hard", kubeletConfig.EvictionHard, "<") + JoinParameterArgsToMap(args, "--eviction-soft", kubeletConfig.EvictionSoft, "<") + JoinParameterArgsToMap(args, "--eviction-soft-grace-period", lo.MapValues(kubeletConfig.EvictionSoftGracePeriod, func(v metav1.Duration, _ string) string { + return v.Duration.String() + }), "=") + + if kubeletConfig.EvictionMaxPodGracePeriod != nil { + args["--eviction-max-pod-grace-period"] = fmt.Sprintf("%d", ptr.Int32Value(kubeletConfig.EvictionMaxPodGracePeriod)) + } + if kubeletConfig.ImageGCHighThresholdPercent != nil { + args["--image-gc-high-threshold"] = fmt.Sprintf("%d", ptr.Int32Value(kubeletConfig.ImageGCHighThresholdPercent)) + } + if kubeletConfig.ImageGCLowThresholdPercent != nil { + args["--image-gc-low-threshold"] = fmt.Sprintf("%d", ptr.Int32Value(kubeletConfig.ImageGCLowThresholdPercent)) + } + if kubeletConfig.CPUCFSQuota != nil { + args["--cpu-cfs-quota"] = fmt.Sprintf("%t", lo.FromPtr(kubeletConfig.CPUCFSQuota)) + } + + return args +} + +// joinParameterArgsToMap joins a map of keys and values by their separator. The separator will sit between the +// arguments in a comma-separated list i.e. arg1val1,arg2val2 +func JoinParameterArgsToMap[K comparable, V any](result map[string]string, name string, m map[K]V, separator string) { + var args []string + + for k, v := range m { + args = append(args, fmt.Sprintf("%v%s%v", k, separator, v)) + } + if len(args) > 0 { + result[name] = strings.Join(args, ",") + } +} diff --git a/pkg/providers/imagefamily/bootstrap/aksbootstrap_test.go b/pkg/providers/imagefamily/bootstrap/aksbootstrap_test.go new file mode 100644 index 000000000..3c501e8fa --- /dev/null +++ b/pkg/providers/imagefamily/bootstrap/aksbootstrap_test.go @@ -0,0 +1,60 @@ +/* +Portions Copyright (c) Microsoft Corporation. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package bootstrap + +import ( + "fmt" + "testing" +) + +func TestKubeBinaryURL(t *testing.T) { + cases := []struct { + name string + version string + expected string + }{ + { + name: "Test version 1.24.x", + version: "1.24.5", + expected: fmt.Sprintf("%s/kubernetes/v1.24.5/binaries/kubernetes-node-linux-amd64.tar.gz", globalAKSMirror), + }, + { + name: "Test version 1.25.x", + version: "1.25.2", + expected: fmt.Sprintf("%s/kubernetes/v1.25.2/binaries/kubernetes-node-linux-amd64.tar.gz", globalAKSMirror), + }, + { + name: "Test version 1.26.x", + version: "1.26.0", + expected: fmt.Sprintf("%s/kubernetes/v1.26.0/binaries/kubernetes-node-linux-amd64.tar.gz", globalAKSMirror), + }, + { + name: "Test version 1.27.x", + version: "1.27.1", + expected: fmt.Sprintf("%s/kubernetes/v1.27.1/binaries/kubernetes-node-linux-amd64.tar.gz", globalAKSMirror), + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + actual := kubeBinaryURL(tc.version, "amd64") + if actual != tc.expected { + t.Errorf("Expected %s but got %s", tc.expected, actual) + } + }) + } +} diff --git a/pkg/providers/imagefamily/bootstrap/bootstrap.go b/pkg/providers/imagefamily/bootstrap/bootstrap.go new file mode 100644 index 000000000..da9d55162 --- /dev/null +++ b/pkg/providers/imagefamily/bootstrap/bootstrap.go @@ -0,0 +1,42 @@ +/* +Portions Copyright (c) Microsoft Corporation. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package bootstrap + +import ( + corev1beta1 "github.com/aws/karpenter-core/pkg/apis/v1beta1" + core "k8s.io/api/core/v1" +) + +// Options is the node bootstrapping parameters passed from Karpenter to the provisioning node +type Options struct { + ClusterName string + ClusterEndpoint string + KubeletConfig *corev1beta1.KubeletConfiguration + Taints []core.Taint `hash:"set"` + Labels map[string]string `hash:"set"` + CABundle *string + GPUNode bool + GPUDriverVersion string +} + +// Bootstrapper can be implemented to generate a bootstrap script +// that uses the params from the Bootstrap type for a specific +// bootstrapping method. +// The only one implemented right now is AKS bootstrap script +type Bootstrapper interface { + Script() (string, error) +} diff --git a/pkg/providers/imagefamily/bootstrap/containerd.toml.gtpl b/pkg/providers/imagefamily/bootstrap/containerd.toml.gtpl new file mode 100644 index 000000000..ed7ef261a --- /dev/null +++ b/pkg/providers/imagefamily/bootstrap/containerd.toml.gtpl @@ -0,0 +1,44 @@ +version = 2 +oom_score = 0 +[plugins."io.containerd.grpc.v1.cri"] + sandbox_image = "mcr.microsoft.com/oss/kubernetes/pause:3.6" + [plugins."io.containerd.grpc.v1.cri".containerd] + {{- if .GPUNode }} + default_runtime_name = "nvidia-container-runtime" + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.nvidia-container-runtime] + runtime_type = "io.containerd.runc.v2" + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.nvidia-container-runtime.options] + BinaryName = "/usr/bin/nvidia-container-runtime" + {{- if .NeedsCgroupV2}} + SystemdCgroup = true + {{- end}} + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.untrusted] + runtime_type = "io.containerd.runc.v2" + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.untrusted.options] + BinaryName = "/usr/bin/nvidia-container-runtime" + {{- else}} + default_runtime_name = "runc" + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc] + runtime_type = "io.containerd.runc.v2" + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options] + BinaryName = "/usr/bin/runc" + {{- if .NeedsCgroupV2}} + SystemdCgroup = true + {{- end}} + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.untrusted] + runtime_type = "io.containerd.runc.v2" + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.untrusted.options] + BinaryName = "/usr/bin/runc" + {{- end}} + {{- if .EnsureNoDupePromiscuousBridge }} + [plugins."io.containerd.grpc.v1.cri".cni] + bin_dir = "/opt/cni/bin" + conf_dir = "/etc/cni/net.d" + conf_template = "/etc/containerd/kubenet_template.conf" + {{- end}} + [plugins."io.containerd.grpc.v1.cri".registry] + config_path = "/etc/containerd/certs.d" + [plugins."io.containerd.grpc.v1.cri".registry.headers] + X-Meta-Source-Client = ["azure/aks"] +[metrics] + address = "0.0.0.0:10257" diff --git a/pkg/providers/imagefamily/bootstrap/cse_cmd.sh.gtpl b/pkg/providers/imagefamily/bootstrap/cse_cmd.sh.gtpl new file mode 100644 index 000000000..926948f14 --- /dev/null +++ b/pkg/providers/imagefamily/bootstrap/cse_cmd.sh.gtpl @@ -0,0 +1,145 @@ +#!/bin/bash + +set -o allexport # export all variables to subshells +echo '#EOF' >> /opt/azure/manifest.json # wait_for_file looks for this +mkdir -p /var/log/azure/Microsoft.Azure.Extensions.CustomScript/events # expected, but not created w/o CSE + +echo $(date),$(hostname) > /var/log/azure/cluster-provision-cse-output.log; +for i in $(seq 1 1200); do +grep -Fq "EOF" /opt/azure/containers/provision.sh && break; +if [ $i -eq 1200 ]; then exit 100; else sleep 1; fi; +done; +{{if .IsAKSCustomCloud}} +for i in $(seq 1 1200); do +grep -Fq "EOF" {{.InitAKSCustomCloudFilepath}} && break; +if [ $i -eq 1200 ]; then exit 100; else sleep 1; fi; +done; +REPO_DEPOT_ENDPOINT="{{.AKSCustomCloudRepoDepotEndpoint}}" +{{.InitAKSCustomCloudFilepath}} >> /var/log/azure/cluster-provision.log 2>&1; +{{end}} +ADMINUSER={{.AdminUsername}} +MOBY_VERSION={{.MobyVersion}} +TENANT_ID={{.TenantID}} +KUBERNETES_VERSION={{.KubernetesVersion}} +HYPERKUBE_URL={{.HyperkubeURL}} +KUBE_BINARY_URL={{.KubeBinaryURL}} +CUSTOM_KUBE_BINARY_URL={{.CustomKubeBinaryURL}} +KUBEPROXY_URL={{.KubeproxyURL}} +APISERVER_PUBLIC_KEY={{.APIServerPublicKey}} +SUBSCRIPTION_ID={{.SubscriptionID}} +RESOURCE_GROUP={{.ResourceGroup}} +LOCATION={{.Location}} +VM_TYPE={{.VMType}} +SUBNET={{.Subnet}} +NETWORK_SECURITY_GROUP={{.NetworkSecurityGroup}} +VIRTUAL_NETWORK={{.VirtualNetwork}} +VIRTUAL_NETWORK_RESOURCE_GROUP={{.VirtualNetworkResourceGroup}} +ROUTE_TABLE={{.RouteTable}} +PRIMARY_AVAILABILITY_SET={{.PrimaryAvailabilitySet}} +PRIMARY_SCALE_SET={{.PrimaryScaleSet}} +SERVICE_PRINCIPAL_CLIENT_ID={{.ServicePrincipalClientID}} +NETWORK_PLUGIN={{.NetworkPlugin}} +NETWORK_POLICY="{{.NetworkPolicy}}" +VNET_CNI_PLUGINS_URL={{.VNETCNILinuxPluginsURL}} +CNI_PLUGINS_URL={{.CNIPluginsURL}} +CLOUDPROVIDER_BACKOFF={{.CloudProviderBackoff}} +CLOUDPROVIDER_BACKOFF_MODE={{.CloudProviderBackoffMode}} +CLOUDPROVIDER_BACKOFF_RETRIES={{.CloudProviderBackoffRetries}} +CLOUDPROVIDER_BACKOFF_EXPONENT={{.CloudProviderBackoffExponent}} +CLOUDPROVIDER_BACKOFF_DURATION={{.CloudProviderBackoffDuration}} +CLOUDPROVIDER_BACKOFF_JITTER={{.CloudProviderBackoffJitter}} +CLOUDPROVIDER_RATELIMIT={{.CloudProviderRatelimit}} +CLOUDPROVIDER_RATELIMIT_QPS={{.CloudProviderRatelimitQPS}} +CLOUDPROVIDER_RATELIMIT_QPS_WRITE={{.CloudProviderRatelimitQPSWrite}} +CLOUDPROVIDER_RATELIMIT_BUCKET={{.CloudProviderRatelimitBucket}} +CLOUDPROVIDER_RATELIMIT_BUCKET_WRITE={{.CloudProviderRatelimitBucketWrite}} +LOAD_BALANCER_DISABLE_OUTBOUND_SNAT={{.LoadBalancerDisableOutboundSNAT}} +USE_MANAGED_IDENTITY_EXTENSION={{.UseManagedIdentityExtension}} +USE_INSTANCE_METADATA={{.UseInstanceMetadata}} +LOAD_BALANCER_SKU={{.LoadBalancerSKU}} +EXCLUDE_MASTER_FROM_STANDARD_LB={{.ExcludeMasterFromStandardLB}} +MAXIMUM_LOADBALANCER_RULE_COUNT={{.MaximumLoadbalancerRuleCount}} +CONTAINER_RUNTIME={{.ContainerRuntime}} +CLI_TOOL={{.CLITool}} +CONTAINERD_DOWNLOAD_URL_BASE={{.ContainerdDownloadURLBase}} +NETWORK_MODE={{.NetworkMode}} +KUBE_BINARY_URL={{.KubeBinaryURL}} +USER_ASSIGNED_IDENTITY_ID={{.UserAssignedIdentityID}} +API_SERVER_NAME={{.APIServerName}} +IS_VHD={{.IsVHD}} +GPU_NODE={{.GPUNode}} +SGX_NODE={{.SGXNode}} +MIG_NODE={{.MIGNode}} +CONFIG_GPU_DRIVER_IF_NEEDED={{.ConfigGPUDriverIfNeeded}} +ENABLE_GPU_DEVICE_PLUGIN_IF_NEEDED={{.EnableGPUDevicePluginIfNeeded}} +TELEPORTD_PLUGIN_DOWNLOAD_URL={{.TeleportdPluginDownloadURL}} +CONTAINERD_VERSION={{.ContainerdVersion}} +CONTAINERD_PACKAGE_URL={{.ContainerdPackageURL}} +RUNC_VERSION={{.RuncVersion}} +RUNC_PACKAGE_URL={{.RuncPackageURL}} +ENABLE_HOSTS_CONFIG_AGENT="{{.EnableHostsConfigAgent}}" +DISABLE_SSH="{{.DisableSSH}}" +NEEDS_CONTAINERD="{{.NeedsContainerd}}" +TELEPORT_ENABLED="{{.TeleportEnabled}}" +SHOULD_CONFIGURE_HTTP_PROXY="{{.ShouldConfigureHTTPProxy}}" +SHOULD_CONFIGURE_HTTP_PROXY_CA="{{.ShouldConfigureHTTPProxyCA}}" +HTTP_PROXY_TRUSTED_CA="{{.HTTPProxyTrustedCA}}" +SHOULD_CONFIGURE_CUSTOM_CA_TRUST="{{.ShouldConfigureCustomCATrust}}" +CUSTOM_CA_TRUST_COUNT="{{len .CustomCATrustConfigCerts}}" +{{range $i, $cert := .CustomCATrustConfigCerts}} +CUSTOM_CA_CERT_{{$i}}="{{$cert}}" +{{end}} +IS_KRUSTLET="{{.IsKrustlet}}" +GPU_NEEDS_FABRIC_MANAGER="{{.GPUNeedsFabricManager}}" +NEEDS_DOCKER_LOGIN="{{.NeedsDockerLogin}}" +IPV6_DUAL_STACK_ENABLED="{{.IPv6DualStackEnabled}}" +OUTBOUND_COMMAND="{{.OutboundCommand}}" +ENABLE_UNATTENDED_UPGRADES="{{.EnableUnattendedUpgrades}}" +ENSURE_NO_DUPE_PROMISCUOUS_BRIDGE="{{.EnsureNoDupePromiscuousBridge}}" +SHOULD_CONFIG_SWAP_FILE="{{.ShouldConfigSwapFile}}" +SHOULD_CONFIG_TRANSPARENT_HUGE_PAGE="{{.ShouldConfigTransparentHugePage}}" +TARGET_CLOUD="{{.TargetCloud}}" +TARGET_ENVIRONMENT="{{.TargetEnvironment}}" +CUSTOM_ENV_JSON="{{.CustomEnvJSON}}" +IS_CUSTOM_CLOUD="{{.IsCustomCloud}}" +CSE_HELPERS_FILEPATH="{{.CSEHelpersFilepath}}" +CSE_DISTRO_HELPERS_FILEPATH="{{.CSEDistroHelpersFilepath}}" +CSE_INSTALL_FILEPATH="{{.CSEInstallFilepath}}" +CSE_DISTRO_INSTALL_FILEPATH="{{.CSEDistroInstallFilepath}}" +CSE_CONFIG_FILEPATH="{{.CSEConfigFilepath}}" +AZURE_PRIVATE_REGISTRY_SERVER="{{.AzurePrivateRegistryServer}}" +HAS_CUSTOM_SEARCH_DOMAIN="{{.HasCustomSearchDomain}}" +CUSTOM_SEARCH_DOMAIN_FILEPATH="{{.CustomSearchDomainFilepath}}" +HTTP_PROXY_URLS="{{.HTTPProxyURLs}}" +HTTPS_PROXY_URLS="{{.HTTPSProxyURLs}}" +NO_PROXY_URLS="{{.NoProxyURLs}}" +ENABLE_TLS_BOOTSTRAPPING="{{.TLSBootstrappingEnabled}}" +ENABLE_SECURE_TLS_BOOTSTRAPPING="{{.SecureTLSBootstrappingEnabled}}" +DHCPV6_SERVICE_FILEPATH="{{.DHCPv6ServiceFilepath}}" +DHCPV6_CONFIG_FILEPATH="{{.DHCPv6ConfigFilepath}}" +THP_ENABLED="{{.THPEnabled}}" +THP_DEFRAG="{{.THPDefrag}}" +SERVICE_PRINCIPAL_FILE_CONTENT="{{.ServicePrincipalFileContent}}" +KUBELET_CLIENT_CONTENT="{{.KubeletClientContent}}" +KUBELET_CLIENT_CERT_CONTENT="{{.KubeletClientCertContent}}" +KUBELET_CONFIG_FILE_ENABLED="{{.KubeletConfigFileEnabled}}" +KUBELET_CONFIG_FILE_CONTENT="{{.KubeletConfigFileContent}}" +SWAP_FILE_SIZE_MB="{{.SwapFileSizeMB}}" +GPU_DRIVER_VERSION="{{.GPUDriverVersion}}" +GPU_INSTANCE_PROFILE="{{.GPUInstanceProfile}}" +CUSTOM_SEARCH_DOMAIN_NAME="{{.CustomSearchDomainName}}" +CUSTOM_SEARCH_REALM_USER="{{.CustomSearchRealmUser}}" +CUSTOM_SEARCH_REALM_PASSWORD="{{.CustomSearchRealmPassword}}" +MESSAGE_OF_THE_DAY="{{.MessageOfTheDay}}" +HAS_KUBELET_DISK_TYPE="{{.HasKubeletDiskType}}" +NEEDS_CGROUPV2="{{.NeedsCgroupV2}}" +SYSCTL_CONTENT="{{.SysctlContent}}" +TLS_BOOTSTRAP_TOKEN="{{.TLSBootstrapToken}}" +KUBELET_FLAGS="{{.KubeletFlags}}" +KUBELET_NODE_LABELS="{{.KubeletNodeLabels}}" +AZURE_ENVIRONMENT_FILEPATH="{{.AzureEnvironmentFilepath}}" +KUBE_CA_CRT="{{.KubeCACrt}}" +KUBENET_TEMPLATE="{{.KubenetTemplate}}" +CONTAINERD_CONFIG_CONTENT="{{.ContainerdConfigContent}}" +IS_KATA="{{.IsKata}}" +/usr/bin/nohup /bin/bash -c "/bin/bash /opt/azure/containers/provision_start.sh" diff --git a/pkg/providers/imagefamily/bootstrap/kubenet-cni.json.gtpl b/pkg/providers/imagefamily/bootstrap/kubenet-cni.json.gtpl new file mode 100644 index 000000000..8fc68d997 --- /dev/null +++ b/pkg/providers/imagefamily/bootstrap/kubenet-cni.json.gtpl @@ -0,0 +1,24 @@ +{ + "cniVersion": "0.3.1", + "name": "kubenet", + "plugins": [{ + "type": "bridge", + "bridge": "cbr0", + "mtu": 1500, + "addIf": "eth0", + "isGateway": true, + "ipMasq": false, + "promiscMode": true, + "hairpinMode": false, + "ipam": { + "type": "host-local", + "ranges": [{{range $i, $range := .PodCIDRRanges}}{{if $i}}, {{end}}[{"subnet": "{{$range}}"}]{{end}}], + "routes": [{{range $i, $route := .Routes}}{{if $i}}, {{end}}{"dst": "{{$route}}"}{{end}}] + } + }, + { + "type": "portmap", + "capabilities": {"portMappings": true}, + "externalSetMarkChain": "KUBE-MARK-MASQ" + }] +} diff --git a/pkg/providers/imagefamily/bootstrap/sysctl.conf b/pkg/providers/imagefamily/bootstrap/sysctl.conf new file mode 100644 index 000000000..a8ddb715e --- /dev/null +++ b/pkg/providers/imagefamily/bootstrap/sysctl.conf @@ -0,0 +1,10 @@ +# This is a partial workaround to this upstream Kubernetes issue: +# https://github.com/kubernetes/kubernetes/issues/41916#issuecomment-312428731 +net.ipv4.tcp_retries2=8 +net.core.message_burst=80 +net.core.message_cost=40 +net.core.somaxconn=16384 +net.ipv4.tcp_max_syn_backlog=16384 +net.ipv4.neigh.default.gc_thresh1=4096 +net.ipv4.neigh.default.gc_thresh2=8192 +net.ipv4.neigh.default.gc_thresh3=16384 diff --git a/pkg/providers/imagefamily/image.go b/pkg/providers/imagefamily/image.go new file mode 100644 index 000000000..98ec39d98 --- /dev/null +++ b/pkg/providers/imagefamily/image.go @@ -0,0 +1,155 @@ +/* +Portions Copyright (c) Microsoft Corporation. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package imagefamily + +import ( + "context" + "fmt" + "regexp" + "strings" + "time" + + "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute/v5" + "github.com/Azure/karpenter/pkg/apis/v1alpha2" + "github.com/aws/karpenter-core/pkg/cloudprovider" + "github.com/aws/karpenter-core/pkg/scheduling" + "github.com/aws/karpenter-core/pkg/utils/pretty" + "github.com/patrickmn/go-cache" + "github.com/samber/lo" + "k8s.io/client-go/kubernetes" + "knative.dev/pkg/logging" +) + +type Provider struct { + kubernetesVersionCache *cache.Cache + cm *pretty.ChangeMonitor + location string + kubernetesInterface kubernetes.Interface + imageCache *cache.Cache + imageVersionsClient CommunityGalleryImageVersionsAPI +} + +const ( + kubernetesVersionCacheKey = "kubernetesVersion" + + imageExpirationInterval = time.Hour * 24 * 3 + imageCacheCleaningInterval = time.Hour * 1 + + imageIDFormat = "/CommunityGalleries/%s/images/%s/versions/%s" +) + +func NewProvider(kubernetesInterface kubernetes.Interface, kubernetesVersionCache *cache.Cache, versionsClient CommunityGalleryImageVersionsAPI, location string) *Provider { + return &Provider{ + kubernetesVersionCache: kubernetesVersionCache, + imageCache: cache.New(imageExpirationInterval, imageCacheCleaningInterval), + location: location, + imageVersionsClient: versionsClient, + cm: pretty.NewChangeMonitor(), + kubernetesInterface: kubernetesInterface, + } +} + +// Get returns Image ID for the given instance type. Images may vary due to architecture, accelerator, etc +func (p *Provider) Get(ctx context.Context, nodeClass *v1alpha2.AKSNodeClass, instanceType *cloudprovider.InstanceType, imageFamily ImageFamily) (string, error) { + if !nodeClass.Spec.IsEmptyImageID() { + logging.FromContext(ctx).Debugf("Using user-provided image %s", *nodeClass.Spec.ImageID) + return *nodeClass.Spec.ImageID, nil + } + + defaultImages := imageFamily.DefaultImages() + for _, defaultImage := range defaultImages { + if err := instanceType.Requirements.Compatible(defaultImage.Requirements, scheduling.AllowUndefinedWellKnownLabelsV1Beta1); err == nil { + communityImageName, publicGalleryURL := defaultImage.CommunityImage, defaultImage.PublicGalleryURL + return p.GetImageID(communityImageName, publicGalleryURL, nodeClass.Spec.GetImageVersion()) + } + } + + return "", fmt.Errorf("no compatible images found for instance type %s", instanceType.Name) +} + +func (p *Provider) KubeServerVersion(ctx context.Context) (string, error) { + if version, ok := p.kubernetesVersionCache.Get(kubernetesVersionCacheKey); ok { + return version.(string), nil + } + serverVersion, err := p.kubernetesInterface.Discovery().ServerVersion() + if err != nil { + return "", err + } + version := strings.TrimPrefix(serverVersion.GitVersion, "v") // v1.24.9 -> 1.24.9 + p.kubernetesVersionCache.SetDefault(kubernetesVersionCacheKey, version) + if p.cm.HasChanged("kubernetes-version", version) { + logging.FromContext(ctx).With("kubernetes-version", version).Debugf("discovered kubernetes version") + } + return version, nil +} + +// Input versionName == "" to get the latest version +func (p *Provider) GetImageID(communityImageName, publicGalleryURL, versionName string) (string, error) { + key := fmt.Sprintf("%s/%s/%s", publicGalleryURL, communityImageName, versionName) + imageID, found := p.imageCache.Get(key) + if found { + return imageID.(string), nil + } + + if versionName == "" { + pager := p.imageVersionsClient.NewListPager(p.location, publicGalleryURL, communityImageName, nil) + topImageVersionCandidate := armcompute.CommunityGalleryImageVersion{} + for pager.More() { + page, err := pager.NextPage(context.Background()) + if err != nil { + return "", err + } + for _, imageVersion := range page.CommunityGalleryImageVersionList.Value { + if lo.IsEmpty(topImageVersionCandidate) || imageVersion.Properties.PublishedDate.After(*topImageVersionCandidate.Properties.PublishedDate) { + topImageVersionCandidate = *imageVersion + } + } + } + versionName = lo.FromPtr(topImageVersionCandidate.Name) + } + + selectedImageID := BuildImageID(publicGalleryURL, communityImageName, versionName) + if p.cm.HasChanged(key, selectedImageID) { + logging.FromContext(context.Background()).With("image-id", selectedImageID).Info("discovered new image id") + } + p.imageCache.Set(key, selectedImageID, imageExpirationInterval) + return selectedImageID, nil +} + +func BuildImageID(publicGalleryURL, communityImageName, imageVersion string) string { + return fmt.Sprintf(imageIDFormat, publicGalleryURL, communityImageName, imageVersion) +} + +// ParseImageIDInfo parses the publicGalleryURL, communityImageName, and imageVersion out of an imageID +func ParseCommunityImageIDInfo(imageID string) (string, string, string, error) { + // TODO (charliedmcb): assess if doing validation on splitting the string and validating the results is better? Mostly is regex too expensive? + // Discussion on the validation here: + // https://github.com/Azure/karpenter/pull/280#discussion_r1343252930 + regexStr := fmt.Sprintf(imageIDFormat, "(?P.*)", "(?P.*)", "(?P.*)") + if imageID == "" { + return "", "", "", fmt.Errorf("can not parse empty string. Expect it of the form \"%s\"", regexStr) + } + r := regexp.MustCompile(regexStr) + matches := r.FindStringSubmatch(imageID) + if matches == nil { + return "", "", "", fmt.Errorf("no matches while parsing image id %s", imageID) + } + if r.SubexpIndex("publicGalleryURL") == -1 || r.SubexpIndex("communityImageName") == -1 || r.SubexpIndex("imageVersion") == -1 { + return "", "", "", fmt.Errorf("failed to find sub expressions in %s, for imageID: %s", regexStr, imageID) + } + return matches[r.SubexpIndex("publicGalleryURL")], matches[r.SubexpIndex("communityImageName")], matches[r.SubexpIndex("imageVersion")], nil +} diff --git a/pkg/providers/imagefamily/image_test.go b/pkg/providers/imagefamily/image_test.go new file mode 100644 index 000000000..35917e0f9 --- /dev/null +++ b/pkg/providers/imagefamily/image_test.go @@ -0,0 +1,201 @@ +/* +Portions Copyright (c) Microsoft Corporation. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package imagefamily_test + +import ( + "context" + "fmt" + "testing" + "time" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute/v5" + "github.com/Azure/karpenter/pkg/apis/v1alpha2" + "github.com/Azure/karpenter/pkg/fake" + "github.com/Azure/karpenter/pkg/providers/imagefamily" + "github.com/Azure/karpenter/pkg/test" + corev1beta1 "github.com/aws/karpenter-core/pkg/apis/v1beta1" + "github.com/aws/karpenter-core/pkg/cloudprovider" + "github.com/aws/karpenter-core/pkg/scheduling" + "github.com/samber/lo" + corev1 "k8s.io/api/core/v1" +) + +var imageProvider *imagefamily.Provider + +func TestAzure(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "Providers/ImageProvider/Azure") +} + +const ( + testImageID = "/CommunityGalleries/previewaks-1a06572d-8508-419c-a0d1-baffcbcb2f3b/images/2204gen2containerd/Versions/1.1685741267.25933" + olderImageVersion = "1.1686127203.20214" + latestImageVersion = "1.1686127203.20217" +) + +var _ = BeforeSuite(func() { + location := fake.Region + + defaultImageVersions := []*armcompute.CommunityGalleryImageVersion{ + { + Name: lo.ToPtr("1.1686127203.20215"), + Location: &location, + Type: lo.ToPtr("Microsoft.Compute/galleries/images/versions"), + Properties: &armcompute.CommunityGalleryImageVersionProperties{ + PublishedDate: lo.ToPtr(time.Now().Add(time.Minute * -10)), + }, + }, + { + Name: lo.ToPtr("1.1686127203.20213"), + Location: &location, + Type: lo.ToPtr("Microsoft.Compute/galleries/images/versions"), + Properties: &armcompute.CommunityGalleryImageVersionProperties{ + PublishedDate: lo.ToPtr(time.Now().Add(time.Minute * -20)), + }, + }, + { + Name: lo.ToPtr(latestImageVersion), + Location: &location, + Type: lo.ToPtr("Microsoft.Compute/galleries/images/versions"), + Properties: &armcompute.CommunityGalleryImageVersionProperties{ + PublishedDate: lo.ToPtr(time.Now().Add(time.Minute * -5)), + }, + }, + { + Name: lo.ToPtr(olderImageVersion), + Location: &location, + Type: lo.ToPtr("Microsoft.Compute/galleries/images/versions"), + Properties: &armcompute.CommunityGalleryImageVersionProperties{ + PublishedDate: lo.ToPtr(time.Now().Add(time.Minute * -15)), + }, + }, + { + Name: lo.ToPtr("1.1686127203.20216"), + Location: &location, + Type: lo.ToPtr("Microsoft.Compute/galleries/images/versions"), + Properties: &armcompute.CommunityGalleryImageVersionProperties{ + PublishedDate: lo.ToPtr(time.Now().Add(time.Minute * -7)), + }, + }, + } + + versionsClient := &fake.CommunityGalleryImageVersionsAPI{} + versionsClient.ImageVersions.Append(defaultImageVersions...) + imageProvider = imagefamily.NewProvider(nil, nil, versionsClient, fake.Region) +}) + +func newTestNodeClass(imageID, imageVersion string) *v1alpha2.AKSNodeClass { + nodeClass := test.AKSNodeClass() + + if imageID != "" { + nodeClass.Spec.ImageID = lo.ToPtr(imageID) + } + if imageVersion != "" { + nodeClass.Spec.ImageVersion = lo.ToPtr(imageVersion) + } + return nodeClass +} + +var _ = Describe("Image ID Resolution", func() { + var ( + nodeClassWithImageID = newTestNodeClass(testImageID, "") + nodeClassWithImageIDAndVersion = newTestNodeClass(testImageID, olderImageVersion) + nodeClassWithImageVersion = newTestNodeClass("", olderImageVersion) + ) + + DescribeTable("Resolution Of Image ID", + func(nodeClass *v1alpha2.AKSNodeClass, instanceType *cloudprovider.InstanceType, imageFamily interface{}, expectedImageID string) { + imageID, err := imageProvider.Get(context.Background(), nodeClass, instanceType, imagefamily.Ubuntu2204{}) + Expect(imageID).To(Equal(expectedImageID)) + Expect(err).To(BeNil()) + }, + Entry("Image ID is specified in the NodeClass", nodeClassWithImageID, &cloudprovider.InstanceType{}, imagefamily.Ubuntu2204{}, testImageID), + Entry("Image ID and ImageVersion are specified in the NodeClass", nodeClassWithImageIDAndVersion, &cloudprovider.InstanceType{}, imagefamily.Ubuntu2204{}, testImageID), + Entry("ImageVersion is specified in the NodeClass", nodeClassWithImageVersion, &cloudprovider.InstanceType{}, imagefamily.Ubuntu2204{}, fmt.Sprintf("/CommunityGalleries/%s/images/%s/versions/%s", imagefamily.AKSUbuntuPublicGalleryURL, imagefamily.Ubuntu2204Gen2CommunityImage, olderImageVersion)), + ) + + DescribeTable("Resolution Of Image ID", + func(communityImageName, publicGalleryURL, versionName string, expectedImageID string) { + imageID, err := imageProvider.GetImageID(communityImageName, publicGalleryURL, versionName) + Expect(imageID).To(Equal(expectedImageID)) + Expect(err).To(BeNil()) + }, + Entry("Image version is empty, should get latest", imagefamily.Ubuntu2204Gen2CommunityImage, imagefamily.AKSUbuntuPublicGalleryURL, "", fmt.Sprintf("/CommunityGalleries/%s/images/%s/versions/%s", imagefamily.AKSUbuntuPublicGalleryURL, imagefamily.Ubuntu2204Gen2CommunityImage, latestImageVersion)), + Entry("Image version is specified, should use it", imagefamily.Ubuntu2204Gen2CommunityImage, imagefamily.AKSUbuntuPublicGalleryURL, olderImageVersion, fmt.Sprintf("/CommunityGalleries/%s/images/%s/versions/%s", imagefamily.AKSUbuntuPublicGalleryURL, imagefamily.Ubuntu2204Gen2CommunityImage, olderImageVersion)), + ) + + DescribeTable("Ubuntu2204 Image Resolution", + func(nodeClass *v1alpha2.AKSNodeClass, instanceType *cloudprovider.InstanceType, expectedImageID string) { + imageID, err := imageProvider.Get(context.Background(), nodeClass, instanceType, imagefamily.Ubuntu2204{}) + Expect(err).To(BeNil()) + Expect(imageID).To(Equal(expectedImageID)) + }, + Entry("Arm64 Image for HyperV Gen 2", + test.AKSNodeClass(), + &cloudprovider.InstanceType{ + Name: "Standard_D8pls_v5", + Requirements: scheduling.NewRequirements(scheduling.NewRequirement(corev1.LabelArchStable, corev1.NodeSelectorOpIn, corev1beta1.ArchitectureArm64), scheduling.NewRequirement(v1alpha2.LabelSKUHyperVGeneration, corev1.NodeSelectorOpIn, v1alpha2.HyperVGenerationV2)), + }, + fmt.Sprintf("/CommunityGalleries/%s/images/2204gen2arm64containerd/versions/1.1686127203.20217", imagefamily.AKSUbuntuPublicGalleryURL)), + Entry("Gen2 Image for HyperV Gen 2", + test.AKSNodeClass(), + &cloudprovider.InstanceType{ + Name: "Standard_D2s_v3", + Requirements: scheduling.NewRequirements( + // scheduling.NewRequirement(v1.LabelArchStable, v1.NodeSelectorOpIn, corev1beta1.ArchitectureAmd64), + scheduling.NewRequirement(v1alpha2.LabelSKUHyperVGeneration, corev1.NodeSelectorOpIn, v1alpha2.HyperVGenerationV2), + ), + }, + fmt.Sprintf("/CommunityGalleries/%s/images/2204gen2containerd/versions/1.1686127203.20217", imagefamily.AKSUbuntuPublicGalleryURL)), + Entry("Gen1 Image for HyperV Gen 1", + test.AKSNodeClass(), + &cloudprovider.InstanceType{ + Name: "Standard_D2s_v3", + Requirements: scheduling.NewRequirements( + // scheduling.NewRequirement(v1.LabelArchStable, v1.NodeSelectorOpIn, corev1beta1.ArchitectureAmd64), + scheduling.NewRequirement(v1alpha2.LabelSKUHyperVGeneration, corev1.NodeSelectorOpIn, v1alpha2.HyperVGenerationV1), + ), + }, + fmt.Sprintf("/CommunityGalleries/%s/images/2204containerd/versions/1.1686127203.20217", imagefamily.AKSUbuntuPublicGalleryURL)), + Entry("Gen2 Image if no preference is specified", + test.AKSNodeClass(), + &cloudprovider.InstanceType{Name: "Standard_D2s_v3", Requirements: scheduling.NewRequirements()}, + fmt.Sprintf("/CommunityGalleries/%s/images/2204gen2containerd/versions/1.1686127203.20217", imagefamily.AKSUbuntuPublicGalleryURL)), + ) +}) + +var _ = Describe("Image ID Parsing", func() { + DescribeTable("Parse Image ID", + func(imageID string, expectedPublicGalleryURL, expectedCommunityImageName, expectedImageVersion string, expectError bool) { + publicGalleryURL, communityImageName, imageVersion, err := imagefamily.ParseCommunityImageIDInfo(imageID) + if expectError { + Expect(err).To(HaveOccurred()) + return + } + Expect(err).To(BeNil()) + Expect(publicGalleryURL).To(Equal(expectedPublicGalleryURL)) + Expect(communityImageName).To(Equal(expectedCommunityImageName)) + Expect(imageVersion).To(Equal(expectedImageVersion)) + }, + Entry("Valid image id should parse", fmt.Sprintf("/CommunityGalleries/%s/images/%s/versions/%s", imagefamily.AKSUbuntuPublicGalleryURL, imagefamily.Ubuntu2204Gen2CommunityImage, olderImageVersion), imagefamily.AKSUbuntuPublicGalleryURL, imagefamily.Ubuntu2204Gen2CommunityImage, olderImageVersion, nil), + Entry("invalid image id should not parse", "badimageid", "", "", "", true), + Entry("empty image id should not parse", "badimageid", "", "", "", true), + ) +}) diff --git a/pkg/providers/imagefamily/resolver.go b/pkg/providers/imagefamily/resolver.go new file mode 100644 index 000000000..b9f2019cc --- /dev/null +++ b/pkg/providers/imagefamily/resolver.go @@ -0,0 +1,130 @@ +/* +Portions Copyright (c) Microsoft Corporation. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package imagefamily + +import ( + "context" + + core "k8s.io/api/core/v1" + "knative.dev/pkg/logging" + "sigs.k8s.io/controller-runtime/pkg/client" + + "github.com/Azure/karpenter/pkg/apis/v1alpha2" + "github.com/Azure/karpenter/pkg/metrics" + "github.com/Azure/karpenter/pkg/providers/imagefamily/bootstrap" + "github.com/Azure/karpenter/pkg/providers/instancetype" + template "github.com/Azure/karpenter/pkg/providers/launchtemplate/parameters" + corev1beta1 "github.com/aws/karpenter-core/pkg/apis/v1beta1" + "github.com/aws/karpenter-core/pkg/cloudprovider" + "github.com/samber/lo" +) + +const ( + networkPluginAzureCNIOverlay = "overlay" + networkPluginKubenet = "kubenet" + + // defaultKubernetesMaxPodsAzureCNIOverlay is the maximum number of pods to run on a node for Azure CNI Overlay. + defaultKubernetesMaxPodsAzureCNIOverlay = 250 + // defaultKubernetesMaxPodsKubenet is the maximum number of pods to run on a node for Kubenet. + defaultKubernetesMaxPodsKubenet = 100 + // defaultKubernetesMaxPods is the maximum number of pods on a node. + defaultKubernetesMaxPods = 110 +) + +// Resolver is able to fill-in dynamic launch template parameters +type Resolver struct { + imageProvider *Provider +} + +// ImageFamily can be implemented to override the default logic for generating dynamic launch template parameters +type ImageFamily interface { + UserData( + kubeletConfig *corev1beta1.KubeletConfiguration, + taints []core.Taint, + labels map[string]string, + caBundle *string, + instanceType *cloudprovider.InstanceType, + ) bootstrap.Bootstrapper + Name() string + // DefaultImages returns a list of default CommunityImage definitions for this ImageFamily. + // Our Image Selection logic relies on the ordering of the default images to be ordered from most preferred to least, then we will select the latest image version available for that CommunityImage definition. + // Our Release pipeline ensures all images are released together within 24 hours of each other for community image gallery, so selecting based on image feature priorities, then by date, and not vice-versa is acceptable. + DefaultImages() []DefaultImageOutput +} + +// New constructs a new launch template Resolver +func New(_ client.Client, imageProvider *Provider) *Resolver { + return &Resolver{ + imageProvider: imageProvider, + } +} + +// Resolve fills in dynamic launch template parameters +func (r Resolver) Resolve(ctx context.Context, nodeClass *v1alpha2.AKSNodeClass, nodeClaim *corev1beta1.NodeClaim, instanceType *cloudprovider.InstanceType, + staticParameters *template.StaticParameters) (*template.Parameters, error) { + imageFamily := getImageFamily(nodeClass.Spec.ImageFamily, staticParameters) + imageID, err := r.imageProvider.Get(ctx, nodeClass, instanceType, imageFamily) + if err != nil { + metrics.ImageSelectionErrorCount.WithLabelValues(imageFamily.Name()).Inc() + return nil, err + } + + kubeletConfig := nodeClaim.Spec.Kubelet + if kubeletConfig == nil { + kubeletConfig = &corev1beta1.KubeletConfiguration{} + } + + // TODO: revist computeResources and maxPods implementation + kubeletConfig.KubeReserved = instanceType.Overhead.KubeReserved + kubeletConfig.SystemReserved = instanceType.Overhead.SystemReserved + kubeletConfig.EvictionHard = map[string]string{ + instancetype.MemoryAvailable: instanceType.Overhead.EvictionThreshold.Memory().String()} + kubeletConfig.MaxPods = lo.ToPtr(getMaxPods(staticParameters.NetworkPlugin)) + + logging.FromContext(ctx).Infof("Resolved image %s for instance type %s", imageID, instanceType.Name) + template := &template.Parameters{ + StaticParameters: staticParameters, + UserData: imageFamily.UserData( + kubeletConfig, + append(nodeClaim.Spec.Taints, nodeClaim.Spec.StartupTaints...), + staticParameters.Labels, + staticParameters.CABundle, + instanceType, + ), + ImageID: imageID, + } + + return template, nil +} + +func getImageFamily(familyName *string, parameters *template.StaticParameters) ImageFamily { + switch lo.FromPtr(familyName) { + case Ubuntu2204ImageFamily: + return &Ubuntu2204{Options: parameters} + default: + return &Ubuntu2204{Options: parameters} + } +} + +func getMaxPods(networkPlugin string) int32 { + if networkPlugin == networkPluginAzureCNIOverlay { + return defaultKubernetesMaxPodsAzureCNIOverlay + } else if networkPlugin == networkPluginKubenet { + return defaultKubernetesMaxPodsKubenet + } + return defaultKubernetesMaxPods +} diff --git a/pkg/providers/imagefamily/types.go b/pkg/providers/imagefamily/types.go new file mode 100644 index 000000000..cc2f9b4ec --- /dev/null +++ b/pkg/providers/imagefamily/types.go @@ -0,0 +1,39 @@ +/* +Portions Copyright (c) Microsoft Corporation. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package imagefamily + +import ( + "github.com/Azure/azure-sdk-for-go/sdk/azcore/runtime" + armcomputev5 "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute/v5" + "github.com/aws/karpenter-core/pkg/scheduling" +) + +const ( + AKSUbuntuPublicGalleryURL = "AKSUbuntu-38d80f77-467a-481f-a8d4-09b6d4220bd2" +) + +// DefaultImageOutput is the Stub of an Image we return from an ImageFamily De +type DefaultImageOutput struct { + CommunityImage string + PublicGalleryURL string + Requirements scheduling.Requirements +} + +// CommunityGalleryImageVersionsAPI is used for listing community gallery image versions. +type CommunityGalleryImageVersionsAPI interface { + NewListPager(location string, publicGalleryName string, galleryImageName string, options *armcomputev5.CommunityGalleryImageVersionsClientListOptions) *runtime.Pager[armcomputev5.CommunityGalleryImageVersionsClientListResponse] +} diff --git a/pkg/providers/imagefamily/ubuntu_2204.go b/pkg/providers/imagefamily/ubuntu_2204.go new file mode 100644 index 000000000..71c201ea2 --- /dev/null +++ b/pkg/providers/imagefamily/ubuntu_2204.go @@ -0,0 +1,109 @@ +/* +Portions Copyright (c) Microsoft Corporation. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package imagefamily + +import ( + v1 "k8s.io/api/core/v1" + + "github.com/Azure/karpenter/pkg/apis/v1alpha2" + "github.com/Azure/karpenter/pkg/providers/imagefamily/bootstrap" + "github.com/Azure/karpenter/pkg/providers/launchtemplate/parameters" + "github.com/Azure/karpenter/pkg/utils" + + corev1beta1 "github.com/aws/karpenter-core/pkg/apis/v1beta1" + "github.com/aws/karpenter-core/pkg/cloudprovider" + "github.com/aws/karpenter-core/pkg/scheduling" +) + +const ( + Ubuntu2204ImageFamily = "Ubuntu2204" + Ubuntu2204Gen2CommunityImage = "2204gen2containerd" + Ubuntu2204Gen1CommunityImage = "2204containerd" + Ubuntu2204Gen2ArmCommunityImage = "2204gen2arm64containerd" +) + +type Ubuntu2204 struct { + Options *parameters.StaticParameters +} + +func (u Ubuntu2204) Name() string { + return Ubuntu2204ImageFamily +} + +func (u Ubuntu2204) DefaultImages() []DefaultImageOutput { + // image provider will select these images in order, first match wins. This is why we chose to put Ubuntu2204Gen2containerd first in the defaultImages + return []DefaultImageOutput{ + { + CommunityImage: Ubuntu2204Gen2CommunityImage, + PublicGalleryURL: AKSUbuntuPublicGalleryURL, + Requirements: scheduling.NewRequirements( + scheduling.NewRequirement(v1.LabelArchStable, v1.NodeSelectorOpIn, corev1beta1.ArchitectureAmd64), + scheduling.NewRequirement(v1alpha2.LabelSKUHyperVGeneration, v1.NodeSelectorOpIn, v1alpha2.HyperVGenerationV2), + ), + }, + { + CommunityImage: Ubuntu2204Gen1CommunityImage, + PublicGalleryURL: AKSUbuntuPublicGalleryURL, + Requirements: scheduling.NewRequirements( + scheduling.NewRequirement(v1.LabelArchStable, v1.NodeSelectorOpIn, corev1beta1.ArchitectureAmd64), + scheduling.NewRequirement(v1alpha2.LabelSKUHyperVGeneration, v1.NodeSelectorOpIn, v1alpha2.HyperVGenerationV1), + ), + }, + { + CommunityImage: Ubuntu2204Gen2ArmCommunityImage, + PublicGalleryURL: AKSUbuntuPublicGalleryURL, + Requirements: scheduling.NewRequirements( + scheduling.NewRequirement(v1.LabelArchStable, v1.NodeSelectorOpIn, corev1beta1.ArchitectureArm64), + scheduling.NewRequirement(v1alpha2.LabelSKUHyperVGeneration, v1.NodeSelectorOpIn, v1alpha2.HyperVGenerationV2), + ), + }, + } +} + +// UserData returns the default userdata script for the image Family +func (u Ubuntu2204) UserData(kubeletConfig *corev1beta1.KubeletConfiguration, taints []v1.Taint, labels map[string]string, caBundle *string, instanceType *cloudprovider.InstanceType) bootstrap.Bootstrapper { + var arch string = corev1beta1.ArchitectureAmd64 + if err := instanceType.Requirements.Compatible(scheduling.NewRequirements(scheduling.NewRequirement(v1.LabelArchStable, v1.NodeSelectorOpIn, corev1beta1.ArchitectureArm64))); err == nil { + arch = corev1beta1.ArchitectureArm64 + } + return bootstrap.AKS{ + Options: bootstrap.Options{ + ClusterName: u.Options.ClusterName, + ClusterEndpoint: u.Options.ClusterEndpoint, + KubeletConfig: kubeletConfig, + Taints: taints, + Labels: labels, + CABundle: caBundle, + // TODO: Move common calculations that can be shared across image families + // to shared options struct the user data can reference + GPUNode: utils.IsNvidiaEnabledSKU(instanceType.Name), + GPUDriverVersion: utils.GetGPUDriverVersion(instanceType.Name), + }, + Arch: arch, + TenantID: u.Options.TenantID, + SubscriptionID: u.Options.SubscriptionID, + Location: u.Options.Location, + UserAssignedIdentityID: u.Options.UserAssignedIdentityID, + ResourceGroup: u.Options.ResourceGroup, + ClusterID: u.Options.ClusterID, + APIServerName: u.Options.APIServerName, + KubeletClientTLSBootstrapToken: u.Options.KubeletClientTLSBootstrapToken, + NetworkPlugin: u.Options.NetworkPlugin, + NetworkPolicy: u.Options.NetworkPolicy, + KubernetesVersion: u.Options.KubernetesVersion, + } +} diff --git a/pkg/providers/instance/argutils.go b/pkg/providers/instance/argutils.go new file mode 100644 index 000000000..4b17f374a --- /dev/null +++ b/pkg/providers/instance/argutils.go @@ -0,0 +1,63 @@ +/* +Portions Copyright (c) Microsoft Corporation. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package instance + +import ( + "context" + "fmt" + + "github.com/Azure/azure-sdk-for-go/sdk/azcore/to" + arg "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resourcegraph/armresourcegraph" +) + +func NewQueryRequest(subscriptionID *string, query string) *arg.QueryRequest { + return &arg.QueryRequest{ + Query: &query, + Options: &arg.QueryRequestOptions{ + ResultFormat: to.Ptr(arg.ResultFormatObjectArray), + }, + Subscriptions: []*string{subscriptionID}, + } +} + +// Queries Azure Resource Graph using Resources() and returns a list of all pages of data. +func GetResourceData(ctx context.Context, client AzureResourceGraphAPI, req arg.QueryRequest) ([]Resource, error) { + dataRemaining := true // used to handle ARG responses > 1 page long + var data []Resource + for dataRemaining { + resp, err := client.Resources(ctx, req, nil) + if err != nil { + return nil, err + } + interfaceArray, ok := resp.Data.([]interface{}) + if !ok { + return nil, fmt.Errorf("type casting query response as interface array failed") + } + for i := range interfaceArray { + switch resource := interfaceArray[i].(type) { + case map[string]interface{}: + data = append(data, resource) + } + } + dataRemaining = false + if resp.SkipToken != nil { + req.Options.SkipToken = resp.SkipToken + dataRemaining = true + } + } + return data, nil +} diff --git a/pkg/providers/instance/armutils.go b/pkg/providers/instance/armutils.go new file mode 100644 index 000000000..11bd6adac --- /dev/null +++ b/pkg/providers/instance/armutils.go @@ -0,0 +1,127 @@ +/* +Portions Copyright (c) Microsoft Corporation. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package instance + +import ( + "context" + + sdkerrors "github.com/Azure/azure-sdk-for-go-extensions/pkg/errors" + "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute" + "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/network/armnetwork" +) + +func CreateVirtualMachine(ctx context.Context, client VirtualMachinesAPI, rg, vmName string, vm armcompute.VirtualMachine) (*armcompute.VirtualMachine, error) { + poller, err := client.BeginCreateOrUpdate(ctx, rg, vmName, vm, nil) + if err != nil { + return nil, err + } + res, err := poller.PollUntilDone(ctx, nil) + if err != nil { + return nil, err + } + return &res.VirtualMachine, nil +} + +func UpdateVirtualMachine(ctx context.Context, client VirtualMachinesAPI, rg, vmName string, updates armcompute.VirtualMachineUpdate) error { + poller, err := client.BeginUpdate(ctx, rg, vmName, updates, nil) + if err != nil { + return err + } + _, err = poller.PollUntilDone(ctx, nil) + if err != nil { + return err + } + return nil +} + +func deleteVirtualMachine(ctx context.Context, client VirtualMachinesAPI, rg, vmName string) error { + poller, err := client.BeginDelete(ctx, rg, vmName, nil) + if err != nil { + return err + } + _, err = poller.PollUntilDone(ctx, nil) + if err != nil { + if sdkerrors.IsNotFoundErr(err) { + return nil + } + return err + } + return nil +} + +func createVirtualMachineExtension(ctx context.Context, client VirtualMachineExtensionsAPI, rg, vmName, extensionName string, vmExt armcompute.VirtualMachineExtension) (*armcompute.VirtualMachineExtension, error) { + poller, err := client.BeginCreateOrUpdate(ctx, rg, vmName, extensionName, vmExt, nil) + if err != nil { + return nil, err + } + res, err := poller.PollUntilDone(ctx, nil) + if err != nil { + return nil, err + } + return &res.VirtualMachineExtension, nil +} + +func createNic(ctx context.Context, client NetworkInterfacesAPI, rg, nicName string, nic armnetwork.Interface) (*armnetwork.Interface, error) { + poller, err := client.BeginCreateOrUpdate(ctx, rg, nicName, nic, nil) + if err != nil { + return nil, err + } + res, err := poller.PollUntilDone(ctx, nil) + + if err != nil { + return nil, err + } + return &res.Interface, nil +} + +func deleteNic(ctx context.Context, client NetworkInterfacesAPI, rg, nicName string) error { + poller, err := client.BeginDelete(ctx, rg, nicName, nil) + if err != nil { + return err + } + _, err = poller.PollUntilDone(ctx, nil) + if err != nil { + if sdkerrors.IsNotFoundErr(err) { + return nil + } + return err + } + return nil +} + +func deleteNicIfExists(ctx context.Context, client NetworkInterfacesAPI, rg, nicName string) error { + _, err := client.Get(ctx, rg, nicName, nil) + if err != nil { + if sdkerrors.IsNotFoundErr(err) { + return nil + } + return err + } + return deleteNic(ctx, client, rg, nicName) +} + +// deleteVirtualMachineIfExists checks if a virtual machine exists, and if it does, we delete it with a cascading delete +func deleteVirtualMachineIfExists(ctx context.Context, client VirtualMachinesAPI, rg, vmName string) error { + _, err := client.Get(ctx, rg, vmName, nil) + if err != nil { + if sdkerrors.IsNotFoundErr(err) { + return nil + } + return err + } + return deleteVirtualMachine(ctx, client, rg, vmName) +} diff --git a/pkg/providers/instance/azure_client.go b/pkg/providers/instance/azure_client.go new file mode 100644 index 000000000..d51a68781 --- /dev/null +++ b/pkg/providers/instance/azure_client.go @@ -0,0 +1,190 @@ +/* +Portions Copyright (c) Microsoft Corporation. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package instance + +import ( + "context" + "fmt" + "os" + + // nolint SA1019 - deprecated package + + "github.com/samber/lo" + + "github.com/Azure/azure-sdk-for-go/sdk/azcore/runtime" + "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute" + armcomputev5 "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute/v5" + "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/network/armnetwork" + "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resourcegraph/armresourcegraph" + "github.com/Azure/go-autorest/autorest/azure" + "github.com/Azure/karpenter/pkg/auth" + "github.com/Azure/karpenter/pkg/providers/imagefamily" + "github.com/Azure/karpenter/pkg/providers/instance/skuclient" + "github.com/Azure/karpenter/pkg/providers/loadbalancer" + + armopts "github.com/Azure/karpenter/pkg/utils/opts" + klog "k8s.io/klog/v2" +) + +type VirtualMachinesAPI interface { + BeginCreateOrUpdate(ctx context.Context, resourceGroupName string, vmName string, parameters armcompute.VirtualMachine, options *armcompute.VirtualMachinesClientBeginCreateOrUpdateOptions) (*runtime.Poller[armcompute.VirtualMachinesClientCreateOrUpdateResponse], error) + Get(ctx context.Context, resourceGroupName string, vmName string, options *armcompute.VirtualMachinesClientGetOptions) (armcompute.VirtualMachinesClientGetResponse, error) + BeginUpdate(ctx context.Context, resourceGroupName string, vmName string, parameters armcompute.VirtualMachineUpdate, options *armcompute.VirtualMachinesClientBeginUpdateOptions) (*runtime.Poller[armcompute.VirtualMachinesClientUpdateResponse], error) + BeginDelete(ctx context.Context, resourceGroupName string, vmName string, options *armcompute.VirtualMachinesClientBeginDeleteOptions) (*runtime.Poller[armcompute.VirtualMachinesClientDeleteResponse], error) +} + +type AzureResourceGraphAPI interface { + Resources(ctx context.Context, query armresourcegraph.QueryRequest, options *armresourcegraph.ClientResourcesOptions) (armresourcegraph.ClientResourcesResponse, error) +} + +type VirtualMachineExtensionsAPI interface { + BeginCreateOrUpdate(ctx context.Context, resourceGroupName string, vmName string, vmExtensionName string, extensionParameters armcompute.VirtualMachineExtension, options *armcompute.VirtualMachineExtensionsClientBeginCreateOrUpdateOptions) (*runtime.Poller[armcompute.VirtualMachineExtensionsClientCreateOrUpdateResponse], error) +} + +type NetworkInterfacesAPI interface { + BeginCreateOrUpdate(ctx context.Context, resourceGroupName string, networkInterfaceName string, parameters armnetwork.Interface, options *armnetwork.InterfacesClientBeginCreateOrUpdateOptions) (*runtime.Poller[armnetwork.InterfacesClientCreateOrUpdateResponse], error) + BeginDelete(ctx context.Context, resourceGroupName string, networkInterfaceName string, options *armnetwork.InterfacesClientBeginDeleteOptions) (*runtime.Poller[armnetwork.InterfacesClientDeleteResponse], error) + Get(ctx context.Context, resourceGroupName string, networkInterfaceName string, options *armnetwork.InterfacesClientGetOptions) (armnetwork.InterfacesClientGetResponse, error) +} + +// TODO: Move this to another package that more correctly reflects its usage across multiple providers +type AZClient struct { + azureResourceGraphClient AzureResourceGraphAPI + virtualMachinesClient VirtualMachinesAPI + virtualMachinesExtensionClient VirtualMachineExtensionsAPI + networkInterfacesClient NetworkInterfacesAPI + + ImageVersionsClient imagefamily.CommunityGalleryImageVersionsAPI + // SKU CLIENT is still using track 1 because skewer does not support the track 2 path. We need to refactor this once skewer supports track 2 + SKUClient skuclient.SkuClient + LoadBalancersClient loadbalancer.LoadBalancersAPI +} + +func NewAZClientFromAPI( + virtualMachinesClient VirtualMachinesAPI, + azureResourceGraphClient AzureResourceGraphAPI, + virtualMachinesExtensionClient VirtualMachineExtensionsAPI, + interfacesClient NetworkInterfacesAPI, + loadBalancersClient loadbalancer.LoadBalancersAPI, + imageVersionsClient imagefamily.CommunityGalleryImageVersionsAPI, + skuClient skuclient.SkuClient, +) *AZClient { + return &AZClient{ + virtualMachinesClient: virtualMachinesClient, + azureResourceGraphClient: azureResourceGraphClient, + virtualMachinesExtensionClient: virtualMachinesExtensionClient, + networkInterfacesClient: interfacesClient, + ImageVersionsClient: imageVersionsClient, + SKUClient: skuClient, + LoadBalancersClient: loadBalancersClient, + } +} + +func CreateAzClient(ctx context.Context, cfg *auth.Config) (*AZClient, error) { + // Defaulting env to Azure Public Cloud. + env := azure.PublicCloud + var err error + if cfg.Cloud != "" { + env, err = azure.EnvironmentFromName(cfg.Cloud) + if err != nil { + return nil, err + } + } + + azClient, err := NewAZClient(ctx, cfg, &env) + if err != nil { + return nil, err + } + + return azClient, nil +} + +func handleVNET(cfg *auth.Config, vnetClient *armnetwork.VirtualNetworksClient) error { + vnet, err := vnetClient.Get(context.Background(), cfg.NodeResourceGroup, cfg.VnetName, nil) + if err != nil { + return err + } + if vnet.Properties == nil || vnet.Properties.ResourceGUID == nil { + return fmt.Errorf("vnet %s does not have a resource GUID", cfg.VnetName) + } + os.Setenv("AZURE_VNET_GUID", lo.FromPtr(vnet.Properties.ResourceGUID)) + return nil +} + +func NewAZClient(ctx context.Context, cfg *auth.Config, env *azure.Environment) (*AZClient, error) { + cred, err := auth.NewCredential(cfg) + if err != nil { + return nil, err + } + + opts := armopts.DefaultArmOpts() + extClient, err := armcompute.NewVirtualMachineExtensionsClient(cfg.SubscriptionID, cred, opts) + if err != nil { + return nil, err + } + interfacesClient, err := armnetwork.NewInterfacesClient(cfg.SubscriptionID, cred, opts) + if err != nil { + return nil, err + } + klog.V(5).Infof("Created network interface client %v using token credential", interfacesClient) + + vnetClient, err := armnetwork.NewVirtualNetworksClient(cfg.SubscriptionID, cred, opts) + if err != nil { + return nil, err + } + err = handleVNET(cfg, vnetClient) + if err != nil { + return nil, err + } + virtualMachinesClient, err := armcompute.NewVirtualMachinesClient(cfg.SubscriptionID, cred, opts) + if err != nil { + return nil, err + } + klog.V(5).Infof("Created virtual machines client %v, using a token credential", virtualMachinesClient) + azureResourceGraphClient, err := armresourcegraph.NewClient(cred, opts) + if err != nil { + return nil, err + } + klog.V(5).Infof("Created azure resource graph client %v, using a token credential", azureResourceGraphClient) + + imageVersionsClient, err := armcomputev5.NewCommunityGalleryImageVersionsClient(cfg.SubscriptionID, cred, opts) + if err != nil { + return nil, err + } + klog.V(5).Infof("Created image versions client %v, using a token credential", imageVersionsClient) + + loadBalancersClient, err := armnetwork.NewLoadBalancersClient(cfg.SubscriptionID, cred, opts) + if err != nil { + return nil, err + } + klog.V(5).Infof("Created load balancers client %v, using a token credential", loadBalancersClient) + + // TODO: this one is not enabled for rate limiting / throttling ... + // TODO Move this over to track 2 when skewer is migrated + skuClient := skuclient.NewSkuClient(ctx, cfg, env) + + return &AZClient{ + networkInterfacesClient: interfacesClient, + virtualMachinesClient: virtualMachinesClient, + virtualMachinesExtensionClient: extClient, + azureResourceGraphClient: azureResourceGraphClient, + + ImageVersionsClient: imageVersionsClient, + SKUClient: skuClient, + LoadBalancersClient: loadBalancersClient, + }, nil +} diff --git a/pkg/providers/instance/instance.go b/pkg/providers/instance/instance.go new file mode 100644 index 000000000..4ce985c44 --- /dev/null +++ b/pkg/providers/instance/instance.go @@ -0,0 +1,691 @@ +/* +Portions Copyright (c) Microsoft Corporation. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package instance + +import ( + "context" + "encoding/json" + "fmt" + "math" + "sort" + "strconv" + "strings" + "time" + + "github.com/samber/lo" + v1 "k8s.io/api/core/v1" + + "k8s.io/apimachinery/pkg/util/sets" + "knative.dev/pkg/logging" + + "github.com/Azure/azure-kusto-go/kusto/kql" + "github.com/Azure/karpenter/pkg/cache" + "github.com/Azure/karpenter/pkg/providers/instancetype" + "github.com/Azure/karpenter/pkg/providers/launchtemplate" + "github.com/Azure/karpenter/pkg/providers/loadbalancer" + "github.com/Azure/karpenter/pkg/utils" + + corecloudprovider "github.com/aws/karpenter-core/pkg/cloudprovider" + "github.com/aws/karpenter-core/pkg/scheduling" + + "github.com/Azure/karpenter/pkg/apis/settings" + "github.com/Azure/karpenter/pkg/apis/v1alpha2" + "github.com/aws/karpenter-core/pkg/apis/v1alpha5" + corev1beta1 "github.com/aws/karpenter-core/pkg/apis/v1beta1" + + //nolint SA1019 - deprecated package + "github.com/Azure/azure-sdk-for-go/services/compute/mgmt/2022-08-01/compute" + + sdkerrors "github.com/Azure/azure-sdk-for-go-extensions/pkg/errors" + "github.com/Azure/azure-sdk-for-go/sdk/azcore/to" + "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute" + "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/network/armnetwork" +) + +var ( + NodePoolTagKey = strings.ReplaceAll(corev1beta1.NodePoolLabelKey, "/", "_") + listQuery string + + CapacityTypeToPriority = map[string]string{ + corev1beta1.CapacityTypeSpot: string(compute.Spot), + corev1beta1.CapacityTypeOnDemand: string(compute.Regular), + } + PriorityToCapacityType = map[string]string{ + string(compute.Spot): corev1beta1.CapacityTypeSpot, + string(compute.Regular): corev1beta1.CapacityTypeOnDemand, + } + + SubscriptionQuotaReached = "SubscriptionQuotaReached" + ZonalAllocationFailure = "ZonalAllocationFailure" + + DurationForNewQuotaRequest = 1 * time.Hour +) + +type Resource = map[string]interface{} + +type Provider struct { + location string + azClient *AZClient + instanceTypeProvider *instancetype.Provider + launchTemplateProvider *launchtemplate.Provider + loadBalancerProvider *loadbalancer.Provider + resourceGroup string + subnetID string + subscriptionID string + unavailableOfferings *cache.UnavailableOfferings +} + +func NewProvider( + _ context.Context, // TODO: Why is this here? + azClient *AZClient, + instanceTypeProvider *instancetype.Provider, + launchTemplateProvider *launchtemplate.Provider, + loadBalancerProvider *loadbalancer.Provider, + offeringsCache *cache.UnavailableOfferings, + location string, + resourceGroup string, + subnetID string, + subscriptionID string, +) *Provider { + listQuery = GetListQueryBuilder(resourceGroup).String() + return &Provider{ + azClient: azClient, + instanceTypeProvider: instanceTypeProvider, + launchTemplateProvider: launchTemplateProvider, + loadBalancerProvider: loadBalancerProvider, + location: location, + resourceGroup: resourceGroup, + subnetID: subnetID, + subscriptionID: subscriptionID, + unavailableOfferings: offeringsCache, + } +} + +// Create an instance given the constraints. +// instanceTypes should be sorted by priority for spot capacity type. +func (p *Provider) Create(ctx context.Context, nodeClass *v1alpha2.AKSNodeClass, nodeClaim *corev1beta1.NodeClaim, instanceTypes []*corecloudprovider.InstanceType) (*armcompute.VirtualMachine, error) { + instanceTypes = orderInstanceTypesByPrice(instanceTypes, scheduling.NewNodeSelectorRequirements(nodeClaim.Spec.Requirements...)) + vm, instanceType, err := p.launchInstance(ctx, nodeClass, nodeClaim, instanceTypes) + if err != nil { + return nil, err + } + zone, err := GetZoneID(vm) + if err != nil { + logging.FromContext(ctx).Error(err) + } + logging.FromContext(ctx).With( + "launched-instance", *vm.ID, + "hostname", *vm.Name, + "type", string(*vm.Properties.HardwareProfile.VMSize), + "zone", zone, + "capacity-type", p.getPriorityForInstanceType(nodeClaim, instanceType)).Infof("launched new instance") + + return vm, nil +} + +func (p *Provider) Link(ctx context.Context, vmName, provisionerName string) error { + vm, err := p.Get(ctx, vmName) + if err != nil { + return fmt.Errorf("linking tags, %w", err) + } + updates := armcompute.VirtualMachineUpdate{ + Tags: lo.Assign(vm.Tags, map[string]*string{ + NodePoolTagKey: lo.ToPtr(provisionerName), + }), + } + if err = UpdateVirtualMachine(ctx, p.azClient.virtualMachinesClient, p.resourceGroup, vmName, updates); err != nil { + return fmt.Errorf("linking tags, %w", err) + } + return nil +} + +func (p *Provider) Update(ctx context.Context, vmName string, update armcompute.VirtualMachineUpdate) error { + return UpdateVirtualMachine(ctx, p.azClient.virtualMachinesClient, p.resourceGroup, vmName, update) +} + +func (p *Provider) Get(ctx context.Context, vmName string) (*armcompute.VirtualMachine, error) { + var vm armcompute.VirtualMachinesClientGetResponse + var err error + + if vm, err = p.azClient.virtualMachinesClient.Get(ctx, p.resourceGroup, vmName, nil); err != nil { + if sdkerrors.IsNotFoundErr(err) { + return nil, corecloudprovider.NewNodeClaimNotFoundError(err) + } + return nil, fmt.Errorf("failed to get VM instance, %w", err) + } + + return &vm.VirtualMachine, nil +} + +func (p *Provider) List(ctx context.Context) ([]*armcompute.VirtualMachine, error) { + req := NewQueryRequest(&(p.subscriptionID), listQuery) + client := p.azClient.azureResourceGraphClient + data, err := GetResourceData(ctx, client, *req) + if err != nil { + return nil, fmt.Errorf("querying azure resource graph, %w", err) + } + var vmList []*armcompute.VirtualMachine + for i := range data { + vm, err := createVMFromQueryResponseData(data[i]) + if err != nil { + return nil, fmt.Errorf("creating VM object from query response data, %w", err) + } + vmList = append(vmList, vm) + } + return vmList, nil +} + +func (p *Provider) Delete(ctx context.Context, nodeClaim *corev1beta1.NodeClaim) error { + vmName, err := utils.GetVMName(nodeClaim.Status.ProviderID) + if err != nil { + return fmt.Errorf("getting vm name, %w", err) + } + + logging.FromContext(ctx).Debugf("Deleting virtual machine %s", vmName) + return deleteVirtualMachine(ctx, p.azClient.virtualMachinesClient, p.resourceGroup, vmName) +} + +// createAKSIdentifyingExtension attaches a VM extension to identify that this VM participates in an AKS cluster +func (p *Provider) createAKSIdentifyingExtension( + ctx context.Context, + vmName, _ string, +) error { + var err error + vmExt := p.getAKSIdentifyingExtension() + vmExtName := *vmExt.Name + logging.FromContext(ctx).Debugf("Creating virtual machine AKS identifying extension for %s", vmName) + v, err := createVirtualMachineExtension(ctx, p.azClient.virtualMachinesExtensionClient, p.resourceGroup, vmName, vmExtName, *vmExt) + if err != nil { + logging.FromContext(ctx).Errorf("Creating VM AKS identifying extension for VM %q failed, %w", vmName, err) + vmErr := deleteVirtualMachine(ctx, p.azClient.virtualMachinesClient, p.resourceGroup, vmName) + if vmErr != nil { + logging.FromContext(ctx).Errorf("virtualMachine.Delete for %s failed: %v", vmName, vmErr) + } + return fmt.Errorf("creating VM AKS identifying extension for VM %q, %w failed", vmName, err) + } + logging.FromContext(ctx).Debugf("Created virtual machine AKS identifying extension for %s, with an id of %s", vmName, *v.ID) + return nil +} + +func (p *Provider) newNetworkInterfaceForVM(vmName string, backendPools *loadbalancer.BackendAddressPools, instanceType *corecloudprovider.InstanceType) armnetwork.Interface { + var ipv4BackendPools []*armnetwork.BackendAddressPool + for _, poolID := range backendPools.IPv4PoolIDs { + poolID := poolID + ipv4BackendPools = append(ipv4BackendPools, &armnetwork.BackendAddressPool{ + ID: &poolID, + }) + } + + skuAcceleratedNetworkingRequirements := scheduling.NewRequirements(scheduling.NewRequirement(v1alpha2.LabelSKUAcceleratedNetworking, v1.NodeSelectorOpIn, "true")) + + enableAcceleratedNetworking := false + if err := instanceType.Requirements.Compatible(skuAcceleratedNetworkingRequirements); err == nil { + enableAcceleratedNetworking = true + } + + return armnetwork.Interface{ + Location: to.Ptr(p.location), + Properties: &armnetwork.InterfacePropertiesFormat{ + IPConfigurations: []*armnetwork.InterfaceIPConfiguration{ + { + Name: &vmName, + Properties: &armnetwork.InterfaceIPConfigurationPropertiesFormat{ + Primary: to.Ptr(true), + PrivateIPAllocationMethod: to.Ptr(armnetwork.IPAllocationMethodDynamic), + Subnet: &armnetwork.Subnet{ + ID: &p.subnetID, + }, + LoadBalancerBackendAddressPools: ipv4BackendPools, + }, + }, + }, + EnableAcceleratedNetworking: to.Ptr(enableAcceleratedNetworking), + EnableIPForwarding: to.Ptr(true), + }, + } +} + +func GenerateVMName(nodeClaimName string) string { + return fmt.Sprintf("aks-%s", nodeClaimName) +} + +func (p *Provider) createNetworkInterface(ctx context.Context, nicName string, launchTemplateConfig *launchtemplate.Template, instanceType *corecloudprovider.InstanceType) (string, error) { + backendPools, err := p.loadBalancerProvider.LoadBalancerBackendPools(ctx) + if err != nil { + return "", err + } + + nic := p.newNetworkInterfaceForVM(nicName, backendPools, instanceType) + p.applyTemplateToNic(&nic, launchTemplateConfig) + logging.FromContext(ctx).Debugf("Creating network interface %s", nicName) + res, err := createNic(ctx, p.azClient.networkInterfacesClient, p.resourceGroup, nicName, nic) + if err != nil { + if nicErr := deleteNicIfExists(ctx, p.azClient.networkInterfacesClient, p.resourceGroup, nicName); nicErr != nil { + logging.FromContext(ctx).Errorf("networkInterface.Delete for %s failed: %v", nicName, nicErr) + } + return "", err + } + logging.FromContext(ctx).Debugf("Successfully created network interface: %v", *res.ID) + return *res.ID, nil +} + +// newVMObject is a helper func that creates a new armcompute.VirtualMachine +// from key input. +func newVMObject( + vmName, + nicReference, + zone, + capacityType string, + location string, + sshPublicKey string, + nodeIdentities []string, + nodeClass *v1alpha2.AKSNodeClass, + nodeClaim *corev1beta1.NodeClaim, + launchTemplate *launchtemplate.Template, + instanceType *corecloudprovider.InstanceType) armcompute.VirtualMachine { + // Build the image reference from template + imageReference := armcompute.ImageReference{} + if v1alpha2.IsComputeGalleryImageID(launchTemplate.ImageID) { + imageReference.ID = &launchTemplate.ImageID + } else { + imageReference.CommunityGalleryImageID = &launchTemplate.ImageID + } + vm := armcompute.VirtualMachine{ + Location: to.Ptr(location), + Identity: ConvertToVirtualMachineIdentity(nodeIdentities), + Properties: &armcompute.VirtualMachineProperties{ + HardwareProfile: &armcompute.HardwareProfile{ + VMSize: to.Ptr(armcompute.VirtualMachineSizeTypes(instanceType.Name)), + }, + + StorageProfile: &armcompute.StorageProfile{ + OSDisk: &armcompute.OSDisk{ + Name: to.Ptr(vmName), + DiskSizeGB: nodeClass.Spec.OSDiskSizeGB, + CreateOption: to.Ptr(armcompute.DiskCreateOptionTypesFromImage), + DeleteOption: to.Ptr(armcompute.DiskDeleteOptionTypesDelete), + }, + ImageReference: &imageReference, + }, + + NetworkProfile: &armcompute.NetworkProfile{ + NetworkInterfaces: []*armcompute.NetworkInterfaceReference{ + { + ID: &nicReference, + Properties: &armcompute.NetworkInterfaceReferenceProperties{ + Primary: to.Ptr(true), + DeleteOption: to.Ptr(armcompute.DeleteOptionsDelete), + }, + }, + }, + }, + + OSProfile: &armcompute.OSProfile{ + AdminUsername: to.Ptr("azureuser"), + ComputerName: &vmName, + LinuxConfiguration: &armcompute.LinuxConfiguration{ + DisablePasswordAuthentication: to.Ptr(true), + SSH: &armcompute.SSHConfiguration{ + PublicKeys: []*armcompute.SSHPublicKey{ + { + KeyData: to.Ptr(sshPublicKey), + Path: to.Ptr("/home/" + "azureuser" + "/.ssh/authorized_keys"), + }, + }, + }, + }, + CustomData: to.Ptr(launchTemplate.UserData), + }, + Priority: to.Ptr(armcompute.VirtualMachinePriorityTypes( + CapacityTypeToPriority[capacityType]), + ), + }, + Zones: []*string{&zone}, + Tags: launchTemplate.Tags, + } + setVMPropertiesStorageProfile(vm.Properties, instanceType, nodeClass) + setVMPropertiesBillingProfile(vm.Properties, capacityType) + setVMTagsProvisionerName(vm.Tags, nodeClaim) + + return vm +} + +// setVMPropertiesStorageProfile enables ephemeral os disk for instance types that support it +func setVMPropertiesStorageProfile(vmProperties *armcompute.VirtualMachineProperties, instanceType *corecloudprovider.InstanceType, nodeClass *v1alpha2.AKSNodeClass) { + // use ephemeral disk if it is large enough + if *nodeClass.Spec.OSDiskSizeGB <= getEphemeralMaxSizeGB(instanceType) { + vmProperties.StorageProfile.OSDisk.DiffDiskSettings = &armcompute.DiffDiskSettings{ + Option: to.Ptr(armcompute.DiffDiskOptionsLocal), + // placement (cache/resource) is left to CRP + } + vmProperties.StorageProfile.OSDisk.Caching = to.Ptr(armcompute.CachingTypesReadOnly) + } +} + +// setVMPropertiesBillingProfile sets a default MaxPrice of -1 for Spot +func setVMPropertiesBillingProfile(vmProperties *armcompute.VirtualMachineProperties, capacityType string) { + if capacityType == v1alpha5.CapacityTypeSpot { + vmProperties.EvictionPolicy = to.Ptr(armcompute.VirtualMachineEvictionPolicyTypesDelete) + vmProperties.BillingProfile = &armcompute.BillingProfile{ + MaxPrice: to.Ptr(float64(-1)), + } + } +} + +// setVMTagsProvisionerName sets "karpenter.sh/provisioner-name" tag +func setVMTagsProvisionerName(tags map[string]*string, nodeClaim *corev1beta1.NodeClaim) { + if val, ok := nodeClaim.Labels[corev1beta1.NodePoolLabelKey]; ok { + tags[NodePoolTagKey] = &val + } +} + +func (p *Provider) createVirtualMachine(ctx context.Context, vm armcompute.VirtualMachine, vmName, _ string) (*armcompute.VirtualMachine, error) { + result, err := CreateVirtualMachine(ctx, p.azClient.virtualMachinesClient, p.resourceGroup, vmName, vm) + if err != nil { + logging.FromContext(ctx).Errorf("Creating virtual machine %q failed: %v", vmName, err) + vmErr := deleteVirtualMachineIfExists(ctx, p.azClient.virtualMachinesClient, p.resourceGroup, vmName) + if vmErr != nil { + logging.FromContext(ctx).Errorf("virtualMachine.Delete for %s failed: %v", vmName, vmErr) + } + return nil, fmt.Errorf("virtualMachine.BeginCreateOrUpdate for VM %q failed: %w", vmName, err) + } + logging.FromContext(ctx).Debugf("Created virtual machine %s", *result.ID) + return result, nil +} + +func (p *Provider) launchInstance( + ctx context.Context, nodeClass *v1alpha2.AKSNodeClass, nodeClaim *corev1beta1.NodeClaim, instanceTypes []*corecloudprovider.InstanceType) (*armcompute.VirtualMachine, *corecloudprovider.InstanceType, error) { + instanceType, capacityType, zone := p.pickSkuSizePriorityAndZone(ctx, nodeClaim, instanceTypes) + if instanceType == nil { + return nil, nil, corecloudprovider.NewInsufficientCapacityError(fmt.Errorf("no instance types available")) + } + launchTemplate, err := p.getLaunchTemplate(ctx, nodeClass, nodeClaim, instanceType, capacityType) + if err != nil { + return nil, nil, fmt.Errorf("getting launch template: %w", err) + } + + vmName := GenerateVMName(nodeClaim.Name) + + // create network interface + nicName := vmName + nicReference, err := p.createNetworkInterface(ctx, vmName, launchTemplate, instanceType) + if err != nil { + return nil, nil, err + } + + sshPublicKey := settings.FromContext(ctx).SSHPublicKey + nodeIdentityIDs := settings.FromContext(ctx).NodeIdentities + vm := newVMObject(vmName, nicReference, zone, capacityType, p.location, sshPublicKey, nodeIdentityIDs, nodeClass, nodeClaim, launchTemplate, instanceType) + + logging.FromContext(ctx).Debugf("Creating virtual machine %s (%s)", vmName, instanceType.Name) + // Uses AZ Client to create a new virtual machine using the vm object we prepared earlier + resp, err := p.createVirtualMachine(ctx, vm, vmName, nicName) + if err != nil { + azErr := p.handleResponseErrors(ctx, instanceType, zone, capacityType, err) + return nil, nil, azErr + } + + err = p.createAKSIdentifyingExtension(ctx, vmName, nicName) + if err != nil { + return nil, nil, err + } + return resp, instanceType, nil +} + +func (p *Provider) handleResponseErrors(ctx context.Context, instanceType *corecloudprovider.InstanceType, zone, capacityType string, err error) error { + if sdkerrors.SubscriptionQuotaHasBeenReached(err) { + // Subscription quota reached, mark the instance type as unavailable in all zones available to the offering + // THis will also update the TTL for an existing offering in the cache that is already unavailable + for _, offering := range instanceType.Offerings { + if offering.CapacityType != capacityType { + continue + } + + // If we have a quota limit of 0 vcpus, we mark the offerings unavailable for an hour. + // CPU limits of 0 are usually due to a subscription having no allocated quota for that instance type at all on the subscription. + if cpuLimitIsZero(err) { + p.unavailableOfferings.MarkUnavailableWithTTL(ctx, SubscriptionQuotaReached, instanceType.Name, offering.Zone, capacityType, DurationForNewQuotaRequest) + } else { + p.unavailableOfferings.MarkUnavailable(ctx, SubscriptionQuotaReached, instanceType.Name, offering.Zone, capacityType) + } + } + } else if sdkerrors.ZonalAllocationFailureOccurred(err) { + p.unavailableOfferings.MarkUnavailable(ctx, ZonalAllocationFailure, instanceType.Name, zone, corev1beta1.CapacityTypeOnDemand) + p.unavailableOfferings.MarkUnavailable(ctx, ZonalAllocationFailure, instanceType.Name, zone, corev1beta1.CapacityTypeSpot) + } + return err +} + +func getEphemeralMaxSizeGB(instanceType *corecloudprovider.InstanceType) int32 { + reqs := instanceType.Requirements.Get(v1alpha2.LabelSKUStorageEphemeralOSMaxSize).Values() + if len(reqs) == 0 || len(reqs) > 1 { + return 0 + } + maxSize, err := strconv.ParseFloat(reqs[0], 32) + if err != nil { + return 0 + } + // decimal places are truncated, so we round down + return int32(maxSize) +} + +func cpuLimitIsZero(err error) bool { + return strings.Contains(err.Error(), "Current Limit: 0") +} + +func (p *Provider) applyTemplateToNic(nic *armnetwork.Interface, template *launchtemplate.Template) { + // set tags + nic.Tags = template.Tags +} + +func (p *Provider) getLaunchTemplate(ctx context.Context, nodeClass *v1alpha2.AKSNodeClass, nodeClaim *corev1beta1.NodeClaim, + instanceType *corecloudprovider.InstanceType, capacityType string) (*launchtemplate.Template, error) { + additionalLabels := lo.Assign(GetAllSingleValuedRequirementLabels(instanceType), map[string]string{corev1beta1.CapacityTypeLabelKey: capacityType}) + + launchTemplate, err := p.launchTemplateProvider.GetTemplate(ctx, nodeClass, nodeClaim, instanceType, additionalLabels) + if err != nil { + return nil, fmt.Errorf("getting launch templates, %w", err) + } + + return launchTemplate, nil +} + +// GetAllSingleValuedRequirementLabels converts instanceType.Requirements to labels +// Like instanceType.Requirements.Labels() it uses single-valued requirements +// Unlike instanceType.Requirements.Labels() it does not filter out restricted Node labels +func GetAllSingleValuedRequirementLabels(instanceType *corecloudprovider.InstanceType) map[string]string { + labels := map[string]string{} + if instanceType == nil { + return labels + } + for key, req := range instanceType.Requirements { + if req.Len() == 1 { + labels[key] = req.Values()[0] + } + } + return labels +} + +// pick the "best" SKU, priority and zone, from InstanceType options (and their offerings) in the request +func (p *Provider) pickSkuSizePriorityAndZone(ctx context.Context, nodeClaim *corev1beta1.NodeClaim, instanceTypes []*corecloudprovider.InstanceType) (*corecloudprovider.InstanceType, string, string) { + if len(instanceTypes) == 0 { + return nil, "", "" + } + // InstanceType/VM SKU - just pick the first one for now. They are presorted by cheapest offering price (taking node requirements into account) + instanceType := instanceTypes[0] + logging.FromContext(ctx).Infof("Selected instance type %s", instanceType.Name) + // Priority - Provisioner defaults to Regular, so pick Spot if it is explicitly included in requirements (and is offered in at least one zone) + priority := p.getPriorityForInstanceType(nodeClaim, instanceType) + // Zone - ideally random/spread from zones that support given Priority + priorityOfferings := lo.Filter(instanceType.Offerings.Available(), func(o corecloudprovider.Offering, _ int) bool { return o.CapacityType == priority }) + zonesWithPriority := lo.Map(priorityOfferings, func(o corecloudprovider.Offering, _ int) string { return o.Zone }) + if zone, ok := sets.New(zonesWithPriority...).PopAny(); ok { + // Zones in Offerings have - format; the zone returned from here will be used for VM instantiation, + // which expects just the zone number, without region + zone = string(zone[len(zone)-1]) + return instanceType, priority, zone + } + return nil, "", "" +} + +// getPriorityForInstanceType selects spot if both constraints are flexible and there is an available offering. +// The Azure Cloud Provider defaults to Regular, so spot must be explicitly included in capacity type requirements. +// +// This returns from a single pre-selected InstanceType, rather than all InstanceType options in nodeRequest, +// because Azure Cloud Provider does client-side selection of particular InstanceType from options +func (p *Provider) getPriorityForInstanceType(nodeClaim *corev1beta1.NodeClaim, instanceType *corecloudprovider.InstanceType) string { + requirements := scheduling.NewNodeSelectorRequirements(nodeClaim. + Spec.Requirements...) + + if requirements.Get(corev1beta1.CapacityTypeLabelKey).Has(corev1beta1.CapacityTypeSpot) { + for _, offering := range instanceType.Offerings.Available() { + if requirements.Get(v1.LabelTopologyZone).Has(offering.Zone) && offering.CapacityType == corev1beta1.CapacityTypeSpot { + return corev1beta1.CapacityTypeSpot + } + } + } + return corev1beta1.CapacityTypeOnDemand +} + +func orderInstanceTypesByPrice(instanceTypes []*corecloudprovider.InstanceType, requirements scheduling.Requirements) []*corecloudprovider.InstanceType { + // Order instance types so that we get the cheapest instance types of the available offerings + sort.Slice(instanceTypes, func(i, j int) bool { + iPrice := math.MaxFloat64 + jPrice := math.MaxFloat64 + if len(instanceTypes[i].Offerings.Available().Requirements(requirements)) > 0 { + iPrice = instanceTypes[i].Offerings.Available().Requirements(requirements).Cheapest().Price + } + if len(instanceTypes[j].Offerings.Available().Requirements(requirements)) > 0 { + jPrice = instanceTypes[j].Offerings.Available().Requirements(requirements).Cheapest().Price + } + if iPrice == jPrice { + return instanceTypes[i].Name < instanceTypes[j].Name + } + return iPrice < jPrice + }) + return instanceTypes +} + +func GetCapacityType(instance *armcompute.VirtualMachine) string { + if instance != nil && instance.Properties != nil && instance.Properties.Priority != nil { + return PriorityToCapacityType[string(*instance.Properties.Priority)] + } + return "" +} + +func GetHyperVGeneration(instance *armcompute.VirtualMachine) string { + if instance != nil && instance.Properties != nil && instance.Properties.InstanceView != nil && instance.Properties.InstanceView.HyperVGeneration != nil { + return string(*instance.Properties.InstanceView.HyperVGeneration) + } + return "" +} + +func (p *Provider) getAKSIdentifyingExtension() *armcompute.VirtualMachineExtension { + const ( + vmExtensionType = "Microsoft.Compute/virtualMachines/extensions" + aksIdentifyingExtensionName = "computeAksLinuxBilling" + aksIdentifyingExtensionPublisher = "Microsoft.AKS" + aksIdentifyingExtensionTypeLinux = "Compute.AKS.Linux.Billing" + ) + + vmExtension := &armcompute.VirtualMachineExtension{ + Location: to.Ptr(p.location), + Name: to.Ptr(aksIdentifyingExtensionName), + Properties: &armcompute.VirtualMachineExtensionProperties{ + Publisher: to.Ptr(aksIdentifyingExtensionPublisher), + TypeHandlerVersion: to.Ptr("1.0"), + AutoUpgradeMinorVersion: to.Ptr(true), + Settings: &map[string]interface{}{}, + Type: to.Ptr(aksIdentifyingExtensionTypeLinux), + }, + Type: to.Ptr(vmExtensionType), + } + + return vmExtension +} + +func GetZoneID(vm *armcompute.VirtualMachine) (string, error) { + if vm == nil { + return "", fmt.Errorf("cannot pass in a nil virtual machine") + } + if vm.Name == nil { + return "", fmt.Errorf("virtual machine is missing name") + } + if vm.Zones == nil { + return "", fmt.Errorf("virtual machine %v zones are nil", *vm.Name) + } + if len(vm.Zones) == 1 { + return *(vm.Zones)[0], nil + } + if len(vm.Zones) > 1 { + return "", fmt.Errorf("virtual machine %v has multiple zones", *vm.Name) + } + return "", fmt.Errorf("virtual machine %v does not have any zones specified", *vm.Name) +} + +func GetListQueryBuilder(rg string) *kql.Builder { + return kql.New(`Resources`). + AddLiteral(` | where type == "microsoft.compute/virtualmachines"`). + AddLiteral(` | where resourceGroup == `).AddString(strings.ToLower(rg)). // ARG VMs appear to have lowercase RG + AddLiteral(` | where tags has_cs `).AddString(NodePoolTagKey) +} + +func createVMFromQueryResponseData(data map[string]interface{}) (*armcompute.VirtualMachine, error) { + jsonString, err := json.Marshal(data) + if err != nil { + return nil, err + } + vm := armcompute.VirtualMachine{} + err = json.Unmarshal(jsonString, &vm) + if err != nil { + return nil, err + } + if vm.ID == nil { + return nil, fmt.Errorf("virtual machine is missing id") + } + if vm.Name == nil { + return nil, fmt.Errorf("virtual machine is missing name") + } + if vm.Tags == nil { + return nil, fmt.Errorf("virtual machine is missing tags") + } + // We see inconsistent casing being returned by ARG for the last segment + // of the vm.ID string. This forces it to be lowercase. + parts := strings.Split(lo.FromPtr(vm.ID), "/") + parts[len(parts)-1] = strings.ToLower(parts[len(parts)-1]) + vm.ID = lo.ToPtr(strings.Join(parts, "/")) + return &vm, nil +} + +func ConvertToVirtualMachineIdentity(nodeIdentities []string) *armcompute.VirtualMachineIdentity { + var identity *armcompute.VirtualMachineIdentity + if len(nodeIdentities) > 0 { + identityMap := make(map[string]*armcompute.UserAssignedIdentitiesValue) + for _, identityID := range nodeIdentities { + identityMap[identityID] = &armcompute.UserAssignedIdentitiesValue{} + } + + if len(identityMap) > 0 { + identity = &armcompute.VirtualMachineIdentity{ + Type: lo.ToPtr(armcompute.ResourceIdentityTypeUserAssigned), + UserAssignedIdentities: identityMap, + } + } + } + + return identity +} diff --git a/pkg/providers/instance/instance_test.go b/pkg/providers/instance/instance_test.go new file mode 100644 index 000000000..f54bdb933 --- /dev/null +++ b/pkg/providers/instance/instance_test.go @@ -0,0 +1,221 @@ +/* +Portions Copyright (c) Microsoft Corporation. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package instance + +import ( + "context" + "testing" + + "github.com/Azure/azure-sdk-for-go/sdk/azcore/to" + "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute" + "github.com/Azure/karpenter/pkg/cache" + corev1beta1 "github.com/aws/karpenter-core/pkg/apis/v1beta1" + "github.com/aws/karpenter-core/pkg/cloudprovider" + "github.com/stretchr/testify/assert" +) + +func TestGetPriorityCapacityAndInstanceType(t *testing.T) { + cases := []struct { + name string + instanceTypes []*cloudprovider.InstanceType + nodeClaim *corev1beta1.NodeClaim + expectedInstanceType string + expectedPriority string + expectedZone string + }{ + { + name: "No instance types in the list", + instanceTypes: []*cloudprovider.InstanceType{}, + nodeClaim: &corev1beta1.NodeClaim{}, + expectedInstanceType: "", + expectedPriority: "", + expectedZone: "", + }, + { + name: "Selects First, Cheapest SKU", + instanceTypes: []*cloudprovider.InstanceType{ + { + Name: "Standard_D2s_v3", + Offerings: []cloudprovider.Offering{ + { + Price: 0.1, + Zone: "westus-2", + CapacityType: corev1beta1.CapacityTypeOnDemand, + Available: true, + }, + }, + }, + { + Name: "Standard_NV16as_v4", + Offerings: []cloudprovider.Offering{ + { + Price: 0.1, + Zone: "westus-2", + CapacityType: corev1beta1.CapacityTypeOnDemand, + Available: true, + }, + }, + }, + }, + nodeClaim: &corev1beta1.NodeClaim{}, + expectedInstanceType: "Standard_D2s_v3", + expectedZone: "2", + expectedPriority: corev1beta1.CapacityTypeOnDemand, + }, + } + provider := NewProvider(context.TODO(), nil, nil, nil, nil, cache.NewUnavailableOfferings(), + "westus-2", + "MC_xxxxx_yyyy-region", + "/subscriptions/0000000-0000-0000-0000-0000000000/resourceGroups/fake-resource-group-name/providers/Microsoft.Network/virtualNetworks/karpenter/subnets/nodesubnet", + "0000000-0000-0000-0000-0000000000", + ) + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + instanceType, priority, zone := provider.pickSkuSizePriorityAndZone(context.TODO(), c.nodeClaim, c.instanceTypes) + if instanceType != nil { + assert.Equal(t, c.expectedInstanceType, instanceType.Name) + } + assert.Equal(t, c.expectedZone, zone) + assert.Equal(t, c.expectedPriority, priority) + }) + } +} + +func TestGetZone(t *testing.T) { + testVMName := "silly-armcompute" + tc := []struct { + testName string + input *armcompute.VirtualMachine + expectedZone string + expectedError string + }{ + { + testName: "missing name", + input: &armcompute.VirtualMachine{ + Name: nil, + }, + expectedError: "virtual machine is missing name", + }, + { + testName: "invalid virtual machine struct", + input: nil, + expectedError: "cannot pass in a nil virtual machine", + }, + { + testName: "invalid zones field in virtual machine struct", + input: &armcompute.VirtualMachine{ + Name: &testVMName, + }, + expectedError: "virtual machine silly-armcompute zones are nil", + }, + { + testName: "happy case", + input: &armcompute.VirtualMachine{ + Name: &testVMName, + Zones: []*string{to.Ptr("poland-central")}, + }, + expectedZone: "poland-central", + }, + { + testName: "emptyZones", + input: &armcompute.VirtualMachine{ + Name: &testVMName, + Zones: []*string{}, + }, + expectedError: "virtual machine silly-armcompute does not have any zones specified", + }, + } + + for _, c := range tc { + zone, err := GetZoneID(c.input) + assert.Equal(t, c.expectedZone, zone, c.testName) + if err != nil { + assert.Equal(t, c.expectedError, err.Error(), c.testName) + } + } +} + +// Currently tested: ID, Name, Tags, Zones +// TODO: Add the below attributes for Properties if needed: +// Priority, InstanceView.HyperVGeneration, TimeCreated +func TestCreateVMFromQueryResponseData(t *testing.T) { + id := "vm_id" + name := "vm_name" + tag := "tag1" + val := "val1" + zone := "us-west" + tags := map[string]*string{tag: &val} + zones := []*string{&zone} + + tc := []struct { + testName string + data map[string]interface{} + expectedError string + expectedVM *armcompute.VirtualMachine + }{ + { + testName: "missing id", + data: map[string]interface{}{ + "name": name, + }, + expectedError: "virtual machine is missing id", + expectedVM: nil, + }, + { + testName: "missing name", + data: map[string]interface{}{ + "id": id, + }, + expectedError: "virtual machine is missing name", + expectedVM: nil, + }, + { + testName: "happy case", + data: map[string]interface{}{ + "id": id, + "name": name, + "tags": map[string]interface{}{tag: val}, + "zones": []interface{}{zone}, + }, + expectedVM: &armcompute.VirtualMachine{ + ID: &id, + Name: &name, + Tags: tags, + Zones: zones, + }, + }, + } + + for _, c := range tc { + vm, err := createVMFromQueryResponseData(c.data) + if vm != nil { + expected := *c.expectedVM + actual := *vm + assert.Equal(t, *expected.ID, *actual.ID, c.testName) + assert.Equal(t, *expected.Name, *actual.Name, c.testName) + for key := range expected.Tags { + assert.Equal(t, *(expected.Tags[key]), *(actual.Tags[key]), c.testName) + } + for i := range expected.Zones { + assert.Equal(t, *(expected.Zones[i]), *(actual.Zones[i]), c.testName) + } + } + if err != nil { + assert.Equal(t, c.expectedError, err.Error(), c.testName) + } + } +} diff --git a/pkg/providers/instance/skuclient/skuclient.go b/pkg/providers/instance/skuclient/skuclient.go new file mode 100644 index 000000000..91f4d5b12 --- /dev/null +++ b/pkg/providers/instance/skuclient/skuclient.go @@ -0,0 +1,91 @@ +/* +Portions Copyright (c) Microsoft Corporation. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package skuclient + +import ( + "context" + "sync" + "time" + + "github.com/Azure/azure-sdk-for-go/profiles/latest/compute/mgmt/compute" + "github.com/Azure/go-autorest/autorest/azure" + "github.com/Azure/karpenter/pkg/auth" + "github.com/Azure/skewer" + klog "k8s.io/klog/v2" +) + +const ( + refreshClient = 12 * time.Hour +) + +type SkuClient interface { + GetInstance() skewer.ResourceClient +} + +type skuClient struct { + cfg *auth.Config + env *azure.Environment + + mu sync.RWMutex + instance compute.ResourceSkusClient +} + +func (sc *skuClient) updateInstance() { + sc.mu.RLock() + defer sc.mu.RUnlock() + + authorizer, err := auth.NewAuthorizer(sc.cfg, sc.env) + if err != nil { + klog.V(5).Infof("Error creating authorizer for sku client: %s", err) + return + } + + azClientConfig := sc.cfg.GetAzureClientConfig(authorizer, sc.env) + azClientConfig.UserAgent = auth.GetUserAgentExtension() + + skuClient := compute.NewResourceSkusClient(sc.cfg.SubscriptionID) + skuClient.Authorizer = azClientConfig.Authorizer + klog.V(5).Infof("Created sku client with authorizer: %v", skuClient) + + sc.instance = skuClient +} + +func NewSkuClient(ctx context.Context, cfg *auth.Config, env *azure.Environment) SkuClient { + sc := &skuClient{ + cfg: cfg, + env: env, + } + sc.updateInstance() + + go func() { + for { + select { + case <-ctx.Done(): + return + case <-time.After(refreshClient): + sc.updateInstance() + } + } + }() + return sc +} + +func (sc *skuClient) GetInstance() skewer.ResourceClient { + sc.mu.RLock() + defer sc.mu.RUnlock() + return sc.instance +} diff --git a/pkg/providers/instance/suite_test.go b/pkg/providers/instance/suite_test.go new file mode 100644 index 000000000..7fa69e47b --- /dev/null +++ b/pkg/providers/instance/suite_test.go @@ -0,0 +1,130 @@ +/* +Portions Copyright (c) Microsoft Corporation. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package instance_test + +import ( + "context" + "fmt" + "testing" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "github.com/samber/lo" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/tools/record" + + "github.com/Azure/karpenter/pkg/apis" + "github.com/Azure/karpenter/pkg/apis/settings" + "github.com/Azure/karpenter/pkg/apis/v1alpha2" + "github.com/Azure/karpenter/pkg/cloudprovider" + "github.com/Azure/karpenter/pkg/fake" + "github.com/Azure/karpenter/pkg/test" + "github.com/aws/karpenter-core/pkg/apis/v1alpha5" + "github.com/aws/karpenter-core/pkg/events" + + corev1beta1 "github.com/aws/karpenter-core/pkg/apis/v1beta1" + corecloudprovider "github.com/aws/karpenter-core/pkg/cloudprovider" + "github.com/aws/karpenter-core/pkg/operator/scheme" + + . "github.com/aws/karpenter-core/pkg/test/expectations" + . "knative.dev/pkg/logging/testing" + + coreoptions "github.com/aws/karpenter-core/pkg/operator/options" + coretest "github.com/aws/karpenter-core/pkg/test" +) + +var ctx context.Context +var stop context.CancelFunc +var env *coretest.Environment +var azureEnv *test.Environment +var cloudProvider *cloudprovider.CloudProvider + +func TestAzure(t *testing.T) { + ctx = TestContextWithLogger(t) + RegisterFailHandler(Fail) + RunSpecs(t, "Provider/Azure") +} + +var _ = BeforeSuite(func() { + ctx = coreoptions.ToContext(ctx, coretest.Options()) + // ctx = options.ToContext(ctx, test.Options()) + ctx = settings.ToContext(ctx, test.Settings()) + + env = coretest.NewEnvironment(scheme.Scheme, coretest.WithCRDs(apis.CRDs...)) + + ctx, stop = context.WithCancel(ctx) + azureEnv = test.NewEnvironment(ctx, env) + cloudProvider = cloudprovider.New(azureEnv.InstanceTypesProvider, azureEnv.InstanceProvider, events.NewRecorder(&record.FakeRecorder{}), env.Client, azureEnv.ImageProvider) +}) + +var _ = AfterSuite(func() { + stop() + Expect(env.Stop()).To(Succeed(), "Failed to stop environment") +}) + +var _ = Describe("InstanceProvider", func() { + + var nodeClass *v1alpha2.AKSNodeClass + var nodePool *corev1beta1.NodePool + var nodeClaim *corev1beta1.NodeClaim + + BeforeEach(func() { + nodeClass = test.AKSNodeClass() + nodePool = coretest.NodePool(corev1beta1.NodePool{ + Spec: corev1beta1.NodePoolSpec{ + Template: corev1beta1.NodeClaimTemplate{ + Spec: corev1beta1.NodeClaimSpec{ + NodeClassRef: &corev1beta1.NodeClassReference{ + Name: nodeClass.Name, + }, + }, + }, + }, + }) + nodeClaim = coretest.NodeClaim(corev1beta1.NodeClaim{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{ + corev1beta1.NodePoolLabelKey: nodePool.Name, + }, + }, + Spec: corev1beta1.NodeClaimSpec{ + NodeClassRef: &corev1beta1.NodeClassReference{ + Name: nodeClass.Name, + }, + }, + }) + }) + + It("should return an ICE error when all attempted instance types return an ICE error", func() { + ExpectApplied(ctx, env.Client, nodeClaim, nodePool, nodeClass) + zones := []string{"1", "2", "3"} + for _, zone := range zones { + azureEnv.UnavailableOfferingsCache.MarkUnavailable(ctx, "SubscriptionQuotaReached", "Standard_D2_v2", fmt.Sprintf("%s-%s", fake.Region, zone), v1alpha5.CapacityTypeSpot) + azureEnv.UnavailableOfferingsCache.MarkUnavailable(ctx, "SubscriptionQuotaReached", "Standard_D2_v2", fmt.Sprintf("%s-%s", fake.Region, zone), v1alpha5.CapacityTypeOnDemand) + } + instanceTypes, err := cloudProvider.GetInstanceTypes(ctx, nodePool) + Expect(err).ToNot(HaveOccurred()) + + // Filter down to a single instance type + instanceTypes = lo.Filter(instanceTypes, func(i *corecloudprovider.InstanceType, _ int) bool { return i.Name == "Standard_D2_v2" }) + + // Since all the offerings are unavailable, this should return back an ICE error + instance, err := azureEnv.InstanceProvider.Create(ctx, nodeClass, nodeClaim, instanceTypes) + Expect(corecloudprovider.IsInsufficientCapacityError(err)).To(BeTrue()) + Expect(instance).To(BeNil()) + }) +}) diff --git a/pkg/providers/instancetype/instancetype.go b/pkg/providers/instancetype/instancetype.go new file mode 100644 index 000000000..e3479b26f --- /dev/null +++ b/pkg/providers/instancetype/instancetype.go @@ -0,0 +1,367 @@ +/* +Portions Copyright (c) Microsoft Corporation. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package instancetype + +import ( + "context" + "fmt" + "math" + + "github.com/Azure/skewer" + "github.com/samber/lo" + v1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" + "knative.dev/pkg/ptr" + + "github.com/Azure/karpenter/pkg/apis/v1alpha2" + "github.com/Azure/karpenter/pkg/utils" + corev1beta1 "github.com/aws/karpenter-core/pkg/apis/v1beta1" + "github.com/aws/karpenter-core/pkg/cloudprovider" + "github.com/aws/karpenter-core/pkg/scheduling" + + "github.com/Azure/karpenter/pkg/apis/settings" + + "github.com/aws/karpenter-core/pkg/utils/resources" +) + +const ( + MemoryAvailable = "memory.available" + DefaultMemoryAvailable = "750Mi" +) + +var ( + // reservedMemoryTaxGi denotes the tax brackets for memory in Gi. + reservedMemoryTaxGi = TaxBrackets{ + { + UpperBound: 4, + Rate: .25, + }, + { + UpperBound: 8, + Rate: .20, + }, + { + UpperBound: 16, + Rate: .10, + }, + { + UpperBound: 128, + Rate: .06, + }, + { + UpperBound: math.MaxFloat64, + Rate: .02, + }, + } + + //reservedCPUTaxVCPU denotes the tax brackets for Virtual CPU cores. + reservedCPUTaxVCPU = TaxBrackets{ + { + UpperBound: 1, + Rate: .06, + }, + { + UpperBound: 2, + Rate: .04, + }, + { + UpperBound: 4, + Rate: .02, + }, + { + UpperBound: math.MaxFloat64, + Rate: .01, + }, + } +) + +// TaxBrackets implements a simple bracketed tax structure. +type TaxBrackets []struct { + // UpperBound is the largest value this bracket is applied to. + // The first bracket's lower bound is always 0. + UpperBound float64 + + // Rate is the percent rate of tax expressed as a float i.e. .5 for 50%. + Rate float64 +} + +// Calculate expects Memory in Gi and CPU in cores. +func (t TaxBrackets) Calculate(amount float64) float64 { + var tax, lower float64 + + for _, bracket := range t { + if lower > amount { + continue + } + upper := bracket.UpperBound + if upper > amount { + upper = amount + } + tax += (upper - lower) * bracket.Rate + lower = bracket.UpperBound + } + + return tax +} + +func NewInstanceType(ctx context.Context, sku *skewer.SKU, vmsize *skewer.VMSizeType, kc *corev1beta1.KubeletConfiguration, region string, + offerings cloudprovider.Offerings, nodeClass *v1alpha2.AKSNodeClass, architecture string) *cloudprovider.InstanceType { + return &cloudprovider.InstanceType{ + Name: sku.GetName(), + Requirements: computeRequirements(sku, vmsize, architecture, offerings, region), + Offerings: offerings, + Capacity: computeCapacity(ctx, sku, kc, nodeClass), + Overhead: &cloudprovider.InstanceTypeOverhead{ + KubeReserved: KubeReservedResources(lo.Must(sku.VCPU()), lo.Must(sku.Memory())), + SystemReserved: SystemReservedResources(), + EvictionThreshold: EvictionThreshold(), + }, + } +} + +func computeRequirements(sku *skewer.SKU, vmsize *skewer.VMSizeType, architecture string, + offerings cloudprovider.Offerings, region string) scheduling.Requirements { + requirements := scheduling.NewRequirements( + // Well Known Upstream + scheduling.NewRequirement(v1.LabelInstanceTypeStable, v1.NodeSelectorOpIn, sku.GetName()), + scheduling.NewRequirement(v1.LabelArchStable, v1.NodeSelectorOpIn, getArchitecture(architecture)), + scheduling.NewRequirement(v1.LabelOSStable, v1.NodeSelectorOpIn, string(v1.Linux)), + scheduling.NewRequirement( + v1.LabelTopologyZone, + v1.NodeSelectorOpIn, + lo.Map(offerings.Available(), + func(o cloudprovider.Offering, _ int) string { return o.Zone })...), + scheduling.NewRequirement(v1.LabelTopologyRegion, v1.NodeSelectorOpIn, region), + + // Well Known to Karpenter + scheduling.NewRequirement( + corev1beta1.CapacityTypeLabelKey, + v1.NodeSelectorOpIn, + lo.Map(offerings.Available(), func(o cloudprovider.Offering, _ int) string { return o.CapacityType })...), + + // Well Known to Azure + scheduling.NewRequirement(v1alpha2.LabelSKUCPU, v1.NodeSelectorOpIn, fmt.Sprint(vcpuCount(sku))), + scheduling.NewRequirement(v1alpha2.LabelSKUMemory, v1.NodeSelectorOpIn, fmt.Sprint((memoryMiB(sku)))), // in MiB + scheduling.NewRequirement(v1alpha2.LabelSKUGPUCount, v1.NodeSelectorOpIn, fmt.Sprint(gpuNvidiaCount(sku).Value())), + scheduling.NewRequirement(v1alpha2.LabelSKUGPUManufacturer, v1.NodeSelectorOpDoesNotExist), + scheduling.NewRequirement(v1alpha2.LabelSKUGPUName, v1.NodeSelectorOpDoesNotExist), + + // composites + scheduling.NewRequirement(v1alpha2.LabelSKUName, v1.NodeSelectorOpDoesNotExist), + + // size parts + scheduling.NewRequirement(v1alpha2.LabelSKUFamily, v1.NodeSelectorOpDoesNotExist), + scheduling.NewRequirement(v1alpha2.LabelSKUAccelerator, v1.NodeSelectorOpDoesNotExist), + scheduling.NewRequirement(v1alpha2.LabelSKUVersion, v1.NodeSelectorOpDoesNotExist), + + // SKU capabilities + scheduling.NewRequirement(v1alpha2.LabelSKUStorageEphemeralOSMaxSize, v1.NodeSelectorOpDoesNotExist), + scheduling.NewRequirement(v1alpha2.LabelSKUStoragePremiumCapable, v1.NodeSelectorOpDoesNotExist), + scheduling.NewRequirement(v1alpha2.LabelSKUEncryptionAtHostSupported, v1.NodeSelectorOpDoesNotExist), + scheduling.NewRequirement(v1alpha2.LabelSKUAcceleratedNetworking, v1.NodeSelectorOpDoesNotExist), + scheduling.NewRequirement(v1alpha2.LabelSKUHyperVGeneration, v1.NodeSelectorOpDoesNotExist), + // all additive feature initialized elsewhere + ) + + // composites + requirements[v1alpha2.LabelSKUName].Insert(sku.GetName()) + + // size parts + requirements[v1alpha2.LabelSKUFamily].Insert(vmsize.Family) + + // everything from additive features + for _, featureLabel := range v1alpha2.SkuFeatureToLabel { + requirements.Add(scheduling.NewRequirement(featureLabel, v1.NodeSelectorOpDoesNotExist)) + } + + setRequirementsAdditiveFeatures(requirements, vmsize) + setRequirementsStoragePremiumCapable(requirements, sku) + setRequirementsEncryptionAtHostSupported(requirements, sku) + setRequirementsEphemeralOSDiskSupported(requirements, sku, vmsize) + setRequirementsAcceleratedNetworking(requirements, sku) + setRequirementsHyperVGeneration(requirements, sku) + setRequirementsGPU(requirements, sku, vmsize) + setRequirementsAccelerator(requirements, vmsize) + setRequirementsVersion(requirements, vmsize) + + return requirements +} + +func setRequirementsAdditiveFeatures(requirements scheduling.Requirements, vmsize *skewer.VMSizeType) { + for _, feature := range vmsize.AdditiveFeatures { + if featureLabel, ok := v1alpha2.SkuFeatureToLabel[feature]; ok { + requirements[featureLabel].Insert("true") + } + } +} + +func setRequirementsStoragePremiumCapable(requirements scheduling.Requirements, sku *skewer.SKU) { + if sku.IsPremiumIO() { + requirements[v1alpha2.LabelSKUStoragePremiumCapable].Insert("true") + } +} + +func setRequirementsEncryptionAtHostSupported(requirements scheduling.Requirements, sku *skewer.SKU) { + if sku.IsEncryptionAtHostSupported() { + requirements[v1alpha2.LabelSKUEncryptionAtHostSupported].Insert("true") + } +} + +func setRequirementsEphemeralOSDiskSupported(requirements scheduling.Requirements, sku *skewer.SKU, vmsize *skewer.VMSizeType) { + if sku.IsEphemeralOSDiskSupported() && vmsize.Series != "Dlds_v5" { // Dlds_v5 does not support ephemeral OS disk, contrary to what it claims + requirements[v1alpha2.LabelSKUStorageEphemeralOSMaxSize].Insert(fmt.Sprint(MaxEphemeralOSDiskSizeGB(sku))) + } +} + +func setRequirementsAcceleratedNetworking(requirements scheduling.Requirements, sku *skewer.SKU) { + if sku.IsAcceleratedNetworkingSupported() { + requirements[v1alpha2.LabelSKUAcceleratedNetworking].Insert("true") + } +} + +func setRequirementsHyperVGeneration(requirements scheduling.Requirements, sku *skewer.SKU) { + if sku.IsHyperVGen1Supported() { + requirements[v1alpha2.LabelSKUHyperVGeneration].Insert(v1alpha2.HyperVGenerationV1) + } + if sku.IsHyperVGen2Supported() { + requirements[v1alpha2.LabelSKUHyperVGeneration].Insert(v1alpha2.HyperVGenerationV2) + } +} + +func setRequirementsGPU(requirements scheduling.Requirements, sku *skewer.SKU, vmsize *skewer.VMSizeType) { + if utils.IsNvidiaEnabledSKU(sku.GetName()) { + requirements[v1alpha2.LabelSKUGPUManufacturer].Insert(v1alpha2.ManufacturerNvidia) + if vmsize.AcceleratorType != nil { + requirements[v1alpha2.LabelSKUGPUName].Insert(*vmsize.AcceleratorType) + } + } +} + +func setRequirementsAccelerator(requirements scheduling.Requirements, vmsize *skewer.VMSizeType) { + if vmsize.AcceleratorType != nil { + requirements[v1alpha2.LabelSKUAccelerator].Insert(*vmsize.AcceleratorType) + } +} + +// setRequirementsVersion sets the SKU version label, dropping "v" prefix and backfilling "1" +func setRequirementsVersion(requirements scheduling.Requirements, vmsize *skewer.VMSizeType) { + version := "1" + if vmsize.Version != "" { + if !(vmsize.Version[0] == 'V' || vmsize.Version[0] == 'v') { + // should never happen; don't capture in label (won't be available for selection by version) + return + } + version = vmsize.Version[1:] + } + requirements[v1alpha2.LabelSKUVersion].Insert(version) +} + +func getArchitecture(architecture string) string { + if value, ok := v1alpha2.AzureToKubeArchitectures[architecture]; ok { + return value + } + return architecture // unrecognized +} + +func computeCapacity(ctx context.Context, sku *skewer.SKU, kc *corev1beta1.KubeletConfiguration, nodeClass *v1alpha2.AKSNodeClass) v1.ResourceList { + return v1.ResourceList{ + v1.ResourceCPU: *cpu(sku), + v1.ResourceMemory: *memory(ctx, sku), + v1.ResourceEphemeralStorage: *ephemeralStorage(nodeClass), + v1.ResourcePods: *pods(sku, kc), + v1.ResourceName("nvidia.com/gpu"): *gpuNvidiaCount(sku), + } +} + +// gpuNvidiaCount returns the number of Nvidia GPUs in the SKU. Currently nvidia is the only gpu manufacturer we support. +func gpuNvidiaCount(sku *skewer.SKU) *resource.Quantity { + count, err := sku.GPU() + if err != nil || !utils.IsNvidiaEnabledSKU(sku.GetName()) { + count = 0 + } + return resources.Quantity(fmt.Sprint(count)) +} + +func vcpuCount(sku *skewer.SKU) int64 { + return lo.Must(sku.VCPU()) +} + +func cpu(sku *skewer.SKU) *resource.Quantity { + return resources.Quantity(fmt.Sprint(vcpuCount(sku))) +} + +func memoryGiB(sku *skewer.SKU) float64 { + return lo.Must(sku.Memory()) // contrary to "MemoryGB" capability name, it is in GiB (!) +} + +func memoryMiB(sku *skewer.SKU) int64 { + return int64(memoryGiB(sku) * 1024) +} + +func memory(ctx context.Context, sku *skewer.SKU) *resource.Quantity { + memory := resources.Quantity(fmt.Sprintf("%dGi", int64(memoryGiB(sku)))) + // Account for VM overhead in calculation + memory.Sub(resource.MustParse(fmt.Sprintf("%dMi", int64(math.Ceil( + float64(memory.Value())*settings.FromContext(ctx).VMMemoryOverheadPercent/1024/1024))))) + return memory +} + +func ephemeralStorage(nodeClass *v1alpha2.AKSNodeClass) *resource.Quantity { + return resource.NewScaledQuantity(int64(*nodeClass.Spec.OSDiskSizeGB), resource.Giga) +} + +func pods(sku *skewer.SKU, kc *corev1beta1.KubeletConfiguration) *resource.Quantity { + // TODO: fine-tune pods calc + var count int64 + switch { + case kc != nil && kc.MaxPods != nil: + count = int64(ptr.Int32Value(kc.MaxPods)) + default: + count = 110 + } + if kc != nil && ptr.Int32Value(kc.PodsPerCore) > 0 { + count = lo.Min([]int64{int64(ptr.Int32Value(kc.PodsPerCore)) * cpu(sku).Value(), count}) + } + return resources.Quantity(fmt.Sprint(count)) +} + +func SystemReservedResources() v1.ResourceList { + // AKS does not set system-reserved values and only CPU and memory are considered + // https://learn.microsoft.com/en-us/azure/aks/concepts-clusters-workloads#resource-reservations + return v1.ResourceList{ + v1.ResourceCPU: resource.Quantity{}, + v1.ResourceMemory: resource.Quantity{}, + } +} + +func KubeReservedResources(vcpus int64, memoryGib float64) v1.ResourceList { + reservedMemoryMi := int64(1024 * reservedMemoryTaxGi.Calculate(memoryGib)) + reservedCPUMilli := int64(1000 * reservedCPUTaxVCPU.Calculate(float64(vcpus))) + + resources := v1.ResourceList{ + v1.ResourceCPU: *resource.NewScaledQuantity(reservedCPUMilli, resource.Milli), + v1.ResourceMemory: *resource.NewQuantity(reservedMemoryMi*1024*1024, resource.BinarySI), + } + + return resources +} + +func EvictionThreshold() v1.ResourceList { + return v1.ResourceList{ + v1.ResourceMemory: resource.MustParse(DefaultMemoryAvailable), + } +} diff --git a/pkg/providers/instancetype/instancetypes.go b/pkg/providers/instancetype/instancetypes.go new file mode 100644 index 000000000..6ae0b9644 --- /dev/null +++ b/pkg/providers/instancetype/instancetypes.go @@ -0,0 +1,219 @@ +/* +Portions Copyright (c) Microsoft Corporation. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package instancetype + +import ( + "context" + "fmt" + "math" + "net/http" + "sync" + "time" + + "github.com/samber/lo" + + "github.com/Azure/karpenter/pkg/apis/v1alpha2" + kcache "github.com/Azure/karpenter/pkg/cache" + "github.com/Azure/karpenter/pkg/utils" + "github.com/patrickmn/go-cache" + "k8s.io/apimachinery/pkg/util/sets" + "knative.dev/pkg/logging" + + corev1beta1 "github.com/aws/karpenter-core/pkg/apis/v1beta1" + "github.com/aws/karpenter-core/pkg/cloudprovider" + + "github.com/Azure/karpenter/pkg/providers/instance/skuclient" + "github.com/Azure/karpenter/pkg/providers/pricing" + "github.com/Azure/skewer" + "github.com/alecthomas/units" +) + +const ( + InstanceTypesCacheKey = "types" + InstanceTypesCacheTTL = 23 * time.Hour +) + +type Provider struct { + sync.Mutex + region string + skuClient skuclient.SkuClient + pricingProvider *pricing.Provider + unavailableOfferings *kcache.UnavailableOfferings + // Has one cache entry for all the instance types (key: InstanceTypesCacheKey) + cache *cache.Cache +} + +func NewProvider(region string, cache *cache.Cache, skuClient skuclient.SkuClient, pricingProvider *pricing.Provider, offeringsCache *kcache.UnavailableOfferings) *Provider { + return &Provider{ + // TODO: skewer api, subnetprovider, pricing provider, unavailable offerings, ... + region: region, + skuClient: skuClient, + pricingProvider: pricingProvider, + unavailableOfferings: offeringsCache, + cache: cache, + } +} + +// Get all instance type options +func (p *Provider) List( + ctx context.Context, kc *corev1beta1.KubeletConfiguration, nodeClass *v1alpha2.AKSNodeClass) ([]*cloudprovider.InstanceType, error) { + p.Lock() + defer p.Unlock() + // Get SKUs from Azure + skus, err := p.getInstanceTypes(ctx) + if err != nil { + return nil, err + } + + // Get Viable offerings + /// Azure has zones availability directly from SKU info + var result []*cloudprovider.InstanceType + for _, sku := range skus { + vmsize, err := sku.GetVMSize() + if err != nil { + logging.FromContext(ctx).Errorf("parsing VM size %s, %v", *sku.Size, err) + continue + } + if !p.isSupportedSize(vmsize) { + continue + } + architecture, err := sku.GetCPUArchitectureType() + if err != nil { + logging.FromContext(ctx).Errorf("parsing SKU architecture %s, %v", *sku.Size, err) + continue + } + instanceTypeZones := instanceTypeZones(sku, p.region) + instanceType := NewInstanceType(ctx, sku, vmsize, kc, p.region, p.createOfferings(sku, instanceTypeZones), nodeClass, architecture) + if len(instanceType.Offerings) == 0 { + continue + } + result = append(result, instanceType) + } + return result, nil +} + +func (p *Provider) LivenessProbe(req *http.Request) error { + p.Lock() + //nolint: staticcheck + p.Unlock() + return p.pricingProvider.LivenessProbe(req) +} + +// instanceTypeZones generates the set of all supported zones for a given SKU +// The strings have to match Zone labels that will be placed on Node +func instanceTypeZones(sku *skewer.SKU, region string) sets.Set[string] { + // skewer returns numerical zones, like "1" (as keys in the map); + // prefix each zone with "-", to have them match the labels placed on Node (e.g. "westus2-1") + // Note this data comes from LocationInfo, then skewer is used to get the SKU info + return sets.New(lo.Map(lo.Keys(sku.AvailabilityZones(region)), func(zone string, _ int) string { + return fmt.Sprintf("%s-%s", region, zone) + })...) +} + +func (p *Provider) createOfferings(sku *skewer.SKU, zones sets.Set[string]) []cloudprovider.Offering { + offerings := []cloudprovider.Offering{} + for zone := range zones { + onDemandPrice, onDemandOk := p.pricingProvider.OnDemandPrice(*sku.Name) + spotPrice, spotOk := p.pricingProvider.SpotPrice(*sku.Name) + availableOnDemand := onDemandOk && !p.unavailableOfferings.IsUnavailable(*sku.Name, zone, corev1beta1.CapacityTypeOnDemand) + availableSpot := spotOk && !p.unavailableOfferings.IsUnavailable(*sku.Name, zone, corev1beta1.CapacityTypeSpot) + offerings = append(offerings, cloudprovider.Offering{Zone: zone, CapacityType: corev1beta1.CapacityTypeSpot, Price: spotPrice, Available: availableSpot}) + offerings = append(offerings, cloudprovider.Offering{Zone: zone, CapacityType: corev1beta1.CapacityTypeOnDemand, Price: onDemandPrice, Available: availableOnDemand}) + } + return offerings +} + +// getInstanceTypes retrieves all instance types from skewer using some opinionated filters +func (p *Provider) getInstanceTypes(ctx context.Context) (map[string]*skewer.SKU, error) { + if cached, ok := p.cache.Get(InstanceTypesCacheKey); ok { + return cached.(map[string]*skewer.SKU), nil + } + instanceTypes := map[string]*skewer.SKU{} + + cache, err := skewer.NewCache(ctx, skewer.WithLocation(p.region), skewer.WithResourceClient(p.skuClient.GetInstance())) + if err != nil { + return nil, fmt.Errorf("fetching SKUs using skewer, %w", err) + } + + skus := cache.List(ctx, skewer.ResourceTypeFilter(skewer.VirtualMachines)) + logging.FromContext(ctx).Debugf("Discovered %d SKUs", len(skus)) + for i := range skus { + if p.isSupported(&skus[i]) { + instanceTypes[skus[i].GetName()] = &skus[i] + } + } + + logging.FromContext(ctx).Debugf("%d SKUs remaining after filtering", len(instanceTypes)) + p.cache.SetDefault(InstanceTypesCacheKey, instanceTypes) + return instanceTypes, nil +} + +// isSupported indicates SKU is supported by AKS, based on SKU properties +func (p *Provider) isSupported(sku *skewer.SKU) bool { + name := lo.FromPtr(sku.Name) + + // less than 2 cpus + if cpu, err := sku.VCPU(); err == nil && cpu < 2 { + return false + } + + // less then 3.5 GiB of memory + if memGiB, err := sku.Memory(); err == nil && memGiB < 3.5 { + return false + } + + // filter out instances AKS does not support + if RestrictedVMSizes.Has(sku.GetName()) { + return false + } + + // filter out GPU SKUs AKS does not support + if gpu, err := sku.GPU(); err == nil && gpu > 0 && !(utils.IsNvidiaEnabledSKU(name) || utils.IsMarinerEnabledGPUSKU(name)) { + return false + } + + return true +} + +// isSupportedSize indicates VM size is supported by AKS, based on additional SKU properties +func (p *Provider) isSupportedSize(size *skewer.VMSizeType) bool { + // any SKU with constrained CPUs + return size.CpusConstrained == nil +} + +// MaxEphemeralOSDiskSizeGB returns the maximum ephemeral OS disk size for a given SKU. +// Ephemeral OS disk size is determined by the larger of the two values: +// 1. MaxResourceVolumeMB (Temp Disk Space) +// 2. MaxCachedDiskBytes (Cached Disk Space) +// For Ephemeral disk creation, CRP will use the larger of the two values to ensure we have enough space for the ephemeral disk. +// Note that generally only older SKUs use the Temp Disk space for ephemeral disks, and newer SKUs use the Cached Disk in most cases. +// The ephemeral OS disk is created with the free space of the larger of the two values in that place. +func MaxEphemeralOSDiskSizeGB(sku *skewer.SKU) float64 { + if sku == nil { + return 0 + } + maxCachedDiskBytes, _ := sku.MaxCachedDiskBytes() + maxResourceVolumeMB, _ := sku.MaxResourceVolumeMB() // NOTE: this is a misnomer, MB is actually MiB, hence the conversion below + + maxResourceVolumeBytes := maxResourceVolumeMB * int64(units.Mebibyte) + maxDiskBytes := math.Max(float64(maxCachedDiskBytes), float64(maxResourceVolumeBytes)) + if maxDiskBytes == 0 { + return 0 + } + // convert bytes to GB + return maxDiskBytes / float64(units.Gigabyte) +} diff --git a/pkg/providers/instancetype/restricted.go b/pkg/providers/instancetype/restricted.go new file mode 100644 index 000000000..5d22a75b6 --- /dev/null +++ b/pkg/providers/instancetype/restricted.go @@ -0,0 +1,36 @@ +/* +Portions Copyright (c) Microsoft Corporation. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package instancetype + +import "k8s.io/apimachinery/pkg/util/sets" + +var ( + RestrictedVMSizes = sets.New( + "Standard_A0", + "Standard_A1", + "Standard_A1_v2", + "Standard_B1s", + "Standard_B1ms", + "Standard_F1", + "Standard_F1s", + "Basic_A0", + "Basic_A1", + "Basic_A2", + "Basic_A3", + "Basic_A4", + ) +) diff --git a/pkg/providers/instancetype/suite_test.go b/pkg/providers/instancetype/suite_test.go new file mode 100644 index 000000000..e43bc1464 --- /dev/null +++ b/pkg/providers/instancetype/suite_test.go @@ -0,0 +1,889 @@ +/* +Portions Copyright (c) Microsoft Corporation. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package instancetype_test + +import ( + "context" + "encoding/base64" + "fmt" + "os" + "strings" + "testing" + "time" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "github.com/samber/lo" + v1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/sets" + "k8s.io/client-go/tools/record" + clock "k8s.io/utils/clock/testing" + . "knative.dev/pkg/logging/testing" + + coreoptions "github.com/aws/karpenter-core/pkg/operator/options" + + "github.com/aws/karpenter-core/pkg/apis/v1alpha5" + corev1beta1 "github.com/aws/karpenter-core/pkg/apis/v1beta1" + corecloudprovider "github.com/aws/karpenter-core/pkg/cloudprovider" + "github.com/aws/karpenter-core/pkg/controllers/provisioning" + "github.com/aws/karpenter-core/pkg/controllers/state" + "github.com/aws/karpenter-core/pkg/events" + "github.com/aws/karpenter-core/pkg/operator/scheme" + coretest "github.com/aws/karpenter-core/pkg/test" + . "github.com/aws/karpenter-core/pkg/test/expectations" + + "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute" + "github.com/Azure/karpenter/pkg/apis" + "github.com/Azure/karpenter/pkg/apis/settings" + "github.com/Azure/karpenter/pkg/apis/v1alpha2" + "github.com/Azure/karpenter/pkg/cloudprovider" + "github.com/Azure/karpenter/pkg/fake" + "github.com/Azure/karpenter/pkg/providers/instancetype" + "github.com/Azure/karpenter/pkg/providers/loadbalancer" + "github.com/Azure/karpenter/pkg/test" + "github.com/Azure/karpenter/pkg/utils" +) + +var ctx context.Context +var stop context.CancelFunc +var env *coretest.Environment +var azureEnv *test.Environment +var fakeClock *clock.FakeClock +var coreProvisioner *provisioning.Provisioner +var cluster *state.Cluster +var cloudProvider *cloudprovider.CloudProvider + +func TestAzure(t *testing.T) { + ctx = TestContextWithLogger(t) + RegisterFailHandler(Fail) + RunSpecs(t, "Provider/Azure") +} + +var _ = BeforeSuite(func() { + ctx = coreoptions.ToContext(ctx, coretest.Options()) + // ctx = options.ToContext(ctx, test.Options()) + ctx = settings.ToContext(ctx, test.Settings()) + + env = coretest.NewEnvironment(scheme.Scheme, coretest.WithCRDs(apis.CRDs...)) + + ctx, stop = context.WithCancel(ctx) + azureEnv = test.NewEnvironment(ctx, env) + + fakeClock = &clock.FakeClock{} + cloudProvider = cloudprovider.New(azureEnv.InstanceTypesProvider, azureEnv.InstanceProvider, events.NewRecorder(&record.FakeRecorder{}), env.Client, azureEnv.ImageProvider) + cluster = state.NewCluster(fakeClock, env.Client, cloudProvider) + coreProvisioner = provisioning.NewProvisioner(env.Client, env.KubernetesInterface.CoreV1(), events.NewRecorder(&record.FakeRecorder{}), cloudProvider, cluster) +}) + +var _ = AfterSuite(func() { + stop() + Expect(env.Stop()).To(Succeed(), "Failed to stop environment") +}) + +var _ = Describe("InstanceType Provider", func() { + + var nodeClass *v1alpha2.AKSNodeClass + var nodePool *corev1beta1.NodePool + + BeforeEach(func() { + os.Setenv("AZURE_VNET_GUID", "test-vnet-guid") + os.Setenv("AZURE_VNET_NAME", "aks-vnet-00000000") + os.Setenv("AZURE_SUBNET_NAME", "test-subnet-name") + + nodeClass = test.AKSNodeClass() + // Sometimes we use nodeClass without applying it, when simulating the List() call. + // In that case, we need to set the default values for the node class. + nodeClass.Spec.OSDiskSizeGB = lo.ToPtr[int32](128) + nodePool = coretest.NodePool(corev1beta1.NodePool{ + Spec: corev1beta1.NodePoolSpec{ + Template: corev1beta1.NodeClaimTemplate{ + Spec: corev1beta1.NodeClaimSpec{ + NodeClassRef: &corev1beta1.NodeClassReference{ + Name: nodeClass.Name, + }, + }, + }, + }, + }) + + cluster.Reset() + azureEnv.Reset() + }) + + AfterEach(func() { + ExpectCleanedUp(ctx, env.Client) + }) + + Context("Filtering in InstanceType Provider List", func() { + It("should filter out skus that are explicitly marked as restricted", func() { + instanceTypes, err := azureEnv.InstanceTypesProvider.List(ctx, &corev1beta1.KubeletConfiguration{}, nodeClass) + Expect(err).NotTo(HaveOccurred()) + for _, instanceType := range instanceTypes { + // We should not see any instance types in the restricted list + Expect(instancetype.RestrictedVMSizes.Has(instanceType.Name)).To(BeFalse()) + } + }) + }) + + Context("Ephemeral Disk", func() { + It("should use ephemeral disk if supported, and has space of at least 128GB by default", func() { + // Create a Provisioner that selects a sku that supports ephemeral + // SKU Standard_D64s_v3 has 1600GB of CacheDisk space, so we expect we can create an ephemeral disk with size 128GB + + np := coretest.NodePool() + np.Spec.Template.Spec.Requirements = append(np.Spec.Template.Spec.Requirements, v1.NodeSelectorRequirement{ + Key: "node.kubernetes.io/instance-type", + Operator: v1.NodeSelectorOpIn, + Values: []string{"Standard_D64s_v3"}, + }) + np.Spec.Template.Spec.NodeClassRef = &corev1beta1.NodeClassReference{ + Name: nodeClass.Name, + } + + ExpectApplied(ctx, env.Client, np, nodeClass) + pod := coretest.UnschedulablePod() + ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, coreProvisioner, pod) + ExpectScheduled(ctx, env.Client, pod) + + vm := azureEnv.VirtualMachinesAPI.VirtualMachineCreateOrUpdateBehavior.CalledWithInput.Pop().VM + Expect(vm).NotTo(BeNil()) + Expect(vm.Properties.StorageProfile.OSDisk.DiskSizeGB).NotTo(BeNil()) + Expect(*vm.Properties.StorageProfile.OSDisk.DiskSizeGB).To(Equal(int32(128))) + // should have local disk attached + Expect(vm.Properties.StorageProfile.OSDisk.DiffDiskSettings).NotTo(BeNil()) + Expect(lo.FromPtr(vm.Properties.StorageProfile.OSDisk.DiffDiskSettings.Option)).To(Equal(armcompute.DiffDiskOptionsLocal)) + }) + + It("should use ephemeral disk if supported, and set disk size to OSDiskSizeGB from node class", func() { + // Create a Provisioner that selects a sku that supports ephemeral + // SKU Standard_D64s_v3 has 1600GB of CacheDisk space, so we expect we can create an ephemeral disk with size 256GB + provider := test.AKSNodeClass() + provider.Spec.OSDiskSizeGB = lo.ToPtr[int32](256) + np := coretest.NodePool() + np.Spec.Template.Spec.Requirements = append(np.Spec.Template.Spec.Requirements, v1.NodeSelectorRequirement{ + Key: "node.kubernetes.io/instance-type", + Operator: v1.NodeSelectorOpIn, + Values: []string{"Standard_D64s_v3"}, + }) + np.Spec.Template.Spec.NodeClassRef = &corev1beta1.NodeClassReference{ + Name: provider.Name, + } + + ExpectApplied(ctx, env.Client, np, provider) + pod := coretest.UnschedulablePod() + ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, coreProvisioner, pod) + ExpectScheduled(ctx, env.Client, pod) + + vm := azureEnv.VirtualMachinesAPI.VirtualMachineCreateOrUpdateBehavior.CalledWithInput.Pop().VM + Expect(vm).NotTo(BeNil()) + Expect(vm.Properties.StorageProfile.OSDisk.DiskSizeGB).NotTo(BeNil()) + Expect(*vm.Properties.StorageProfile.OSDisk.DiskSizeGB).To(Equal(int32(256))) + Expect(lo.FromPtr(vm.Properties.StorageProfile.OSDisk.DiffDiskSettings.Option)).To(Equal(armcompute.DiffDiskOptionsLocal)) + }) + It("if ephemeral is supported, but we don't have enough space, we should not use ephemeral disk", func() { + // Create a Provisioner that selects a sku that supports ephemeral Standard_D2s_v3 + // Standard_D2s_V3 has 53GB Of CacheDisk space, + // and has 16GB of Temp Disk Space. + // With our rule of 100GB being the minimum OSDiskSize, this VM should be created without local disk + np := coretest.NodePool() + np.Spec.Template.Spec.Requirements = append(np.Spec.Template.Spec.Requirements, v1.NodeSelectorRequirement{ + Key: "node.kubernetes.io/instance-type", + Operator: v1.NodeSelectorOpIn, + Values: []string{"Standard_D2s_v3"}, + }) + np.Spec.Template.Spec.NodeClassRef = &corev1beta1.NodeClassReference{ + Name: nodeClass.Name, + } + + ExpectApplied(ctx, env.Client, nodePool, nodeClass) + pod := coretest.UnschedulablePod() + ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, coreProvisioner, pod) + ExpectScheduled(ctx, env.Client, pod) + vm := azureEnv.VirtualMachinesAPI.VirtualMachineCreateOrUpdateBehavior.CalledWithInput.Pop().VM + Expect(vm).NotTo(BeNil()) + Expect(vm.Properties.StorageProfile.OSDisk.DiskSizeGB).NotTo(BeNil()) + Expect(*vm.Properties.StorageProfile.OSDisk.DiskSizeGB).To(Equal(int32(128))) + Expect(vm.Properties.StorageProfile.OSDisk.DiffDiskSettings).To(BeNil()) + }) + }) + + Context("Provisioner with KubeletConfig", func() { + kubeletConfig := &corev1beta1.KubeletConfiguration{ + PodsPerCore: lo.ToPtr(int32(110)), + EvictionSoft: map[string]string{ + instancetype.MemoryAvailable: "1Gi", + }, + EvictionSoftGracePeriod: map[string]metav1.Duration{ + instancetype.MemoryAvailable: {Duration: 10 * time.Second}, + }, + EvictionMaxPodGracePeriod: lo.ToPtr(int32(15)), + ImageGCHighThresholdPercent: lo.ToPtr(int32(30)), + ImageGCLowThresholdPercent: lo.ToPtr(int32(20)), + CPUCFSQuota: lo.ToPtr(true), + } + + It("should support provisioning with kubeletConfig, computeResources & maxPods not specified", func() { + + nodePool.Spec.Template.Spec.Kubelet = kubeletConfig + ExpectApplied(ctx, env.Client, nodePool, nodeClass) + + pod := coretest.UnschedulablePod() + ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, coreProvisioner, pod) + ExpectScheduled(ctx, env.Client, pod) + + Expect(azureEnv.VirtualMachinesAPI.VirtualMachineCreateOrUpdateBehavior.CalledWithInput.Len()).To(Equal(1)) + vm := azureEnv.VirtualMachinesAPI.VirtualMachineCreateOrUpdateBehavior.CalledWithInput.Pop().VM + customData := *vm.Properties.OSProfile.CustomData + Expect(customData).ToNot(BeNil()) + decodedBytes, err := base64.StdEncoding.DecodeString(customData) + Expect(err).To(Succeed()) + decodedString := string(decodedBytes[:]) + kubeletFlags := decodedString[strings.Index(decodedString, "KUBELET_FLAGS=")+len("KUBELET_FLAGS="):] + + Expect(kubeletFlags).To(SatisfyAny( + ContainSubstring("--system-reserved=cpu=0,memory=0"), + ContainSubstring("--system-reserved=memory=0,cpu=0"), + )) + Expect(kubeletFlags).To(SatisfyAny( + ContainSubstring("--kube-reserved=cpu=100m,memory=1843Mi"), + ContainSubstring("--kube-reserved=memory=1843Mi,cpu=100m"), + )) + + Expect(kubeletFlags).To(ContainSubstring("--eviction-hard=memory.available<750Mi")) + Expect(kubeletFlags).To(ContainSubstring("--eviction-soft=memory.available<1Gi")) + Expect(kubeletFlags).To(ContainSubstring("--eviction-soft-grace-period=memory.available=10s")) + Expect(kubeletFlags).To(ContainSubstring("--max-pods=100")) // kubenet + Expect(kubeletFlags).To(ContainSubstring("--pods-per-core=110")) + Expect(kubeletFlags).To(ContainSubstring("--image-gc-low-threshold=20")) + Expect(kubeletFlags).To(ContainSubstring("--image-gc-high-threshold=30")) + Expect(kubeletFlags).To(ContainSubstring("--cpu-cfs-quota=true")) + }) + + It("should support provisioning with kubeletConfig, computeResources and maxPods specified", func() { + kubeletConfig.SystemReserved = v1.ResourceList{ + v1.ResourceCPU: resource.MustParse("200m"), + v1.ResourceMemory: resource.MustParse("1Gi"), + } + kubeletConfig.KubeReserved = v1.ResourceList{ + v1.ResourceCPU: resource.MustParse("100m"), + v1.ResourceMemory: resource.MustParse("500Mi"), + } + kubeletConfig.EvictionHard = map[string]string{ + instancetype.MemoryAvailable: "10Mi", + } + kubeletConfig.MaxPods = lo.ToPtr(int32(15)) + + nodePool.Spec.Template.Spec.Kubelet = kubeletConfig + + ExpectApplied(ctx, env.Client, nodePool, nodeClass) + pod := coretest.UnschedulablePod() + ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, coreProvisioner, pod) + ExpectScheduled(ctx, env.Client, pod) + + Expect(azureEnv.VirtualMachinesAPI.VirtualMachineCreateOrUpdateBehavior.CalledWithInput.Len()).To(Equal(1)) + vm := azureEnv.VirtualMachinesAPI.VirtualMachineCreateOrUpdateBehavior.CalledWithInput.Pop().VM + customData := *vm.Properties.OSProfile.CustomData + Expect(customData).ToNot(BeNil()) + decodedBytes, err := base64.StdEncoding.DecodeString(customData) + Expect(err).To(Succeed()) + decodedString := string(decodedBytes[:]) + kubeletFlags := decodedString[strings.Index(decodedString, "KUBELET_FLAGS=")+len("KUBELET_FLAGS="):] + + Expect(kubeletFlags).To(SatisfyAny( + ContainSubstring("--system-reserved=cpu=0,memory=0"), + ContainSubstring("--system-reserved=memory=0,cpu=0"), + )) + Expect(kubeletFlags).To(SatisfyAny( + ContainSubstring("--kube-reserved=cpu=100m,memory=1843Mi"), + ContainSubstring("--kube-reserved=memory=1843Mi,cpu=100m"), + )) + + Expect(kubeletFlags).To(ContainSubstring("--eviction-hard=memory.available<750Mi")) + Expect(kubeletFlags).To(ContainSubstring("--eviction-soft=memory.available<1Gi")) + Expect(kubeletFlags).To(ContainSubstring("--eviction-soft-grace-period=memory.available=10s")) + Expect(kubeletFlags).To(ContainSubstring("--max-pods=100")) // kubenet + Expect(kubeletFlags).To(ContainSubstring("--pods-per-core=110")) + Expect(kubeletFlags).To(ContainSubstring("--image-gc-low-threshold=20")) + Expect(kubeletFlags).To(ContainSubstring("--image-gc-high-threshold=30")) + Expect(kubeletFlags).To(ContainSubstring("--cpu-cfs-quota=true")) + }) + }) + + Context("Provisioner with VnetNodeLabel", func() { + It("should support provisioning with Vnet node labels", func() { + ExpectApplied(ctx, env.Client, nodePool, nodeClass) + pod := coretest.UnschedulablePod() + ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, coreProvisioner, pod) + ExpectScheduled(ctx, env.Client, pod) + + Expect(azureEnv.VirtualMachinesAPI.VirtualMachineCreateOrUpdateBehavior.CalledWithInput.Len()).To(Equal(1)) + vm := azureEnv.VirtualMachinesAPI.VirtualMachineCreateOrUpdateBehavior.CalledWithInput.Pop().VM + customData := *vm.Properties.OSProfile.CustomData + Expect(customData).ToNot(BeNil()) + decodedBytes, err := base64.StdEncoding.DecodeString(customData) + Expect(err).To(Succeed()) + decodedString := string(decodedBytes[:]) + Expect(decodedString).To(SatisfyAll( + ContainSubstring("kubernetes.azure.com/ebpf-dataplane=cilium"), + ContainSubstring("kubernetes.azure.com/network-name=aks-vnet-00000000"), + ContainSubstring("kubernetes.azure.com/network-subnet=test-subnet-name"), + ContainSubstring("kubernetes.azure.com/network-subscription=test-subscription"), + ContainSubstring("kubernetes.azure.com/nodenetwork-vnetguid=test-vnet-guid"), + ContainSubstring("kubernetes.azure.com/podnetwork-type=overlay"), + )) + }) + }) + + Context("Unavailable Offerings", func() { + It("should not allocate a vm in a zone marked as unavailable", func() { + azureEnv.UnavailableOfferingsCache.MarkUnavailable(ctx, "ZonalAllocationFailure", "Standard_D2_v2", fmt.Sprintf("%s-1", fake.Region), corev1beta1.CapacityTypeSpot) + azureEnv.UnavailableOfferingsCache.MarkUnavailable(ctx, "ZonalAllocationFailure", "Standard_D2_v2", fmt.Sprintf("%s-1", fake.Region), corev1beta1.CapacityTypeOnDemand) + coretest.ReplaceRequirements(nodePool, v1.NodeSelectorRequirement{ + Key: v1.LabelInstanceTypeStable, + Operator: v1.NodeSelectorOpIn, + Values: []string{"Standard_D2_v2"}, + }, + ) + + ExpectApplied(ctx, env.Client, nodePool, nodeClass) + // Try this 100 times to make sure we don't get a node in eastus-1, + // we pick from 3 zones so the likelihood of this test passing by chance is 1/3^100 + for i := 0; i < 100; i++ { + pod := coretest.UnschedulablePod() + ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, coreProvisioner, pod) + ExpectScheduled(ctx, env.Client, pod) + nodes := &v1.NodeList{} + Expect(env.Client.List(ctx, nodes)).To(Succeed()) + for _, node := range nodes.Items { + Expect(node.Labels["karpenter.k8s.azure/zone"]).ToNot(Equal(fmt.Sprintf("%s-1", fake.Region))) + Expect(node.Labels["node.kubernetes.io/instance-type"]).To(Equal("Standard_D2_v2")) + + } + } + + }) + + It("Should not return unavailable offerings", func() { + zones := []string{"1", "2", "3"} + for _, zone := range zones { + + azureEnv.UnavailableOfferingsCache.MarkUnavailable(ctx, "SubscriptionQuotaReached", "Standard_D2_v2", fmt.Sprintf("%s-%s", fake.Region, zone), corev1beta1.CapacityTypeSpot) + azureEnv.UnavailableOfferingsCache.MarkUnavailable(ctx, "SubscriptionQuotaReached", "Standard_D2_v2", fmt.Sprintf("%s-%s", fake.Region, zone), corev1beta1.CapacityTypeOnDemand) + } + instanceTypes, err := azureEnv.InstanceTypesProvider.List(ctx, &corev1beta1.KubeletConfiguration{}, nodeClass) + Expect(err).ToNot(HaveOccurred()) + + seeUnavailable := false + for _, instanceType := range instanceTypes { + if instanceType.Name == "Standard_D2_v2" { + // We want to validate we see the offering in the list, + // but we also expect it to not have any available offerings + seeUnavailable = true + Expect(len(instanceType.Offerings.Available())).To(Equal(0)) + } else { + Expect(len(instanceType.Offerings.Available())).To(Not(Equal(0))) + } + } + // we should see the unavailable offering in the list + Expect(seeUnavailable).To(BeTrue()) + }) + + It("should launch instances in a different zone than preferred", func() { + azureEnv.UnavailableOfferingsCache.MarkUnavailable(ctx, "ZonalAllocationFailure", "Standard_D2_v2", fmt.Sprintf("%s-1", fake.Region), v1alpha5.CapacityTypeOnDemand) + azureEnv.UnavailableOfferingsCache.MarkUnavailable(ctx, "ZonalAllocationFailure", "Standard_D2_v2", fmt.Sprintf("%s-1", fake.Region), v1alpha5.CapacityTypeSpot) + + ExpectApplied(ctx, env.Client, nodeClass, nodePool) + pod := coretest.UnschedulablePod(coretest.PodOptions{ + NodeSelector: map[string]string{v1.LabelInstanceTypeStable: "Standard_D2_v2"}, + }) + pod.Spec.Affinity = &v1.Affinity{NodeAffinity: &v1.NodeAffinity{PreferredDuringSchedulingIgnoredDuringExecution: []v1.PreferredSchedulingTerm{ + { + Weight: 1, Preference: v1.NodeSelectorTerm{MatchExpressions: []v1.NodeSelectorRequirement{ + {Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpIn, Values: []string{fmt.Sprintf("%s-1", fake.Region)}}, + }}, + }, + }}} + ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, coreProvisioner, pod) + node := ExpectScheduled(ctx, env.Client, pod) + Expect(node.Labels["karpenter.k8s.azure/zone"]).ToNot(Equal(fmt.Sprintf("%s-1", fake.Region))) + Expect(node.Labels["node.kubernetes.io/instance-type"]).To(Equal("Standard_D2_v2")) + }) + It("should launch smaller instances than optimal if larger instance launch results in Insufficient Capacity Error", func() { + azureEnv.UnavailableOfferingsCache.MarkUnavailable(ctx, "SubscriptionQuotaReached", "Standard_F16s_v2", fmt.Sprintf("%s-1", fake.Region), v1alpha5.CapacityTypeOnDemand) + azureEnv.UnavailableOfferingsCache.MarkUnavailable(ctx, "SubscriptionQuotaReached", "Standard_F16s_v2", fmt.Sprintf("%s-1", fake.Region), v1alpha5.CapacityTypeSpot) + coretest.ReplaceRequirements(nodePool, v1.NodeSelectorRequirement{ + Key: v1.LabelInstanceTypeStable, + Operator: v1.NodeSelectorOpIn, + Values: []string{"Standard_DS2_v2", "Standard_F16s_v2"}, + }) + pods := []*v1.Pod{} + for i := 0; i < 2; i++ { + pods = append(pods, coretest.UnschedulablePod(coretest.PodOptions{ + ResourceRequirements: v1.ResourceRequirements{ + Requests: v1.ResourceList{v1.ResourceCPU: resource.MustParse("1")}, + }, + NodeSelector: map[string]string{ + v1.LabelTopologyZone: fmt.Sprintf("%s-1", fake.Region), + }, + })) + } + // Provisions 2 smaller instances since larger was ICE'd + ExpectApplied(ctx, env.Client, nodeClass, nodePool) + ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, coreProvisioner, pods...) + nodeNames := sets.New[string]() + for _, pod := range pods { + node := ExpectScheduled(ctx, env.Client, pod) + Expect(node.Labels[v1.LabelInstanceTypeStable]).To(Equal("Standard_DS2_v2")) + nodeNames.Insert(node.Name) + } + Expect(nodeNames.Len()).To(Equal(2)) + }) + It("should launch instances on later reconciliation attempt with Insufficient Capacity Error Cache expiry", func() { + zones := []string{"1", "2", "3"} + for _, zone := range zones { + azureEnv.UnavailableOfferingsCache.MarkUnavailable(ctx, "SubscriptionQuotaReached", "Standard_D2_v2", fmt.Sprintf("%s-%s", fake.Region, zone), v1alpha5.CapacityTypeSpot) + azureEnv.UnavailableOfferingsCache.MarkUnavailable(ctx, "SubscriptionQuotaReached", "Standard_D2_v2", fmt.Sprintf("%s-%s", fake.Region, zone), v1alpha5.CapacityTypeOnDemand) + } + + ExpectApplied(ctx, env.Client, nodeClass, nodePool) + pod := coretest.UnschedulablePod(coretest.PodOptions{ + NodeSelector: map[string]string{v1.LabelInstanceTypeStable: "Standard_D2_v2"}, + }) + ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, coreProvisioner, pod) + ExpectNotScheduled(ctx, env.Client, pod) + // capacity shortage is over - expire the items from the cache and try again + azureEnv.UnavailableOfferingsCache.Flush() + ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, coreProvisioner, pod) + node := ExpectScheduled(ctx, env.Client, pod) + Expect(node.Labels).To(HaveKeyWithValue(v1.LabelInstanceTypeStable, "Standard_D2_v2")) + }) + }) + Context("Provider List", func() { + var instanceTypes corecloudprovider.InstanceTypes + var err error + + BeforeEach(func() { + // disable VM memory overhead for simpler capacity testing + ctx = settings.ToContext(ctx, test.Settings(test.SettingOptions{ + VMMemoryOverheadPercent: lo.ToPtr[float64](0), + })) + instanceTypes, err = azureEnv.InstanceTypesProvider.List(ctx, &corev1beta1.KubeletConfiguration{}, nodeClass) + Expect(err).ToNot(HaveOccurred()) + }) + + It("should have all the requirements on every sku", func() { + for _, instanceType := range instanceTypes { + reqs := instanceType.Requirements + + Expect(reqs.Has(v1.LabelArchStable)).To(BeTrue()) + Expect(reqs.Has(v1.LabelOSStable)).To(BeTrue()) + Expect(reqs.Has(v1.LabelInstanceTypeStable)).To(BeTrue()) + + Expect(reqs.Has(v1alpha2.LabelSKUName)).To(BeTrue()) + + Expect(reqs.Has(v1alpha2.LabelSKUStoragePremiumCapable)).To(BeTrue()) + Expect(reqs.Has(v1alpha2.LabelSKUEncryptionAtHostSupported)).To(BeTrue()) + Expect(reqs.Has(v1alpha2.LabelSKUAcceleratedNetworking)).To(BeTrue()) + Expect(reqs.Has(v1alpha2.LabelSKUHyperVGeneration)).To(BeTrue()) + Expect(reqs.Has(v1alpha2.LabelSKUStorageEphemeralOSMaxSize)).To(BeTrue()) + } + }) + + It("should have all compute capacity", func() { + for _, instanceType := range instanceTypes { + capList := instanceType.Capacity + Expect(capList).To(HaveKey(v1.ResourceCPU)) + Expect(capList).To(HaveKey(v1.ResourceMemory)) + Expect(capList).To(HaveKey(v1.ResourcePods)) + Expect(capList).To(HaveKey(v1.ResourceEphemeralStorage)) + } + }) + + It("should propagate all values to requirements from skewer", func() { + var gpuNode *corecloudprovider.InstanceType + var normalNode *corecloudprovider.InstanceType + + for _, instanceType := range instanceTypes { + if instanceType.Name == "Standard_D2_v2" { + normalNode = instanceType + } + // #nosec G101 + if instanceType.Name == "Standard_NC24ads_A100_v4" { + gpuNode = instanceType + } + } + + Expect(normalNode.Name).To(Equal("Standard_D2_v2")) + Expect(gpuNode.Name).To(Equal("Standard_NC24ads_A100_v4")) + + Expect(normalNode.Requirements.Get(v1alpha2.LabelSKUName).Values()).To(ConsistOf("Standard_D2_v2")) + Expect(gpuNode.Requirements.Get(v1alpha2.LabelSKUName).Values()).To(ConsistOf("Standard_NC24ads_A100_v4")) + + Expect(normalNode.Requirements.Get(v1alpha2.LabelSKUHyperVGeneration).Values()).To(ConsistOf(v1alpha2.HyperVGenerationV1)) + Expect(gpuNode.Requirements.Get(v1alpha2.LabelSKUHyperVGeneration).Values()).To(ConsistOf(v1alpha2.HyperVGenerationV2)) + + Expect(gpuNode.Requirements.Get(v1alpha2.LabelSKUAccelerator).Values()).To(ConsistOf("A100")) + + Expect(normalNode.Requirements.Get(v1alpha2.LabelSKUVersion).Values()).To(ConsistOf("2")) + Expect(gpuNode.Requirements.Get(v1alpha2.LabelSKUVersion).Values()).To(ConsistOf("4")) + + // CPU (requirements and capacity) + Expect(normalNode.Requirements.Get(v1alpha2.LabelSKUCPU).Values()).To(ConsistOf("2")) + Expect(normalNode.Capacity.Cpu().Value()).To(Equal(int64(2))) + Expect(gpuNode.Requirements.Get(v1alpha2.LabelSKUCPU).Values()).To(ConsistOf("24")) + Expect(gpuNode.Capacity.Cpu().Value()).To(Equal(int64(24))) + + // Memory (requirements and capacity) + Expect(normalNode.Requirements.Get(v1alpha2.LabelSKUMemory).Values()).To(ConsistOf(fmt.Sprint(7 * 1024))) // 7GiB in MiB + Expect(normalNode.Capacity.Memory().Value()).To(Equal(int64(7 * 1024 * 1024 * 1024))) // 7GiB in bytes + Expect(gpuNode.Requirements.Get(v1alpha2.LabelSKUMemory).Values()).To(ConsistOf(fmt.Sprint(220 * 1024))) // 220GiB in MiB + Expect(gpuNode.Capacity.Memory().Value()).To(Equal(int64(220 * 1024 * 1024 * 1024))) // 220GiB in bytes + + // GPU -- Number of GPUs + gpuQuantity, ok := gpuNode.Capacity["nvidia.com/gpu"] + Expect(ok).To(BeTrue(), "Expected nvidia.com/gpu to be present in capacity") + Expect(gpuQuantity.Value()).To(Equal(int64(1))) + + gpuQuanityNonGPU, ok := normalNode.Capacity["nvidia.com/gpu"] + Expect(ok).To(BeTrue(), "Expected nvidia.com/gpu to be present in capacity, and be zero") + Expect(gpuQuanityNonGPU.Value()).To(Equal(int64(0))) + }) + + It("should not include SKUs with constrained CPUs, but include unconstrained ones", func() { + getName := func(instanceType *corecloudprovider.InstanceType) string { return instanceType.Name } + Expect(instanceTypes).ShouldNot(ContainElement(WithTransform(getName, Equal("Standard_M8-2ms")))) + Expect(instanceTypes).Should(ContainElement(WithTransform(getName, Equal("Standard_D2_v2")))) + }) + }) + + Context("Instance Types", func() { + It("should support provisioning with no labels", func() { + ExpectApplied(ctx, env.Client, nodePool, nodeClass) + pod := coretest.UnschedulablePod(coretest.PodOptions{}) + ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, coreProvisioner, pod) + ExpectScheduled(ctx, env.Client, pod) + }) + Context("VM profile", func() { + It("should have OS disk and network interface set to auto-delete", func() { + ExpectApplied(ctx, env.Client, nodePool, nodeClass) + pod := coretest.UnschedulablePod() + ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, coreProvisioner, pod) + ExpectScheduled(ctx, env.Client, pod) + + Expect(azureEnv.VirtualMachinesAPI.VirtualMachineCreateOrUpdateBehavior.CalledWithInput.Len()).To(Equal(1)) + vm := azureEnv.VirtualMachinesAPI.VirtualMachineCreateOrUpdateBehavior.CalledWithInput.Pop().VM + Expect(vm.Properties).ToNot(BeNil()) + + Expect(vm.Properties.StorageProfile).ToNot(BeNil()) + Expect(vm.Properties.StorageProfile.OSDisk).ToNot(BeNil()) + osDiskDeleteOption := vm.Properties.StorageProfile.OSDisk.DeleteOption + Expect(osDiskDeleteOption).ToNot(BeNil()) + Expect(lo.FromPtr(osDiskDeleteOption)).To(Equal(armcompute.DiskDeleteOptionTypesDelete)) + + Expect(vm.Properties.StorageProfile.ImageReference).ToNot(BeNil()) + + for _, nic := range vm.Properties.NetworkProfile.NetworkInterfaces { + nicDeleteOption := nic.Properties.DeleteOption + Expect(nicDeleteOption).To(Not(BeNil())) + Expect(lo.FromPtr(nicDeleteOption)).To(Equal(armcompute.DeleteOptionsDelete)) + } + }) + }) + + It("should have VM identity set", func() { + ctx = settings.ToContext( + ctx, + test.Settings(test.SettingOptions{ + NodeIdentities: []string{ + "/subscriptions/1234/resourceGroups/mcrg/providers/Microsoft.ManagedIdentity/userAssignedIdentities/myid1", + "/subscriptions/1234/resourceGroups/mcrg/providers/Microsoft.ManagedIdentity/userAssignedIdentities/myid2", + }, + })) + + ExpectApplied(ctx, env.Client, nodePool, nodeClass) + pod := coretest.UnschedulablePod() + ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, coreProvisioner, pod) + ExpectScheduled(ctx, env.Client, pod) + + Expect(azureEnv.VirtualMachinesAPI.VirtualMachineCreateOrUpdateBehavior.CalledWithInput.Len()).To(Equal(1)) + vm := azureEnv.VirtualMachinesAPI.VirtualMachineCreateOrUpdateBehavior.CalledWithInput.Pop().VM + Expect(vm.Identity).ToNot(BeNil()) + + Expect(lo.FromPtr(vm.Identity.Type)).To(Equal(armcompute.ResourceIdentityTypeUserAssigned)) + Expect(vm.Identity.UserAssignedIdentities).ToNot(BeNil()) + Expect(vm.Identity.UserAssignedIdentities).To(HaveLen(2)) + Expect(vm.Identity.UserAssignedIdentities).To(HaveKey("/subscriptions/1234/resourceGroups/mcrg/providers/Microsoft.ManagedIdentity/userAssignedIdentities/myid1")) + Expect(vm.Identity.UserAssignedIdentities).To(HaveKey("/subscriptions/1234/resourceGroups/mcrg/providers/Microsoft.ManagedIdentity/userAssignedIdentities/myid2")) + }) + }) + + Context("GPU workloads and Nodes", func() { + It("should schedule non-GPU pod onto the cheapest non-GPU capable node", func() { + ExpectApplied(ctx, env.Client, nodePool, nodeClass) + pod := coretest.UnschedulablePod(coretest.PodOptions{}) + ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, coreProvisioner, pod) + ExpectScheduled(ctx, env.Client, pod) + + Expect(azureEnv.VirtualMachinesAPI.VirtualMachineCreateOrUpdateBehavior.CalledWithInput.Len()).To(Equal(1)) + vm := azureEnv.VirtualMachinesAPI.VirtualMachineCreateOrUpdateBehavior.CalledWithInput.Pop().VM + Expect(vm.Properties).ToNot(BeNil()) + Expect(vm.Properties.HardwareProfile).ToNot(BeNil()) + Expect(utils.IsNvidiaEnabledSKU(string(*vm.Properties.HardwareProfile.VMSize))).To(BeFalse()) + + clusterNodes := cluster.Nodes() + node := clusterNodes[0] + if node.Name() == pod.Spec.NodeName { + nodeLabels := node.GetLabels() + Expect(nodeLabels).To(HaveKeyWithValue("karpenter.k8s.azure/sku-gpu-count", "0")) + } + }) + + It("should schedule GPU pod on GPU capable node", func() { + ExpectApplied(ctx, env.Client, nodePool, nodeClass) + pod := coretest.UnschedulablePod(coretest.PodOptions{ + ObjectMeta: metav1.ObjectMeta{ + Name: "samples-tf-mnist-demo", + Labels: map[string]string{ + "app": "samples-tf-mnist-demo", + }, + }, + Image: "mcr.microsoft.com/azuredocs/samples-tf-mnist-demo:gpu", + ResourceRequirements: v1.ResourceRequirements{ + Limits: v1.ResourceList{ + "nvidia.com/gpu": resource.MustParse("1"), + }, + }, + RestartPolicy: v1.RestartPolicy("OnFailure"), + Tolerations: []v1.Toleration{ + { + Key: "sku", + Operator: v1.TolerationOpEqual, + Value: "gpu", + Effect: v1.TaintEffectNoSchedule, + }, + }, + }) + + ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, coreProvisioner, pod) + ExpectScheduled(ctx, env.Client, pod) + + // Verify that the node has the GPU label set that the pod was scheduled on + clusterNodes := cluster.Nodes() + Expect(clusterNodes).ToNot(BeEmpty()) + Expect(len(clusterNodes)).To(Equal(1)) + node := clusterNodes[0] + Expect(node.Node.Status.Allocatable).To(HaveKeyWithValue(v1.ResourceName("nvidia.com/gpu"), resource.MustParse("1"))) + + if node.Name() == pod.Spec.NodeName { + nodeLabels := node.GetLabels() + + Expect(nodeLabels).To(HaveKeyWithValue("karpenter.k8s.azure/sku-gpu-name", "A100")) + Expect(nodeLabels).To(HaveKeyWithValue("karpenter.k8s.azure/sku-gpu-manufacturer", v1alpha2.ManufacturerNvidia)) + Expect(nodeLabels).To(HaveKeyWithValue("karpenter.k8s.azure/sku-gpu-count", "1")) + + } + }) + + Context("Provisioner with KubeletConfig", func() { + It("Should support provisioning with kubeletConfig, computeResources and maxPods not specified", func() { + + nodePool.Spec.Template.Spec.Kubelet = &corev1beta1.KubeletConfiguration{ + PodsPerCore: lo.ToPtr(int32(110)), + EvictionSoft: map[string]string{ + instancetype.MemoryAvailable: "1Gi", + }, + EvictionSoftGracePeriod: map[string]metav1.Duration{ + instancetype.MemoryAvailable: {Duration: 10 * time.Second}, + }, + EvictionMaxPodGracePeriod: lo.ToPtr(int32(15)), + ImageGCHighThresholdPercent: lo.ToPtr(int32(30)), + ImageGCLowThresholdPercent: lo.ToPtr(int32(20)), + CPUCFSQuota: lo.ToPtr(true), + } + + ExpectApplied(ctx, env.Client, nodePool, nodeClass) + pod := coretest.UnschedulablePod() + ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, coreProvisioner, pod) + ExpectScheduled(ctx, env.Client, pod) + + Expect(azureEnv.VirtualMachinesAPI.VirtualMachineCreateOrUpdateBehavior.CalledWithInput.Len()).To(Equal(1)) + vm := azureEnv.VirtualMachinesAPI.VirtualMachineCreateOrUpdateBehavior.CalledWithInput.Pop().VM + customData := *vm.Properties.OSProfile.CustomData + Expect(customData).ToNot(BeNil()) + decodedBytes, err := base64.StdEncoding.DecodeString(customData) + Expect(err).To(Succeed()) + decodedString := string(decodedBytes[:]) + kubeletFlags := decodedString[strings.Index(decodedString, "KUBELET_FLAGS=")+len("KUBELET_FLAGS="):] + + Expect(kubeletFlags).To(SatisfyAny( // AKS default + ContainSubstring("--system-reserved=cpu=0,memory=0"), + ContainSubstring("--system-reserved=memory=0,cpu=0"), + )) + Expect(kubeletFlags).To(SatisfyAny( // AKS calculation based on cpu and memory + ContainSubstring("--kube-reserved=cpu=100m,memory=1843Mi"), + ContainSubstring("--kube-reserved=memory=1843Mi,cpu=100m"), + )) + Expect(kubeletFlags).To(ContainSubstring("--eviction-hard=memory.available<750Mi")) // AKS default + Expect(kubeletFlags).To(ContainSubstring("--eviction-soft=memory.available<1Gi")) + Expect(kubeletFlags).To(ContainSubstring("--eviction-soft-grace-period=memory.available=10s")) + Expect(kubeletFlags).To(ContainSubstring("--max-pods=100")) // kubenet + Expect(kubeletFlags).To(ContainSubstring("--pods-per-core=110")) + Expect(kubeletFlags).To(ContainSubstring("--image-gc-low-threshold=20")) + Expect(kubeletFlags).To(ContainSubstring("--image-gc-high-threshold=30")) + Expect(kubeletFlags).To(ContainSubstring("--cpu-cfs-quota=true")) + }) + It("Should support provisioning with kubeletConfig, computeResources and maxPods specified", func() { + + nodePool.Spec.Template.Spec.Kubelet = &corev1beta1.KubeletConfiguration{ + PodsPerCore: lo.ToPtr(int32(110)), + EvictionSoft: map[string]string{ + instancetype.MemoryAvailable: "1Gi", + }, + EvictionSoftGracePeriod: map[string]metav1.Duration{ + instancetype.MemoryAvailable: {Duration: 10 * time.Second}, + }, + EvictionMaxPodGracePeriod: lo.ToPtr(int32(15)), + ImageGCHighThresholdPercent: lo.ToPtr(int32(30)), + ImageGCLowThresholdPercent: lo.ToPtr(int32(20)), + CPUCFSQuota: lo.ToPtr(true), + + SystemReserved: v1.ResourceList{ + v1.ResourceCPU: resource.MustParse("200m"), + v1.ResourceMemory: resource.MustParse("1Gi"), + }, + KubeReserved: v1.ResourceList{ + v1.ResourceCPU: resource.MustParse("100m"), + v1.ResourceMemory: resource.MustParse("500Mi"), + }, + EvictionHard: map[string]string{ + instancetype.MemoryAvailable: "10Mi", + }, + MaxPods: lo.ToPtr(int32(15)), + } + + ExpectApplied(ctx, env.Client, nodePool, nodeClass) + pod := coretest.UnschedulablePod() + ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, coreProvisioner, pod) + ExpectScheduled(ctx, env.Client, pod) + + Expect(azureEnv.VirtualMachinesAPI.VirtualMachineCreateOrUpdateBehavior.CalledWithInput.Len()).To(Equal(1)) + vm := azureEnv.VirtualMachinesAPI.VirtualMachineCreateOrUpdateBehavior.CalledWithInput.Pop().VM + customData := *vm.Properties.OSProfile.CustomData + Expect(customData).ToNot(BeNil()) + decodedBytes, err := base64.StdEncoding.DecodeString(customData) + Expect(err).To(Succeed()) + decodedString := string(decodedBytes[:]) + kubeletFlags := decodedString[strings.Index(decodedString, "KUBELET_FLAGS=")+len("KUBELET_FLAGS="):] + + Expect(kubeletFlags).To(SatisfyAny( // AKS default + ContainSubstring("--system-reserved=cpu=0,memory=0"), + ContainSubstring("--system-reserved=memory=0,cpu=0"), + )) + Expect(kubeletFlags).To(SatisfyAny( // AKS calculation based on cpu and memory + ContainSubstring("--kube-reserved=cpu=100m,memory=1843Mi"), + ContainSubstring("--kube-reserved=memory=1843Mi,cpu=100m"), + )) + Expect(kubeletFlags).To(ContainSubstring("--eviction-hard=memory.available<750Mi")) // AKS default + Expect(kubeletFlags).To(ContainSubstring("--eviction-soft=memory.available<1Gi")) + Expect(kubeletFlags).To(ContainSubstring("--eviction-soft-grace-period=memory.available=10s")) + Expect(kubeletFlags).To(ContainSubstring("--max-pods=100")) // kubenet + Expect(kubeletFlags).To(ContainSubstring("--pods-per-core=110")) + Expect(kubeletFlags).To(ContainSubstring("--image-gc-low-threshold=20")) + Expect(kubeletFlags).To(ContainSubstring("--image-gc-high-threshold=30")) + Expect(kubeletFlags).To(ContainSubstring("--cpu-cfs-quota=true")) + }) + }) + }) + + Context("LoadBalancer backend pools", func() { + resourceGroup := "test-resourceGroup" + + It("should include loadbalancer backend pools the allocated VMs", func() { + standardLB := test.MakeStandardLoadBalancer(resourceGroup, loadbalancer.SLBName, true) + internalLB := test.MakeStandardLoadBalancer(resourceGroup, loadbalancer.InternalSLBName, false) + + azureEnv.LoadBalancersAPI.LoadBalancers.Store(standardLB.ID, standardLB) + azureEnv.LoadBalancersAPI.LoadBalancers.Store(internalLB.ID, internalLB) + + ExpectApplied(ctx, env.Client, nodePool, nodeClass) + pod := coretest.UnschedulablePod() + ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, coreProvisioner, pod) + ExpectScheduled(ctx, env.Client, pod) + + Expect(azureEnv.VirtualMachinesAPI.VirtualMachineCreateOrUpdateBehavior.CalledWithInput.Len()).To(Equal(1)) + iface := azureEnv.NetworkInterfacesAPI.NetworkInterfacesCreateOrUpdateBehavior.CalledWithInput.Pop().Interface + + Expect(iface.Properties.IPConfigurations).ToNot(BeEmpty()) + Expect(lo.FromPtr(iface.Properties.IPConfigurations[0].Properties.Primary)).To(Equal(true)) + + backendPools := iface.Properties.IPConfigurations[0].Properties.LoadBalancerBackendAddressPools + Expect(backendPools).To(HaveLen(3)) + Expect(lo.FromPtr(backendPools[0].ID)).To(Equal("/subscriptions/subscriptionID/resourceGroups/test-resourceGroup/providers/Microsoft.Network/loadBalancers/kubernetes/backendAddressPools/kubernetes")) + Expect(lo.FromPtr(backendPools[1].ID)).To(Equal("/subscriptions/subscriptionID/resourceGroups/test-resourceGroup/providers/Microsoft.Network/loadBalancers/kubernetes/backendAddressPools/aksOutboundBackendPool")) + Expect(lo.FromPtr(backendPools[2].ID)).To(Equal("/subscriptions/subscriptionID/resourceGroups/test-resourceGroup/providers/Microsoft.Network/loadBalancers/kubernetes-internal/backendAddressPools/kubernetes")) + }) + }) + +}) + +var _ = Describe("Tax Calculator", func() { + Context("KubeReservedResources", func() { + It("4 cores, 7GiB", func() { + cpus := int64(4) // 4 cores + memory := 7.0 // 7 GiB + expectedCPU := "140m" + expectedMemory := "1638Mi" + + resources := instancetype.KubeReservedResources(cpus, memory) + gotCPU := resources[v1.ResourceCPU] + gotMemory := resources[v1.ResourceMemory] + + Expect(gotCPU.String()).To(Equal(expectedCPU)) + Expect(gotMemory.String()).To(Equal(expectedMemory)) + }) + + It("2 cores, 8GiB", func() { + cpus := int64(2) // 2 cores + memory := 8.0 // 8 GiB + expectedCPU := "100m" + expectedMemory := "1843Mi" + + resources := instancetype.KubeReservedResources(cpus, memory) + gotCPU := resources[v1.ResourceCPU] + gotMemory := resources[v1.ResourceMemory] + + Expect(gotCPU.String()).To(Equal(expectedCPU)) + Expect(gotMemory.String()).To(Equal(expectedMemory)) + }) + + It("3 cores, 64GiB", func() { + cpus := int64(3) // 3 cores + memory := 64.0 // 64 GiB + expectedCPU := "120m" + expectedMemory := "5611Mi" + + resources := instancetype.KubeReservedResources(cpus, memory) + gotCPU := resources[v1.ResourceCPU] + gotMemory := resources[v1.ResourceMemory] + + Expect(gotCPU.String()).To(Equal(expectedCPU)) + Expect(gotMemory.String()).To(Equal(expectedMemory)) + }) + }) +}) diff --git a/pkg/providers/launchtemplate/launchtemplate.go b/pkg/providers/launchtemplate/launchtemplate.go new file mode 100644 index 000000000..a317d9169 --- /dev/null +++ b/pkg/providers/launchtemplate/launchtemplate.go @@ -0,0 +1,140 @@ +/* +Portions Copyright (c) Microsoft Corporation. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package launchtemplate + +import ( + "context" + "strings" + + "github.com/Azure/go-autorest/autorest/to" + "github.com/Azure/karpenter/pkg/providers/imagefamily" + "github.com/Azure/karpenter/pkg/providers/launchtemplate/parameters" + + "github.com/samber/lo" + + "github.com/Azure/karpenter/pkg/apis/settings" + "github.com/Azure/karpenter/pkg/apis/v1alpha2" + corev1beta1 "github.com/aws/karpenter-core/pkg/apis/v1beta1" + + "github.com/aws/karpenter-core/pkg/cloudprovider" +) + +const ( + karpenterManagedTagKey = "karpenter.azure.com/cluster" +) + +type Template struct { + UserData string + ImageID string + Tags map[string]*string +} + +type Provider struct { + imageFamily *imagefamily.Resolver + imageProvider *imagefamily.Provider + caBundle *string + clusterEndpoint string + tenantID string + subscriptionID string + userAssignedIdentityID string + resourceGroup string + location string +} + +// TODO: add caching of launch templates + +func NewProvider(_ context.Context, imageFamily *imagefamily.Resolver, imageProvider *imagefamily.Provider, caBundle *string, clusterEndpoint string, + tenantID, subscriptionID, userAssignedIdentityID, resourceGroup, location string, +) *Provider { + return &Provider{ + imageFamily: imageFamily, + imageProvider: imageProvider, + caBundle: caBundle, + clusterEndpoint: clusterEndpoint, + tenantID: tenantID, + subscriptionID: subscriptionID, + userAssignedIdentityID: userAssignedIdentityID, + resourceGroup: resourceGroup, + location: location, + } +} + +func (p *Provider) GetTemplate(ctx context.Context, nodeClass *v1alpha2.AKSNodeClass, nodeClaim *corev1beta1.NodeClaim, + instanceType *cloudprovider.InstanceType, additionalLabels map[string]string) (*Template, error) { + staticParameters := p.getStaticParameters(ctx, nodeClass, lo.Assign(nodeClaim.Labels, additionalLabels)) + kubeServerVersion, err := p.imageProvider.KubeServerVersion(ctx) + if err != nil { + return nil, err + } + staticParameters.KubernetesVersion = kubeServerVersion + templateParameters, err := p.imageFamily.Resolve(ctx, nodeClass, nodeClaim, instanceType, staticParameters) + if err != nil { + return nil, err + } + launchTemplate, err := p.createLaunchTemplate(ctx, templateParameters) + if err != nil { + return nil, err + } + + return launchTemplate, nil +} + +func (p *Provider) getStaticParameters(ctx context.Context, nodeTemplate *v1alpha2.AKSNodeClass, labels map[string]string) *parameters.StaticParameters { + return ¶meters.StaticParameters{ + ClusterName: settings.FromContext(ctx).ClusterName, + ClusterEndpoint: p.clusterEndpoint, + Tags: lo.Assign(settings.FromContext(ctx).Tags, nodeTemplate.Spec.Tags), + Labels: labels, + CABundle: p.caBundle, + + TenantID: p.tenantID, + SubscriptionID: p.subscriptionID, + UserAssignedIdentityID: p.userAssignedIdentityID, + ResourceGroup: p.resourceGroup, + Location: p.location, + ClusterID: settings.FromContext(ctx).ClusterID, + APIServerName: settings.FromContext(ctx).GetAPIServerName(), + KubeletClientTLSBootstrapToken: settings.FromContext(ctx).KubeletClientTLSBootstrapToken, + NetworkPlugin: settings.FromContext(ctx).NetworkPlugin, + NetworkPolicy: settings.FromContext(ctx).NetworkPolicy, + } +} + +func (p *Provider) createLaunchTemplate(_ context.Context, options *parameters.Parameters) (*Template, error) { + // render user data + userData, err := options.UserData.Script() + if err != nil { + return nil, err + } + + // merge and convert to ARM tags + azureTags := mergeTags(options.Tags, map[string]string{karpenterManagedTagKey: options.ClusterName}) + template := &Template{ + UserData: userData, + ImageID: options.ImageID, + Tags: azureTags, + } + return template, nil +} + +// MergeTags takes a variadic list of maps and merges them together +// with format acceptable to ARM (no / in keys, pointer to strings as values) +func mergeTags(tags ...map[string]string) (result map[string]*string) { + return lo.MapEntries(lo.Assign(tags...), func(key string, value string) (string, *string) { + return strings.ReplaceAll(key, "/", "_"), to.StringPtr(value) + }) +} diff --git a/pkg/providers/launchtemplate/parameters/types.go b/pkg/providers/launchtemplate/parameters/types.go new file mode 100644 index 000000000..45a11c6d7 --- /dev/null +++ b/pkg/providers/launchtemplate/parameters/types.go @@ -0,0 +1,50 @@ +/* +Portions Copyright (c) Microsoft Corporation. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package parameters + +import ( + "github.com/Azure/karpenter/pkg/providers/imagefamily/bootstrap" +) + +// StaticParameters define the static launch template parameters +type StaticParameters struct { + ClusterName string + ClusterEndpoint string + CABundle *string + + TenantID string + SubscriptionID string + UserAssignedIdentityID string + Location string + ResourceGroup string + ClusterID string + APIServerName string + KubeletClientTLSBootstrapToken string + NetworkPlugin string + NetworkPolicy string + KubernetesVersion string + + Tags map[string]string + Labels map[string]string +} + +// Parameters adds the dynamically generated launch template parameters +type Parameters struct { + *StaticParameters + UserData bootstrap.Bootstrapper + ImageID string +} diff --git a/pkg/providers/loadbalancer/client.go b/pkg/providers/loadbalancer/client.go new file mode 100644 index 000000000..f4e2d176d --- /dev/null +++ b/pkg/providers/loadbalancer/client.go @@ -0,0 +1,29 @@ +/* +Portions Copyright (c) Microsoft Corporation. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package loadbalancer + +import ( + "context" + + "github.com/Azure/azure-sdk-for-go/sdk/azcore/runtime" + "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/network/armnetwork" +) + +type LoadBalancersAPI interface { + Get(ctx context.Context, resourceGroupName string, loadBalancerName string, options *armnetwork.LoadBalancersClientGetOptions) (armnetwork.LoadBalancersClientGetResponse, error) + NewListPager(resourceGroupName string, options *armnetwork.LoadBalancersClientListOptions) *runtime.Pager[armnetwork.LoadBalancersClientListResponse] +} diff --git a/pkg/providers/loadbalancer/loadbalancer.go b/pkg/providers/loadbalancer/loadbalancer.go new file mode 100644 index 000000000..d85a707cc --- /dev/null +++ b/pkg/providers/loadbalancer/loadbalancer.go @@ -0,0 +1,186 @@ +/* +Portions Copyright (c) Microsoft Corporation. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package loadbalancer + +import ( + "context" + "fmt" + "strings" + "sync" + "time" + + "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/network/armnetwork" + "github.com/patrickmn/go-cache" + "github.com/samber/lo" + "knative.dev/pkg/logging" +) + +const ( + SLBName = "kubernetes" + SLBNameIPv6 = "kubernetes-ipv6" + + // InternalSLBName is the name of the internal SLB created by cloudprovider when Services with type LoadBalancer are created. This may not exist + // when the pod launches/Karpenter is created. We query it as an optimization to save cloudprovider work, as otherwise cloudprovider must edit + // ever VM we deploy to include this LB. + InternalSLBName = "kubernetes-internal" + + // SLBOutboundBackendPoolName is the AKS SLB outbound backend pool name + SLBOutboundBackendPoolName = "aksOutboundBackendPool" + // SLBOutboundBackendPoolNameIPv6 is the AKS SLB outbound backend pool name for IPv6 traffic + SLBOutboundBackendPoolNameIPv6 = "aksOutboundBackendPool-ipv6" + // SLBInboundBackendPoolName is the AKS SLB inbound backend pool name + SLBInboundBackendPoolName = "kubernetes" + // SLBInboundBackendPoolNameIPv6 is the AKS SLB inbound backend pool name for IPv6 traffic + SLBInboundBackendPoolNameIPv6 = "kubernetes-ipv6" + + loadBalancersCacheKey = "LoadBalancers" + + // LoadBalancersCacheTTL configures how freuqently we check for updates to the LBs. + // Currently the choice of this value is entirely "how much work do we want to save cloudprovider". + // The faster we do this, the faster we notice the creation of a kubernetes-internal LB and start + // including it on new VMs, which saves CloudProvider needing to do that. + LoadBalancersCacheTTL = 2 * time.Hour +) + +type Provider struct { + loadBalancersAPI LoadBalancersAPI + resourceGroup string + cache *cache.Cache + mu sync.Mutex +} + +type BackendAddressPools struct { + IPv4PoolIDs []string + IPv6PoolIDs []string // TODO: This is always empty currently +} + +// NewProvider creates a new LoadBalancer provider +func NewProvider(loadBalancersAPI LoadBalancersAPI, cache *cache.Cache, resourceGroup string) *Provider { + return &Provider{ + loadBalancersAPI: loadBalancersAPI, + cache: cache, + resourceGroup: resourceGroup, + } +} + +// LoadBalancerBackendPools returns a collection of IPv4 and IPv6 LoadBalancer backend pools. +// This collection is collected from Azure periodically but usually served from a cache to reduce +// Azure request load. +func (p *Provider) LoadBalancerBackendPools(ctx context.Context) (*BackendAddressPools, error) { + loadBalancers, err := p.getLoadBalancers(ctx) + if err != nil { + return nil, err + } + + backendAddressPools := lo.FlatMap(loadBalancers, extractBackendAddressPools) + ipv4PoolIDs := lo.FilterMap(backendAddressPools, func(backendPool *armnetwork.BackendAddressPool, idx int) (string, bool) { + if !isBackendAddressPoolApplicable(backendPool, idx) { + return "", false + } + + return lo.FromPtr(backendPool.ID), true + }) + + logging.FromContext(ctx).Debugf("Returning %d IPv4 backend pools: %s", len(ipv4PoolIDs), ipv4PoolIDs) + + // RP only actually assigns the LB backend pools to VMs if OutboundType is LoadBalancer, + // but that's also the only OutboundType which creates the LoadBalancer, so as long as we're not allowing + // OutboundType changes, we can just infer that if the LBs exist we should assign them. + return &BackendAddressPools{ + IPv4PoolIDs: ipv4PoolIDs, + // TODO: IPv6 deferred for now. When they're used they must be put onto a non-primary NIC. + }, nil +} + +func (p *Provider) getLoadBalancers(ctx context.Context) ([]*armnetwork.LoadBalancer, error) { + p.mu.Lock() + defer p.mu.Unlock() + + if cached, ok := p.cache.Get(loadBalancersCacheKey); ok { + return cached.([]*armnetwork.LoadBalancer), nil + } + + lbs, err := p.loadFromAzure(ctx) + if err != nil { + return nil, err + } + + // If we wanted to hyper-optimize, we could set a much longer timeout once we find the -internal LB, as at that point we're "done" and + // aren't particularly interested in LB changes anymore. + p.cache.SetDefault(loadBalancersCacheKey, lbs) + + return lbs, nil +} + +func (p *Provider) loadFromAzure(ctx context.Context) ([]*armnetwork.LoadBalancer, error) { + logging.FromContext(ctx).Infof("Querying load balancers in resource group %s", p.resourceGroup) + + pager := p.loadBalancersAPI.NewListPager(p.resourceGroup, nil) + + var lbs []*armnetwork.LoadBalancer + for pager.More() { + page, err := pager.NextPage(ctx) + if err != nil { + return nil, fmt.Errorf("failed to get next loadbalancer page: %w", err) + } + lbs = append(lbs, page.Value...) + } + + // Only return the LBs we actually care about + result := lo.Filter(lbs, isClusterLoadBalancer) + logging.FromContext(ctx).Infof("Found %d load balancers of interest", len(result)) + return result, nil +} + +func isClusterLoadBalancer(lb *armnetwork.LoadBalancer, _ int) bool { + name := lo.FromPtr(lb.Name) + return strings.EqualFold(name, SLBName) || strings.EqualFold(name, InternalSLBName) // TODO: Not currently supporting IPv6 +} + +func extractBackendAddressPools(lb *armnetwork.LoadBalancer, _ int) []*armnetwork.BackendAddressPool { + if lb.Properties == nil { + return nil + } + + return lb.Properties.BackendAddressPools +} + +func isBackendAddressPoolApplicable(backendPool *armnetwork.BackendAddressPool, _ int) bool { + if backendPool.Properties == nil || backendPool.Name == nil { + return false // shouldn't ever happen + } + + name := *backendPool.Name + // Ignore well-known named ipv6 pools for now + if strings.EqualFold(name, SLBOutboundBackendPoolNameIPv6) || strings.EqualFold(name, SLBInboundBackendPoolNameIPv6) { + return false + } + + // Ignore IP-based pools, which are a thing in NodeIP mode. We don't need to assign these pools. + // See isIPBasedBackendPool in RP. + for _, backendAddress := range backendPool.Properties.LoadBalancerBackendAddresses { + if backendAddress.Properties == nil || backendAddress.Properties.IPAddress == nil { + continue + } + + if *backendAddress.Properties.IPAddress != "" { + return false + } + } + + return true +} diff --git a/pkg/providers/loadbalancer/suite_test.go b/pkg/providers/loadbalancer/suite_test.go new file mode 100644 index 000000000..801b2e026 --- /dev/null +++ b/pkg/providers/loadbalancer/suite_test.go @@ -0,0 +1,133 @@ +/* +Portions Copyright (c) Microsoft Corporation. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package loadbalancer_test + +import ( + "context" + "testing" + "time" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "github.com/patrickmn/go-cache" + "github.com/samber/lo" + . "knative.dev/pkg/logging/testing" + + "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/network/armnetwork" + "github.com/Azure/karpenter/pkg/fake" + "github.com/Azure/karpenter/pkg/providers/loadbalancer" + "github.com/Azure/karpenter/pkg/test" +) + +var ctx context.Context +var stop context.CancelFunc + +var resourceGroup string +var fakeLoadBalancersAPI *fake.LoadBalancersAPI +var loadBalancerProvider *loadbalancer.Provider +var loadBalancerCache *cache.Cache + +func TestAKS(t *testing.T) { + ctx = TestContextWithLogger(t) + RegisterFailHandler(Fail) + RunSpecs(t, "Providers/LoadBalancer/AKS") +} + +var _ = BeforeSuite(func() { + ctx, stop = context.WithCancel(ctx) + + fakeLoadBalancersAPI = &fake.LoadBalancersAPI{} + resourceGroup = "test-rg" + loadBalancerCache = cache.New(time.Second, time.Second) + loadBalancerProvider = loadbalancer.NewProvider(fakeLoadBalancersAPI, loadBalancerCache, resourceGroup) +}) + +var _ = AfterSuite(func() { + stop() +}) + +var _ = BeforeEach(func() { + fakeLoadBalancersAPI.Reset() + loadBalancerCache.Flush() +}) + +var _ = Describe("LoadBalancer Provider", func() { + Context("Backend pools", func() { + It("should return only well-known loadbalancer pools", func() { + standardLB := test.MakeStandardLoadBalancer(resourceGroup, loadbalancer.SLBName, true) + internalLB := test.MakeStandardLoadBalancer(resourceGroup, loadbalancer.InternalSLBName, false) + otherLB := test.MakeStandardLoadBalancer(resourceGroup, "some-lb", true) + + fakeLoadBalancersAPI.LoadBalancers.Store(standardLB.ID, standardLB) + fakeLoadBalancersAPI.LoadBalancers.Store(internalLB.ID, internalLB) + fakeLoadBalancersAPI.LoadBalancers.Store(otherLB.ID, otherLB) + + pools, err := loadBalancerProvider.LoadBalancerBackendPools(ctx) + Expect(err).ToNot(HaveOccurred()) + + Expect(pools.IPv4PoolIDs).To(HaveLen(3)) + Expect(pools.IPv6PoolIDs).To(HaveLen(0)) + Expect(pools.IPv4PoolIDs[0]).To(Equal("/subscriptions/subscriptionID/resourceGroups/test-rg/providers/Microsoft.Network/loadBalancers/kubernetes/backendAddressPools/kubernetes")) + Expect(pools.IPv4PoolIDs[1]).To(Equal("/subscriptions/subscriptionID/resourceGroups/test-rg/providers/Microsoft.Network/loadBalancers/kubernetes/backendAddressPools/aksOutboundBackendPool")) + Expect(pools.IPv4PoolIDs[2]).To(Equal("/subscriptions/subscriptionID/resourceGroups/test-rg/providers/Microsoft.Network/loadBalancers/kubernetes-internal/backendAddressPools/kubernetes")) + }) + + It("should not return IPV6 pools", func() { + standardLB := test.MakeStandardLoadBalancer(resourceGroup, loadbalancer.SLBName, true) + internalLB := test.MakeStandardLoadBalancer(resourceGroup, loadbalancer.InternalSLBName, false) + otherLB := test.MakeStandardLoadBalancer(resourceGroup, "some-lb", true) + ipv6LB := test.MakeStandardLoadBalancer(resourceGroup, loadbalancer.SLBNameIPv6, true) + + fakeLoadBalancersAPI.LoadBalancers.Store(standardLB.ID, standardLB) + fakeLoadBalancersAPI.LoadBalancers.Store(internalLB.ID, internalLB) + fakeLoadBalancersAPI.LoadBalancers.Store(otherLB.ID, otherLB) + fakeLoadBalancersAPI.LoadBalancers.Store(ipv6LB.ID, ipv6LB) + + pools, err := loadBalancerProvider.LoadBalancerBackendPools(ctx) + Expect(err).ToNot(HaveOccurred()) + + Expect(pools.IPv4PoolIDs).To(HaveLen(3)) + Expect(pools.IPv6PoolIDs).To(HaveLen(0)) + Expect(pools.IPv4PoolIDs[0]).To(Equal("/subscriptions/subscriptionID/resourceGroups/test-rg/providers/Microsoft.Network/loadBalancers/kubernetes/backendAddressPools/kubernetes")) + Expect(pools.IPv4PoolIDs[1]).To(Equal("/subscriptions/subscriptionID/resourceGroups/test-rg/providers/Microsoft.Network/loadBalancers/kubernetes/backendAddressPools/aksOutboundBackendPool")) + Expect(pools.IPv4PoolIDs[2]).To(Equal("/subscriptions/subscriptionID/resourceGroups/test-rg/providers/Microsoft.Network/loadBalancers/kubernetes-internal/backendAddressPools/kubernetes")) + }) + + It("should not return IP-based pools", func() { + standardLB := test.MakeStandardLoadBalancer(resourceGroup, loadbalancer.SLBName, true) + standardLB.Properties.BackendAddressPools[1].Properties.LoadBalancerBackendAddresses = []*armnetwork.LoadBalancerBackendAddress{ + { + Properties: &armnetwork.LoadBalancerBackendAddressPropertiesFormat{ + IPAddress: lo.ToPtr("1.2.3.4"), + }, + }, + } + internalLB := test.MakeStandardLoadBalancer(resourceGroup, loadbalancer.InternalSLBName, false) + + fakeLoadBalancersAPI.LoadBalancers.Store(standardLB.ID, standardLB) + fakeLoadBalancersAPI.LoadBalancers.Store(internalLB.ID, internalLB) + + pools, err := loadBalancerProvider.LoadBalancerBackendPools(ctx) + Expect(err).ToNot(HaveOccurred()) + + Expect(pools.IPv4PoolIDs).To(HaveLen(2)) + Expect(pools.IPv6PoolIDs).To(HaveLen(0)) + Expect(pools.IPv4PoolIDs[0]).To(Equal("/subscriptions/subscriptionID/resourceGroups/test-rg/providers/Microsoft.Network/loadBalancers/kubernetes/backendAddressPools/kubernetes")) + Expect(pools.IPv4PoolIDs[1]).To(Equal("/subscriptions/subscriptionID/resourceGroups/test-rg/providers/Microsoft.Network/loadBalancers/kubernetes-internal/backendAddressPools/kubernetes")) + }) + }) +}) diff --git a/pkg/providers/pricing/client/filter.go b/pkg/providers/pricing/client/filter.go new file mode 100644 index 000000000..1736764f6 --- /dev/null +++ b/pkg/providers/pricing/client/filter.go @@ -0,0 +1,40 @@ +/* +Portions Copyright (c) Microsoft Corporation. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package client + +import ( + "fmt" +) + +type comparisonOperator string + +const ( + Equals comparisonOperator = "eq" +) + +// https://learn.microsoft.com/en-us/graph/filter-query-parameter?tabs=http +// +// Note: this is only for storing basic operators +type Filter struct { + Field string + Operator comparisonOperator + Value string +} + +func (f *Filter) String() string { + return fmt.Sprintf("%s %s '%s'", f.Field, f.Operator, f.Value) +} diff --git a/pkg/providers/pricing/client/pricingapi.go b/pkg/providers/pricing/client/pricingapi.go new file mode 100644 index 000000000..3b96807f0 --- /dev/null +++ b/pkg/providers/pricing/client/pricingapi.go @@ -0,0 +1,83 @@ +/* +Portions Copyright (c) Microsoft Corporation. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package client + +import ( + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "net/url" + "strings" +) + +const ( + apiVersion = "2021-10-01-preview" + pricingURL = "https://prices.azure.com/api/retail/prices?api-version=" + apiVersion +) + +type PricingAPI interface { + GetProductsPricePages(context.Context, []*Filter, func(output *ProductsPricePage)) error +} + +type pricingAPI struct{} + +func New() PricingAPI { + return &pricingAPI{} +} + +func (papi *pricingAPI) GetProductsPricePages(_ context.Context, filters []*Filter, pageHandler func(output *ProductsPricePage)) error { + nextURL := pricingURL + + if len(filters) > 0 { + filterParams := []string{} + for _, filter := range filters { + filterParams = append(filterParams, filter.String()) + } + + filterParamsEscaped := url.QueryEscape(strings.Join(filterParams[:], " and ")) + + nextURL += fmt.Sprintf("&$filter=%s", filterParamsEscaped) + } + + for nextURL != "" { + res, err := http.Get(nextURL) + if err != nil { + return err + } + + if res.StatusCode != 200 { + return fmt.Errorf("got a non-200 status code: %d", res.StatusCode) + } + + resBody, err := io.ReadAll(res.Body) + if err != nil { + return err + } + + page := ProductsPricePage{} + err = json.Unmarshal(resBody, &page) + if err != nil { + return err + } + + pageHandler(&page) + nextURL = page.NextPageLink + } + return nil +} diff --git a/pkg/providers/pricing/client/pricingapi_types.go b/pkg/providers/pricing/client/pricingapi_types.go new file mode 100644 index 000000000..5ef70bec9 --- /dev/null +++ b/pkg/providers/pricing/client/pricingapi_types.go @@ -0,0 +1,58 @@ +/* +Portions Copyright (c) Microsoft Corporation. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Generated structs based off of the json retrieved from the https://prices.azure.com/api/retail/prices API + +package client + +import ( + "time" +) + +type Item struct { + CurrencyCode string `json:"currencyCode"` + TierMinimumUnits float64 `json:"tierMinimumUnits"` + RetailPrice float64 `json:"retailPrice"` + UnitPrice float64 `json:"unitPrice"` + ArmRegionName string `json:"armRegionName"` + Location string `json:"location"` + EffectiveStartDate time.Time `json:"effectiveStartDate"` + MeterID string `json:"meterId"` + MeterName string `json:"meterName"` + ProductID string `json:"productId"` + SkuID string `json:"skuId"` + AvailabilityID any `json:"availabilityId"` + ProductName string `json:"productName"` + SkuName string `json:"skuName"` + ServiceName string `json:"serviceName"` + ServiceID string `json:"serviceId"` + ServiceFamily string `json:"serviceFamily"` + UnitOfMeasure string `json:"unitOfMeasure"` + Type string `json:"type"` + IsPrimaryMeterRegion bool `json:"isPrimaryMeterRegion"` + ArmSkuName string `json:"armSkuName"` + EffectiveEndDate time.Time `json:"effectiveEndDate,omitempty"` + ReservationTerm string `json:"reservationTerm,omitempty"` +} + +type ProductsPricePage struct { + BillingCurrency string `json:"BillingCurrency"` + CustomerEntityID string `json:"CustomerEntityId"` + CustomerEntityType string `json:"CustomerEntityType"` + Items []Item `json:"Items"` + NextPageLink string `json:"NextPageLink"` + Count int `json:"Count"` +} diff --git a/pkg/providers/pricing/pricing.go b/pkg/providers/pricing/pricing.go new file mode 100644 index 000000000..373ad6079 --- /dev/null +++ b/pkg/providers/pricing/pricing.go @@ -0,0 +1,333 @@ +/* +Portions Copyright (c) Microsoft Corporation. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package pricing + +import ( + "context" + "errors" + "net/http" + "strings" + "sync" + "time" + + "github.com/Azure/karpenter/pkg/providers/pricing/client" + "github.com/aws/karpenter-core/pkg/utils/pretty" + "github.com/samber/lo" + "knative.dev/pkg/logging" +) + +// pricingUpdatePeriod is how often we try to update our pricing information after the initial update on startup +const pricingUpdatePeriod = 12 * time.Hour + +const defaultRegion = "eastus" + +// Provider provides actual pricing data to the Azure cloud provider to allow it to make more informed decisions +// regarding which instances to launch. This is initialized at startup with a periodically updated static price list to +// support running in locations where pricing data is unavailable. In those cases the static pricing data provides a +// relative ordering that is still more accurate than our previous pricing model. In the event that a pricing update +// fails, the previous pricing information is retained and used which may be the static initial pricing data if pricing +// updates never succeed. +type Provider struct { + pricing client.PricingAPI + region string + cm *pretty.ChangeMonitor + + mu sync.RWMutex + onDemandUpdateTime time.Time + onDemandPrices map[string]float64 + spotUpdateTime time.Time + spotPrices map[string]float64 +} + +type Err struct { + error + lastUpdateTime time.Time +} + +// NewPricingAPI returns a pricing API +func NewAPI() client.PricingAPI { + return client.New() +} + +func NewProvider(ctx context.Context, pricing client.PricingAPI, region string, startAsync <-chan struct{}) *Provider { + // see if we've got region specific pricing data + staticPricing, ok := initialOnDemandPrices[region] + if !ok { + // and if not, fall back to the always available eastus + staticPricing = initialOnDemandPrices[defaultRegion] + } + + p := &Provider{ + region: region, + onDemandUpdateTime: initialPriceUpdate, + onDemandPrices: staticPricing, + spotUpdateTime: initialPriceUpdate, + // default our spot pricing to the same as the on-demand pricing until a price update + spotPrices: staticPricing, + pricing: pricing, + cm: pretty.NewChangeMonitor(), + } + ctx = logging.WithLogger(ctx, logging.FromContext(ctx).Named("pricing")) + + go func() { + // perform an initial price update at startup + p.updatePricing(ctx) + + startup := time.Now() + // wait for leader election or to be signaled to exit + select { + case <-startAsync: + case <-ctx.Done(): + return + } + // if it took many hours to be elected leader, we want to re-fetch pricing before we start our periodic + // polling + if time.Since(startup) > pricingUpdatePeriod { + p.updatePricing(ctx) + } + + for { + select { + case <-ctx.Done(): + return + case <-time.After(pricingUpdatePeriod): + p.updatePricing(ctx) + } + } + }() + return p +} + +// InstanceTypes returns the list of all instance types for which either a price is known. +func (p *Provider) InstanceTypes() []string { + p.mu.RLock() + defer p.mu.RUnlock() + return lo.Union(lo.Keys(p.onDemandPrices), lo.Keys(p.spotPrices)) +} + +// OnDemandLastUpdated returns the time that the on-demand pricing was last updated +func (p *Provider) OnDemandLastUpdated() time.Time { + p.mu.RLock() + defer p.mu.RUnlock() + return p.onDemandUpdateTime +} + +// SpotLastUpdated returns the time that the spot pricing was last updated +func (p *Provider) SpotLastUpdated() time.Time { + p.mu.RLock() + defer p.mu.RUnlock() + return p.spotUpdateTime +} + +// OnDemandPrice returns the last known on-demand price for a given instance type, returning false if there is no +// known on-demand pricing for the instance type. +func (p *Provider) OnDemandPrice(instanceType string) (float64, bool) { + p.mu.RLock() + defer p.mu.RUnlock() + price, ok := p.onDemandPrices[instanceType] + if !ok { + return 0.0, false + } + return price, true +} + +// SpotPrice returns the last known spot price for a given instance type, returning false +// if there is no known spot pricing for that instance type +func (p *Provider) SpotPrice(instanceType string) (float64, bool) { + p.mu.RLock() + defer p.mu.RUnlock() + price, ok := p.spotPrices[instanceType] + if !ok { + return 0.0, false + } + return price, true +} + +func (p *Provider) updatePricing(ctx context.Context) { + var wg sync.WaitGroup + wg.Add(1) + go func() { + defer wg.Done() + if err := p.UpdateOnDemandPricing(ctx); err != nil { + logging.FromContext(ctx).Errorf("error updating on-demand pricing for region %s, %s, using existing pricing data from %s", p.region, err, err.lastUpdateTime.Format(time.RFC3339)) + } + }() + + wg.Add(1) + go func() { + defer wg.Done() + if err := p.UpdateSpotPricing(ctx); err != nil { + logging.FromContext(ctx).Errorf("error updating spot pricing for region %s, %s, using existing pricing data from %s", p.region, err, err.lastUpdateTime.Format(time.RFC3339)) + } + }() + + wg.Wait() +} + +func (p *Provider) UpdateOnDemandPricing(ctx context.Context) *Err { + // standard on-demand instances + var wg sync.WaitGroup + var onDemandPrices = map[string]float64{} + var onDemandErr error + + wg.Add(1) + go func() { + defer wg.Done() + onDemandErr = p.fetchPricing(ctx, onDemandPage(onDemandPrices)) + }() + + wg.Wait() + + p.mu.Lock() + defer p.mu.Unlock() + err := onDemandErr + if err != nil { + return &Err{error: err, lastUpdateTime: p.onDemandUpdateTime} + } + + if len(onDemandPrices) == 0 { + return &Err{error: errors.New("no on-demand pricing found"), lastUpdateTime: p.onDemandUpdateTime} + } + + p.onDemandPrices = lo.Assign(onDemandPrices) + p.onDemandUpdateTime = time.Now() + if p.cm.HasChanged("on-demand-prices", p.onDemandPrices) { + logging.FromContext(ctx).With("instance-type-count", len(p.onDemandPrices)).Infof("updated on-demand pricing for region %s", p.region) + } + return nil +} + +func (p *Provider) fetchPricing(ctx context.Context, pageHandler func(output *client.ProductsPricePage)) error { + filters := []*client.Filter{ + { + Field: "priceType", + Operator: client.Equals, + Value: "Consumption", + }, + { + Field: "currencyCode", + Operator: client.Equals, + Value: "USD", + }, + { + Field: "serviceFamily", + Operator: client.Equals, + Value: "Compute", + }, + { + Field: "serviceName", + Operator: client.Equals, + Value: "Virtual Machines", + }, + { + Field: "armRegionName", + Operator: client.Equals, + Value: p.region, + }} + return p.pricing.GetProductsPricePages(ctx, filters, pageHandler) +} + +func onDemandPage(prices map[string]float64) func(page *client.ProductsPricePage) { + return func(page *client.ProductsPricePage) { + for _, pItem := range page.Items { + if strings.HasSuffix(pItem.ProductName, " Windows") { + continue + } + if strings.HasSuffix(pItem.MeterName, " Low Priority") { + // https://learn.microsoft.com/en-us/azure/batch/batch-spot-vms#differences-between-spot-and-low-priority-vms + continue + } + if strings.HasSuffix(pItem.SkuName, " Spot") { + continue + } + prices[pItem.ArmSkuName] = pItem.RetailPrice + } + } +} + +func (p *Provider) UpdateSpotPricing(ctx context.Context) *Err { + // standard on-demand instances + var wg sync.WaitGroup + var spotPrices = map[string]float64{} + var spotErr error + + wg.Add(1) + go func() { + defer wg.Done() + spotErr = p.fetchPricing(ctx, spotPage(spotPrices)) + }() + + wg.Wait() + + p.mu.Lock() + defer p.mu.Unlock() + err := spotErr + if err != nil { + return &Err{error: err, lastUpdateTime: p.spotUpdateTime} + } + + if len(spotPrices) == 0 { + return &Err{error: errors.New("no spot pricing found"), lastUpdateTime: p.spotUpdateTime} + } + + p.spotPrices = lo.Assign(spotPrices) + p.spotUpdateTime = time.Now() + if p.cm.HasChanged("spot-prices", p.spotPrices) { + logging.FromContext(ctx).With("instance-type-count", len(p.spotPrices)).Infof("updated spot pricing for region %s", p.region) + } + return nil +} + +func spotPage(prices map[string]float64) func(page *client.ProductsPricePage) { + return func(page *client.ProductsPricePage) { + for _, pItem := range page.Items { + if strings.HasSuffix(pItem.ProductName, " Windows") { + continue + } + if strings.HasSuffix(pItem.MeterName, " Low Priority") { + // https://learn.microsoft.com/en-us/azure/batch/batch-spot-vms#differences-between-spot-and-low-priority-vms + continue + } + if !strings.HasSuffix(pItem.SkuName, " Spot") { + continue + } + prices[pItem.ArmSkuName] = pItem.RetailPrice + } + } +} + +func (p *Provider) LivenessProbe(_ *http.Request) error { + // ensure we don't deadlock and nolint for the empty critical section + p.mu.Lock() + //nolint: staticcheck + p.mu.Unlock() + return nil +} + +func (p *Provider) Reset() { + // see if we've got region specific pricing data + staticPricing, ok := initialOnDemandPrices[p.region] + if !ok { + // and if not, fall back to the always available eastus + staticPricing = initialOnDemandPrices[defaultRegion] + } + + p.mu.Lock() + defer p.mu.Unlock() + p.onDemandPrices = staticPricing + p.onDemandUpdateTime = initialPriceUpdate +} diff --git a/pkg/providers/pricing/suite_test.go b/pkg/providers/pricing/suite_test.go new file mode 100644 index 000000000..8eef6f940 --- /dev/null +++ b/pkg/providers/pricing/suite_test.go @@ -0,0 +1,113 @@ +/* +Portions Copyright (c) Microsoft Corporation. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package pricing_test + +import ( + "context" + "fmt" + "testing" + "time" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + . "knative.dev/pkg/logging/testing" + + "github.com/Azure/karpenter/pkg/fake" + "github.com/Azure/karpenter/pkg/providers/pricing" + "github.com/Azure/karpenter/pkg/providers/pricing/client" +) + +var ctx context.Context +var stop context.CancelFunc + +var fakePricingAPI *fake.PricingAPI +var pricingProvider *pricing.Provider + +func TestAzure(t *testing.T) { + ctx = TestContextWithLogger(t) + RegisterFailHandler(Fail) + RunSpecs(t, "Providers/Pricing/Azure") +} + +var _ = BeforeSuite(func() { + ctx, stop = context.WithCancel(ctx) + + fakePricingAPI = &fake.PricingAPI{} + pricingProvider = pricing.NewProvider(ctx, fakePricingAPI, "", make(chan struct{})) +}) + +var _ = AfterSuite(func() { + stop() +}) + +var _ = BeforeEach(func() { + fakePricingAPI.Reset() +}) + +var _ = Describe("Pricing", func() { + BeforeEach(func() { + fakePricingAPI.Reset() + }) + It("should return static on-demand data if pricing API fails", func() { + fakePricingAPI.NextError.Set(fmt.Errorf("failed")) + p := pricing.NewProvider(ctx, fakePricingAPI, "", make(chan struct{})) + price, ok := p.OnDemandPrice("Standard_D1") + Expect(ok).To(BeTrue()) + Expect(price).To(BeNumerically(">", 0)) + }) + It("should update on-demand pricing with response from the pricing API", func() { + // modify our API before creating the pricing provider as it performs an initial update on creation. + fakePricingAPI.ProductsPricePage.Set(&client.ProductsPricePage{ + Items: []client.Item{ + fake.NewProductPrice("Standard_D1", 1.20), + fake.NewProductPrice("Standard_D14", 1.23), + }, + }) + updateStart := time.Now() + p := pricing.NewProvider(ctx, fakePricingAPI, "", make(chan struct{})) + Eventually(func() bool { return p.OnDemandLastUpdated().After(updateStart) }).Should(BeTrue()) + + price, ok := p.OnDemandPrice("Standard_D1") + Expect(ok).To(BeTrue()) + Expect(price).To(BeNumerically("==", 1.20)) + + price, ok = p.OnDemandPrice("Standard_D14") + Expect(ok).To(BeTrue()) + Expect(price).To(BeNumerically("==", 1.23)) + }) + + It("should update spot pricing with response from the pricing API", func() { + // modify our API before creating the pricing provider as it performs an initial update on creation. + fakePricingAPI.ProductsPricePage.Set(&client.ProductsPricePage{ + Items: []client.Item{ + fake.NewSpotProductPrice("Standard_D1", 1.10), + fake.NewSpotProductPrice("Standard_D14", 1.13), + }, + }) + updateStart := time.Now() + p := pricing.NewProvider(ctx, fakePricingAPI, "", make(chan struct{})) + Eventually(func() bool { return p.SpotLastUpdated().After(updateStart) }).Should(BeTrue()) + + price, ok := p.SpotPrice("Standard_D1") + Expect(ok).To(BeTrue()) + Expect(price).To(BeNumerically("==", 1.10)) + + price, ok = p.SpotPrice("Standard_D14") + Expect(ok).To(BeTrue()) + Expect(price).To(BeNumerically("==", 1.13)) + }) +}) diff --git a/pkg/providers/pricing/zz_generated.pricing.go b/pkg/providers/pricing/zz_generated.pricing.go new file mode 100644 index 000000000..4a91d613e --- /dev/null +++ b/pkg/providers/pricing/zz_generated.pricing.go @@ -0,0 +1,30566 @@ +//go:build !ignore_autogenerated + +/* +Portions Copyright (c) Microsoft Corporation. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package pricing + +import "time" + +// generated at 2023-10-27T06:44:32Z + +var initialPriceUpdate, _ = time.Parse(time.RFC3339, "2023-10-27T06:44:32Z") +var initialOnDemandPrices = map[string]map[string]float64{} + +func init() { + // australiacentral + initialOnDemandPrices["australiacentral"] = map[string]float64{ + "Basic_A0": 0.024000, + "Basic_A1": 0.032000, + "Basic_A2": 0.098000, + "Basic_A3": 0.232000, + "Basic_A4": 0.464000, + "Dadsv5_Type1": 8.008000, + "Dasv4_Type1": 6.600000, + "Dasv4_Type2": 6.600000, + "Dasv5_Type1": 6.653000, + "Ddsv4_Type 1": 4.998000, + "Ddsv4_Type2": 5.936000, + "Ddsv5_Type1": 7.498000, + "Dsv3_Type3": 6.599000, + "Dsv3_Type4": 8.249000, + "Dsv4_Type1": 5.280000, + "Dsv4_Type2": 6.336000, + "Dsv5_Type1": 6.600000, + "Eadsv5_Type1": 8.395000, + "Easv4_Type1": 7.973000, + "Easv4_Type2": 7.973000, + "Easv5_Type1": 7.181000, + "Ebdsv5-Type1": 7.075000, + "Ebsv5-Type1": 6.266000, + "Edsv4_Type 1": 6.125000, + "Edsv4_Type2": 8.736000, + "Edsv5_Type1": 9.187000, + "Esv3_Type3": 6.434000, + "Esv3_Type4": 8.717000, + "Esv4_Type1": 5.315000, + "Esv4_Type2": 6.976000, + "Esv5_Type1": 7.973000, + "Fsv2 Type3": 5.128000, + "Fsv2_Type2": 4.396000, + "Fsv2_Type4": 5.861000, + "Lasv3_Type1": 8.228000, + "Lsv3_Type1": 9.196000, + "Msmv2_Type1": 158.140000, + "Msv2_Type1": 79.080000, + "Standard_A0": 0.029000, + "Standard_A1": 0.071000, + "Standard_A1_v2": 0.050000, + "Standard_A2": 0.142000, + "Standard_A2_v2": 0.106000, + "Standard_A2m_v2": 0.149000, + "Standard_A3": 0.284000, + "Standard_A4": 0.568000, + "Standard_A4_v2": 0.222000, + "Standard_A4m_v2": 0.312000, + "Standard_A5": 0.278000, + "Standard_A6": 0.556000, + "Standard_A7": 1.112000, + "Standard_A8_v2": 0.467000, + "Standard_A8m_v2": 0.655000, + "Standard_B12ms": 0.634000, + "Standard_B16als_v2": 0.747000, + "Standard_B16as_v2": 0.834000, + "Standard_B16ls_v2": 0.748000, + "Standard_B16ms": 0.845000, + "Standard_B16s_v2": 0.918000, + "Standard_B1ls": 0.006600, + "Standard_B1ms": 0.026400, + "Standard_B1s": 0.013200, + "Standard_B20ms": 1.056000, + "Standard_B2als_v2": 0.047500, + "Standard_B2as_v2": 0.104000, + "Standard_B2ats_v2": 0.011900, + "Standard_B2ls_v2": 0.062000, + "Standard_B2ms": 0.106000, + "Standard_B2s": 0.052800, + "Standard_B2s_v2": 0.106000, + "Standard_B2ts_v2": 0.013200, + "Standard_B32als_v2": 1.346000, + "Standard_B32as_v2": 1.520000, + "Standard_B32ls_v2": 1.496000, + "Standard_B32s_v2": 1.837000, + "Standard_B4als_v2": 0.187000, + "Standard_B4as_v2": 0.190000, + "Standard_B4ls_v2": 0.187000, + "Standard_B4ms": 0.211000, + "Standard_B4s_v2": 0.211000, + "Standard_B8als_v2": 0.337000, + "Standard_B8as_v2": 0.417000, + "Standard_B8ls_v2": 0.411000, + "Standard_B8ms": 0.422000, + "Standard_B8s_v2": 0.459000, + "Standard_D11_v2": 0.200000, + "Standard_D11_v2_Promo": 0.249000, + "Standard_D12_v2": 0.399000, + "Standard_D12_v2_Promo": 0.498750, + "Standard_D13_v2": 0.798000, + "Standard_D13_v2_Promo": 0.997000, + "Standard_D14_v2": 1.596000, + "Standard_D14_v2_Promo": 1.995000, + "Standard_D15_v2": 1.995000, + "Standard_D15i_v2": 1.995000, + "Standard_D16_v3": 1.000000, + "Standard_D16_v4": 0.960000, + "Standard_D16_v5": 0.960000, + "Standard_D16a_v4": 0.960000, + "Standard_D16ads_v5": 1.040000, + "Standard_D16as_v4": 0.960000, + "Standard_D16as_v5": 1.600000, + "Standard_D16d_v4": 1.136000, + "Standard_D16d_v5": 1.136000, + "Standard_D16ds_v4": 1.136000, + "Standard_D16ds_v5": 1.136000, + "Standard_D16lds_v5": 1.008000, + "Standard_D16ls_v5": 0.888000, + "Standard_D16s_v3": 1.000000, + "Standard_D16s_v4": 0.960000, + "Standard_D16s_v5": 0.960000, + "Standard_D1_v2": 0.084000, + "Standard_D2_v2": 0.168000, + "Standard_D2_v2_Promo": 0.210000, + "Standard_D2_v3": 0.125000, + "Standard_D2_v4": 0.120000, + "Standard_D2_v5": 0.120000, + "Standard_D2a_v4": 0.120000, + "Standard_D2ads_v5": 0.130000, + "Standard_D2as_v4": 0.120000, + "Standard_D2as_v5": 0.108000, + "Standard_D2d_v4": 0.142000, + "Standard_D2d_v5": 0.142000, + "Standard_D2ds_v4": 0.142000, + "Standard_D2ds_v5": 0.142000, + "Standard_D2lds_v5": 0.126000, + "Standard_D2ls_v5": 0.111000, + "Standard_D2s_v3": 0.125000, + "Standard_D2s_v4": 0.120000, + "Standard_D2s_v5": 0.120000, + "Standard_D32-16s_v3": 2.000000, + "Standard_D32-8s_v3": 2.000000, + "Standard_D32_v3": 2.000000, + "Standard_D32_v4": 1.920000, + "Standard_D32_v5": 1.920000, + "Standard_D32a_v4": 1.920000, + "Standard_D32ads_v5": 3.552000, + "Standard_D32as_v4": 1.920000, + "Standard_D32as_v5": 3.200000, + "Standard_D32d_v4": 2.272000, + "Standard_D32d_v5": 2.272000, + "Standard_D32ds_v4": 2.272000, + "Standard_D32ds_v5": 2.272000, + "Standard_D32lds_v5": 2.016000, + "Standard_D32ls_v5": 1.776000, + "Standard_D32s_v3": 2.000000, + "Standard_D32s_v4": 1.920000, + "Standard_D32s_v5": 1.920000, + "Standard_D3_v2": 0.336000, + "Standard_D3_v2_Promo": 0.420000, + "Standard_D48_v3": 3.000000, + "Standard_D48_v4": 2.880000, + "Standard_D48_v5": 2.880000, + "Standard_D48a_v4": 2.880000, + "Standard_D48ads_v5": 5.328000, + "Standard_D48as_v4": 2.880000, + "Standard_D48as_v5": 2.592000, + "Standard_D48d_v4": 3.408000, + "Standard_D48d_v5": 3.408000, + "Standard_D48ds_v4": 3.408000, + "Standard_D48ds_v5": 3.408000, + "Standard_D48lds_v5": 3.024000, + "Standard_D48ls_v5": 2.664000, + "Standard_D48s_v3": 3.000000, + "Standard_D48s_v4": 2.880000, + "Standard_D48s_v5": 2.880000, + "Standard_D4_v2": 0.673000, + "Standard_D4_v2_Promo": 0.841000, + "Standard_D4_v3": 0.250000, + "Standard_D4_v4": 0.240000, + "Standard_D4_v5": 0.240000, + "Standard_D4a_v4": 0.240000, + "Standard_D4ads_v5": 0.260000, + "Standard_D4as_v4": 0.240000, + "Standard_D4as_v5": 0.400000, + "Standard_D4d_v4": 0.284000, + "Standard_D4d_v5": 0.284000, + "Standard_D4ds_v4": 0.284000, + "Standard_D4ds_v5": 0.284000, + "Standard_D4lds_v5": 0.252000, + "Standard_D4ls_v5": 0.222000, + "Standard_D4s_v3": 0.250000, + "Standard_D4s_v4": 0.240000, + "Standard_D4s_v5": 0.240000, + "Standard_D5_v2": 1.345000, + "Standard_D5_v2_Promo": 1.681000, + "Standard_D64-16s_v3": 4.000000, + "Standard_D64-32s_v3": 4.000000, + "Standard_D64_v3": 4.000000, + "Standard_D64_v4": 3.840000, + "Standard_D64_v5": 3.840000, + "Standard_D64a_v4": 3.840000, + "Standard_D64ads_v5": 4.160000, + "Standard_D64as_v4": 3.840000, + "Standard_D64as_v5": 6.400000, + "Standard_D64d_v4": 4.544000, + "Standard_D64d_v5": 4.544000, + "Standard_D64ds_v4": 4.544000, + "Standard_D64ds_v5": 4.544000, + "Standard_D64lds_v5": 4.032000, + "Standard_D64ls_v5": 3.552000, + "Standard_D64s_v3": 4.000000, + "Standard_D64s_v4": 3.840000, + "Standard_D64s_v5": 3.840000, + "Standard_D8_v3": 0.500000, + "Standard_D8_v4": 0.480000, + "Standard_D8_v5": 0.480000, + "Standard_D8a_v4": 0.480000, + "Standard_D8ads_v5": 0.888000, + "Standard_D8as_v4": 0.480000, + "Standard_D8as_v5": 0.800000, + "Standard_D8d_v4": 0.568000, + "Standard_D8d_v5": 0.568000, + "Standard_D8ds_v4": 0.568000, + "Standard_D8ds_v5": 0.568000, + "Standard_D8lds_v5": 0.504000, + "Standard_D8ls_v5": 0.444000, + "Standard_D8s_v3": 0.500000, + "Standard_D8s_v4": 0.480000, + "Standard_D8s_v5": 0.480000, + "Standard_D96_v5": 5.760000, + "Standard_D96a_v4": 5.760000, + "Standard_D96ads_v5": 10.656000, + "Standard_D96as_v4": 5.760000, + "Standard_D96as_v5": 5.184000, + "Standard_D96d_v5": 6.816000, + "Standard_D96ds_v5": 6.816000, + "Standard_D96lds_v5": 6.048000, + "Standard_D96ls_v5": 5.328000, + "Standard_D96s_v5": 5.760000, + "Standard_DS11-1_v2": 0.200000, + "Standard_DS11_v2": 0.200000, + "Standard_DS11_v2_Promo": 0.249000, + "Standard_DS12-1_v2": 0.399000, + "Standard_DS12-2_v2": 0.399000, + "Standard_DS12_v2": 0.399000, + "Standard_DS12_v2_Promo": 0.498750, + "Standard_DS13-2_v2": 0.798000, + "Standard_DS13-4_v2": 0.798000, + "Standard_DS13_v2": 0.798000, + "Standard_DS13_v2_Promo": 0.997000, + "Standard_DS14-4_v2": 1.596000, + "Standard_DS14-8_v2": 1.596000, + "Standard_DS14_v2": 1.596000, + "Standard_DS14_v2_Promo": 1.995000, + "Standard_DS15_v2": 1.995000, + "Standard_DS15i_v2": 1.995000, + "Standard_DS1_v2": 0.084000, + "Standard_DS2_v2": 0.168000, + "Standard_DS2_v2_Promo": 0.210000, + "Standard_DS3_v2": 0.336000, + "Standard_DS3_v2_Promo": 0.420000, + "Standard_DS4_v2": 0.673000, + "Standard_DS4_v2_Promo": 0.841000, + "Standard_DS5_v2": 1.345000, + "Standard_DS5_v2_Promo": 1.681000, + "Standard_E104i_v5": 8.637000, + "Standard_E104id_v5": 9.953000, + "Standard_E104ids_v5": 9.953000, + "Standard_E104is_v5": 8.637000, + "Standard_E112iads_v5": 9.794000, + "Standard_E112ias_v5": 8.378000, + "Standard_E112ibds_v5": 12.382000, + "Standard_E112ibs_v5": 10.965000, + "Standard_E16-4ads_v5": 1.272000, + "Standard_E16-4as_v4": 1.208000, + "Standard_E16-4as_v5": 1.824000, + "Standard_E16-4ds_v4": 1.392000, + "Standard_E16-4ds_v5": 1.392000, + "Standard_E16-4s_v3": 1.277000, + "Standard_E16-4s_v4": 1.208000, + "Standard_E16-4s_v5": 1.208000, + "Standard_E16-8ads_v5": 2.008000, + "Standard_E16-8as_v4": 1.208000, + "Standard_E16-8as_v5": 1.088000, + "Standard_E16-8ds_v4": 1.392000, + "Standard_E16-8ds_v5": 1.392000, + "Standard_E16-8s_v3": 1.277000, + "Standard_E16-8s_v4": 1.208000, + "Standard_E16-8s_v5": 1.208000, + "Standard_E16_v3": 1.277000, + "Standard_E16_v4": 1.208000, + "Standard_E16_v5": 1.208000, + "Standard_E16a_v4": 1.208000, + "Standard_E16ads_v5": 1.318000, + "Standard_E16as_v4": 1.208000, + "Standard_E16as_v5": 1.088000, + "Standard_E16bds_v5": 1.608000, + "Standard_E16bs_v5": 1.424000, + "Standard_E16d_v4": 1.392000, + "Standard_E16d_v5": 1.392000, + "Standard_E16ds_v4": 1.392000, + "Standard_E16ds_v5": 1.392000, + "Standard_E16s_v3": 1.277000, + "Standard_E16s_v4": 1.208000, + "Standard_E16s_v5": 1.208000, + "Standard_E20_v3": 1.509000, + "Standard_E20_v4": 1.510000, + "Standard_E20_v5": 1.510000, + "Standard_E20a_v4": 1.510000, + "Standard_E20ads_v5": 1.636000, + "Standard_E20as_v4": 1.510000, + "Standard_E20as_v5": 1.406000, + "Standard_E20d_v4": 1.740000, + "Standard_E20d_v5": 1.740000, + "Standard_E20ds_v4": 1.740000, + "Standard_E20ds_v5": 1.740000, + "Standard_E20s_v3": 1.509000, + "Standard_E20s_v4": 1.510000, + "Standard_E20s_v5": 1.510000, + "Standard_E2_v3": 0.160000, + "Standard_E2_v4": 0.151000, + "Standard_E2_v5": 0.151000, + "Standard_E2a_v4": 0.151000, + "Standard_E2ads_v5": 0.159000, + "Standard_E2as_v4": 0.151000, + "Standard_E2as_v5": 0.136000, + "Standard_E2bds_v5": 0.201000, + "Standard_E2bs_v5": 0.178000, + "Standard_E2d_v4": 0.174000, + "Standard_E2d_v5": 0.174000, + "Standard_E2ds_v4": 0.174000, + "Standard_E2ds_v5": 0.174000, + "Standard_E2s_v3": 0.160000, + "Standard_E2s_v4": 0.151000, + "Standard_E2s_v5": 0.151000, + "Standard_E32-16ads_v5": 2.544000, + "Standard_E32-16as_v4": 2.416000, + "Standard_E32-16as_v5": 2.176000, + "Standard_E32-16ds_v4": 2.784000, + "Standard_E32-16ds_v5": 2.784000, + "Standard_E32-16s_v3": 2.554000, + "Standard_E32-16s_v4": 2.416000, + "Standard_E32-16s_v5": 2.416000, + "Standard_E32-8ads_v5": 4.016000, + "Standard_E32-8as_v4": 2.416000, + "Standard_E32-8as_v5": 3.648000, + "Standard_E32-8ds_v4": 2.784000, + "Standard_E32-8ds_v5": 2.784000, + "Standard_E32-8s_v3": 2.554000, + "Standard_E32-8s_v4": 2.416000, + "Standard_E32-8s_v5": 2.416000, + "Standard_E32_v3": 2.554000, + "Standard_E32_v4": 2.416000, + "Standard_E32_v5": 2.416000, + "Standard_E32a_v4": 2.416000, + "Standard_E32ads_v5": 2.544000, + "Standard_E32as_v4": 2.416000, + "Standard_E32as_v5": 2.176000, + "Standard_E32bds_v5": 3.216000, + "Standard_E32bs_v5": 2.848000, + "Standard_E32d_v4": 2.784000, + "Standard_E32d_v5": 2.784000, + "Standard_E32ds_v4": 2.784000, + "Standard_E32ds_v5": 2.784000, + "Standard_E32s_v3": 2.554000, + "Standard_E32s_v4": 2.416000, + "Standard_E32s_v5": 2.416000, + "Standard_E4-2ads_v5": 0.502000, + "Standard_E4-2as_v4": 0.302000, + "Standard_E4-2as_v5": 0.456000, + "Standard_E4-2ds_v4": 0.348000, + "Standard_E4-2ds_v5": 0.348000, + "Standard_E4-2s_v3": 0.319000, + "Standard_E4-2s_v4": 0.302000, + "Standard_E4-2s_v5": 0.302000, + "Standard_E48_v3": 3.623000, + "Standard_E48_v4": 3.624000, + "Standard_E48_v5": 3.624000, + "Standard_E48a_v4": 3.624000, + "Standard_E48ads_v5": 3.862000, + "Standard_E48as_v4": 3.624000, + "Standard_E48as_v5": 3.264000, + "Standard_E48bds_v5": 4.824000, + "Standard_E48bs_v5": 4.272000, + "Standard_E48d_v4": 4.176000, + "Standard_E48d_v5": 4.176000, + "Standard_E48ds_v4": 4.176000, + "Standard_E48ds_v5": 4.176000, + "Standard_E48s_v3": 3.623000, + "Standard_E48s_v4": 3.624000, + "Standard_E48s_v5": 3.624000, + "Standard_E4_v3": 0.319000, + "Standard_E4_v4": 0.302000, + "Standard_E4_v5": 0.302000, + "Standard_E4a_v4": 0.302000, + "Standard_E4ads_v5": 0.364000, + "Standard_E4as_v4": 0.302000, + "Standard_E4as_v5": 0.318000, + "Standard_E4bds_v5": 0.402000, + "Standard_E4bs_v5": 0.356000, + "Standard_E4d_v4": 0.348000, + "Standard_E4d_v5": 0.348000, + "Standard_E4ds_v4": 0.348000, + "Standard_E4ds_v5": 0.348000, + "Standard_E4s_v3": 0.319000, + "Standard_E4s_v4": 0.302000, + "Standard_E4s_v5": 0.302000, + "Standard_E64-16ads_v5": 5.088000, + "Standard_E64-16as_v4": 4.832000, + "Standard_E64-16as_v5": 4.352000, + "Standard_E64-16ds_v4": 5.568000, + "Standard_E64-16ds_v5": 5.568000, + "Standard_E64-16s_v3": 4.347000, + "Standard_E64-16s_v4": 4.832000, + "Standard_E64-16s_v5": 4.832000, + "Standard_E64-32ads_v5": 8.032000, + "Standard_E64-32as_v4": 4.832000, + "Standard_E64-32as_v5": 7.296000, + "Standard_E64-32ds_v4": 5.568000, + "Standard_E64-32ds_v5": 5.568000, + "Standard_E64-32s_v3": 4.347000, + "Standard_E64-32s_v4": 4.832000, + "Standard_E64-32s_v5": 4.832000, + "Standard_E64_v3": 4.347000, + "Standard_E64_v4": 4.832000, + "Standard_E64_v5": 4.832000, + "Standard_E64a_v4": 4.832000, + "Standard_E64ads_v5": 5.088000, + "Standard_E64as_v4": 4.832000, + "Standard_E64as_v5": 4.352000, + "Standard_E64bds_v5": 6.432000, + "Standard_E64bs_v5": 5.696000, + "Standard_E64d_v4": 5.568000, + "Standard_E64d_v5": 5.568000, + "Standard_E64ds_v4": 5.568000, + "Standard_E64ds_v5": 5.568000, + "Standard_E64s_v3": 4.347000, + "Standard_E64s_v4": 4.832000, + "Standard_E64s_v5": 4.832000, + "Standard_E8-2ads_v5": 0.636000, + "Standard_E8-2as_v4": 0.604000, + "Standard_E8-2as_v5": 0.544000, + "Standard_E8-2ds_v4": 0.696000, + "Standard_E8-2ds_v5": 0.696000, + "Standard_E8-2s_v3": 0.638000, + "Standard_E8-2s_v4": 0.604000, + "Standard_E8-2s_v5": 0.604000, + "Standard_E8-4ads_v5": 1.004000, + "Standard_E8-4as_v4": 0.604000, + "Standard_E8-4as_v5": 0.912000, + "Standard_E8-4ds_v4": 0.696000, + "Standard_E8-4ds_v5": 0.696000, + "Standard_E8-4s_v3": 0.638000, + "Standard_E8-4s_v4": 0.604000, + "Standard_E8-4s_v5": 0.604000, + "Standard_E80ids_v4": 6.960000, + "Standard_E80is_v4": 6.040000, + "Standard_E8_v3": 0.638000, + "Standard_E8_v4": 0.604000, + "Standard_E8_v5": 0.604000, + "Standard_E8a_v4": 0.604000, + "Standard_E8ads_v5": 0.682000, + "Standard_E8as_v4": 0.604000, + "Standard_E8as_v5": 0.544000, + "Standard_E8bds_v5": 0.804000, + "Standard_E8bs_v5": 0.712000, + "Standard_E8d_v4": 0.696000, + "Standard_E8d_v5": 0.696000, + "Standard_E8ds_v4": 0.696000, + "Standard_E8ds_v5": 0.696000, + "Standard_E8s_v3": 0.638000, + "Standard_E8s_v4": 0.604000, + "Standard_E8s_v5": 0.604000, + "Standard_E96-24ads_v5": 7.632000, + "Standard_E96-24as_v4": 7.248000, + "Standard_E96-24as_v5": 6.528000, + "Standard_E96-24ds_v5": 8.352000, + "Standard_E96-24s_v5": 7.248000, + "Standard_E96-48ads_v5": 7.632000, + "Standard_E96-48as_v4": 7.248000, + "Standard_E96-48as_v5": 10.944000, + "Standard_E96-48ds_v5": 8.352000, + "Standard_E96-48s_v5": 7.248000, + "Standard_E96_v5": 7.248000, + "Standard_E96a_v4": 7.248000, + "Standard_E96ads_v5": 7.678000, + "Standard_E96as_v4": 7.248000, + "Standard_E96as_v5": 6.574000, + "Standard_E96bds_v5": 9.648000, + "Standard_E96bs_v5": 8.544000, + "Standard_E96d_v5": 8.352000, + "Standard_E96ds_v5": 8.352000, + "Standard_E96iads_v5": 8.395000, + "Standard_E96ias_v4": 7.248000, + "Standard_E96ias_v5": 7.181000, + "Standard_E96s_v5": 7.248000, + "Standard_F1": 0.065000, + "Standard_F16": 1.042000, + "Standard_F16s": 1.042000, + "Standard_F16s_v2": 0.888000, + "Standard_F1s": 0.065000, + "Standard_F2": 0.130000, + "Standard_F2s": 0.130000, + "Standard_F2s_v2": 0.111000, + "Standard_F32s_v2": 1.776000, + "Standard_F4": 0.260000, + "Standard_F48s_v2": 2.664000, + "Standard_F4s": 0.260000, + "Standard_F4s_v2": 0.222000, + "Standard_F64s_v2": 3.552000, + "Standard_F72s_v2": 3.996000, + "Standard_F8": 0.521000, + "Standard_F8s": 0.521000, + "Standard_F8s_v2": 0.444000, + "Standard_L16as_v3": 1.496000, + "Standard_L16s_v3": 1.672000, + "Standard_L32as_v3": 2.992000, + "Standard_L32s_v3": 3.344000, + "Standard_L48as_v3": 4.488000, + "Standard_L48s_v3": 5.016000, + "Standard_L64as_v3": 5.984000, + "Standard_L64s_v3": 6.688000, + "Standard_L80as_v3": 7.480000, + "Standard_L80s_v3": 8.360000, + "Standard_L8as_v3": 0.748000, + "Standard_L8s_v3": 0.836000, + "Standard_M128": 19.340000, + "Standard_M128-32ms": 38.697600, + "Standard_M128-64ms": 38.697600, + "Standard_M128dms_v2": 38.700000, + "Standard_M128ds_v2": 19.340000, + "Standard_M128m": 38.697600, + "Standard_M128ms": 38.697600, + "Standard_M128ms_v2": 38.090000, + "Standard_M128s": 19.340000, + "Standard_M128s_v2": 18.730000, + "Standard_M16-4ms": 4.455900, + "Standard_M16-8ms": 4.455900, + "Standard_M16ms": 4.455900, + "Standard_M16s": 3.461000, + "Standard_M192idms_v2": 46.490000, + "Standard_M192ids_v2": 23.250000, + "Standard_M192ims_v2": 45.880000, + "Standard_M192is_v2": 22.640000, + "Standard_M208-104ms_v2": 64.699000, + "Standard_M208-104s_v2": 32.350000, + "Standard_M208-52ms_v2": 64.699000, + "Standard_M208-52s_v2": 32.350000, + "Standard_M208ms_v2": 64.699000, + "Standard_M208s_v2": 32.350000, + "Standard_M32-16ms": 8.911700, + "Standard_M32-8ms": 8.911700, + "Standard_M32dms_v2": 8.910000, + "Standard_M32ls": 3.447600, + "Standard_M32ms": 8.911700, + "Standard_M32ms_v2": 8.760000, + "Standard_M32s": 4.835800, + "Standard_M32ts": 3.248000, + "Standard_M416-104ms_v2": 143.770000, + "Standard_M416-104s_v2": 71.891000, + "Standard_M416-208ms_v2": 143.770000, + "Standard_M416-208s_v2": 71.891000, + "Standard_M416is_v2": 71.891000, + "Standard_M416ms_v2": 143.770000, + "Standard_M416s_8_v2": 95.860000, + "Standard_M416s_v2": 71.891000, + "Standard_M64": 9.670000, + "Standard_M64-16ms": 14.989000, + "Standard_M64-32ms": 14.989000, + "Standard_M64dms_v2": 14.990000, + "Standard_M64ds_v2": 9.670000, + "Standard_M64ls": 6.498000, + "Standard_M64m": 14.989000, + "Standard_M64ms": 14.989000, + "Standard_M64ms_v2": 14.690000, + "Standard_M64s": 9.670000, + "Standard_M64s_v2": 9.370000, + "Standard_M8-2ms": 2.227900, + "Standard_M8-4ms": 2.227900, + "Standard_M8ms": 2.227900, + "Standard_NC16as_T4_v3": 1.565000, + "Standard_NC4as_T4_v3": 0.684000, + "Standard_NC64as_T4_v3": 5.645000, + "Standard_NC8as_T4_v3": 0.977000, + } + // australiacentral2 + initialOnDemandPrices["australiacentral2"] = map[string]float64{ + "Basic_A0": 0.024000, + "Basic_A1": 0.032000, + "Basic_A2": 0.098000, + "Basic_A3": 0.232000, + "Basic_A4": 0.464000, + "Dadsv5_Type1": 8.008000, + "Dasv4_Type1": 6.600000, + "Dasv4_Type2": 6.600000, + "Dasv5_Type1": 6.653000, + "Ddsv4_Type 1": 4.998000, + "Ddsv4_Type2": 5.936000, + "Ddsv5_Type1": 7.498000, + "Dsv3_Type3": 6.599000, + "Dsv3_Type4": 8.249000, + "Dsv4_Type1": 5.280000, + "Dsv4_Type2": 6.336000, + "Dsv5_Type1": 6.600000, + "Eadsv5_Type1": 8.395000, + "Easv4_Type1": 7.973000, + "Easv4_Type2": 7.973000, + "Easv5_Type1": 7.181000, + "Ebdsv5-Type1": 7.075000, + "Ebsv5-Type1": 6.266000, + "Edsv4_Type 1": 6.125000, + "Edsv4_Type2": 9.008000, + "Edsv5_Type1": 9.187000, + "Esv3_Type3": 6.434000, + "Esv3_Type4": 8.717000, + "Esv4_Type1": 5.315000, + "Esv4_Type2": 6.976000, + "Esv5_Type1": 7.973000, + "Fsv2_Type2": 4.396000, + "Lasv3_Type1": 8.228000, + "Lsv3_Type1": 9.196000, + "Msmv2_Type1": 158.140000, + "Msv2_Type1": 79.080000, + "Standard_A0": 0.029000, + "Standard_A1": 0.071000, + "Standard_A1_v2": 0.050000, + "Standard_A2": 0.142000, + "Standard_A2_v2": 0.106000, + "Standard_A2m_v2": 0.149000, + "Standard_A3": 0.284000, + "Standard_A4": 0.568000, + "Standard_A4_v2": 0.222000, + "Standard_A4m_v2": 0.312000, + "Standard_A5": 0.278000, + "Standard_A6": 0.556000, + "Standard_A7": 1.112000, + "Standard_A8_v2": 0.467000, + "Standard_A8m_v2": 0.655000, + "Standard_B12ms": 0.634000, + "Standard_B16als_v2": 0.673000, + "Standard_B16as_v2": 0.760000, + "Standard_B16ls_v2": 0.822000, + "Standard_B16ms": 0.845000, + "Standard_B16s_v2": 0.918000, + "Standard_B1ls": 0.006600, + "Standard_B1ms": 0.026400, + "Standard_B1s": 0.013200, + "Standard_B20ms": 1.056000, + "Standard_B2als_v2": 0.047500, + "Standard_B2as_v2": 0.095000, + "Standard_B2ats_v2": 0.021100, + "Standard_B2ls_v2": 0.062000, + "Standard_B2ms": 0.106000, + "Standard_B2s": 0.052800, + "Standard_B2s_v2": 0.106000, + "Standard_B2ts_v2": 0.013200, + "Standard_B32als_v2": 1.493000, + "Standard_B32as_v2": 1.520000, + "Standard_B32ls_v2": 1.496000, + "Standard_B32s_v2": 1.837000, + "Standard_B4als_v2": 0.168000, + "Standard_B4as_v2": 0.208000, + "Standard_B4ls_v2": 0.206000, + "Standard_B4ms": 0.211000, + "Standard_B4s_v2": 0.230000, + "Standard_B8als_v2": 0.337000, + "Standard_B8as_v2": 0.417000, + "Standard_B8ls_v2": 0.374000, + "Standard_B8ms": 0.422000, + "Standard_B8s_v2": 0.459000, + "Standard_D11_v2": 0.200000, + "Standard_D11_v2_Promo": 0.249000, + "Standard_D12_v2": 0.399000, + "Standard_D12_v2_Promo": 0.498750, + "Standard_D13_v2": 0.798000, + "Standard_D13_v2_Promo": 0.997000, + "Standard_D14_v2": 1.596000, + "Standard_D14_v2_Promo": 1.995000, + "Standard_D15_v2": 1.995000, + "Standard_D15i_v2": 1.995000, + "Standard_D16_v3": 1.000000, + "Standard_D16_v4": 0.960000, + "Standard_D16_v5": 0.960000, + "Standard_D16a_v4": 0.960000, + "Standard_D16ads_v5": 1.776000, + "Standard_D16as_v4": 0.960000, + "Standard_D16as_v5": 1.600000, + "Standard_D16d_v4": 1.136000, + "Standard_D16d_v5": 1.136000, + "Standard_D16ds_v4": 1.136000, + "Standard_D16ds_v5": 1.136000, + "Standard_D16lds_v5": 1.008000, + "Standard_D16ls_v5": 0.888000, + "Standard_D16s_v3": 1.000000, + "Standard_D16s_v4": 0.960000, + "Standard_D16s_v5": 0.960000, + "Standard_D1_v2": 0.084000, + "Standard_D2_v2": 0.168000, + "Standard_D2_v2_Promo": 0.210000, + "Standard_D2_v3": 0.125000, + "Standard_D2_v4": 0.120000, + "Standard_D2_v5": 0.120000, + "Standard_D2a_v4": 0.120000, + "Standard_D2ads_v5": 0.222000, + "Standard_D2as_v4": 0.120000, + "Standard_D2as_v5": 0.200000, + "Standard_D2d_v4": 0.142000, + "Standard_D2d_v5": 0.142000, + "Standard_D2ds_v4": 0.142000, + "Standard_D2ds_v5": 0.142000, + "Standard_D2lds_v5": 0.126000, + "Standard_D2ls_v5": 0.111000, + "Standard_D2s_v3": 0.125000, + "Standard_D2s_v4": 0.120000, + "Standard_D2s_v5": 0.120000, + "Standard_D32-16s_v3": 2.000000, + "Standard_D32-8s_v3": 2.000000, + "Standard_D32_v3": 2.000000, + "Standard_D32_v4": 1.920000, + "Standard_D32_v5": 1.920000, + "Standard_D32a_v4": 1.920000, + "Standard_D32ads_v5": 2.080000, + "Standard_D32as_v4": 1.920000, + "Standard_D32as_v5": 1.728000, + "Standard_D32d_v4": 2.272000, + "Standard_D32d_v5": 2.272000, + "Standard_D32ds_v4": 2.272000, + "Standard_D32ds_v5": 2.272000, + "Standard_D32lds_v5": 2.016000, + "Standard_D32ls_v5": 1.776000, + "Standard_D32s_v3": 2.000000, + "Standard_D32s_v4": 1.920000, + "Standard_D32s_v5": 1.920000, + "Standard_D3_v2": 0.336000, + "Standard_D3_v2_Promo": 0.420000, + "Standard_D48_v3": 3.000000, + "Standard_D48_v4": 2.880000, + "Standard_D48_v5": 2.880000, + "Standard_D48a_v4": 2.880000, + "Standard_D48ads_v5": 5.328000, + "Standard_D48as_v4": 2.880000, + "Standard_D48as_v5": 2.592000, + "Standard_D48d_v4": 3.408000, + "Standard_D48d_v5": 3.408000, + "Standard_D48ds_v4": 3.408000, + "Standard_D48ds_v5": 3.408000, + "Standard_D48lds_v5": 3.024000, + "Standard_D48ls_v5": 2.664000, + "Standard_D48s_v3": 3.000000, + "Standard_D48s_v4": 2.880000, + "Standard_D48s_v5": 2.880000, + "Standard_D4_v2": 0.673000, + "Standard_D4_v2_Promo": 0.841000, + "Standard_D4_v3": 0.250000, + "Standard_D4_v4": 0.240000, + "Standard_D4_v5": 0.240000, + "Standard_D4a_v4": 0.240000, + "Standard_D4ads_v5": 0.260000, + "Standard_D4as_v4": 0.240000, + "Standard_D4as_v5": 0.216000, + "Standard_D4d_v4": 0.284000, + "Standard_D4d_v5": 0.284000, + "Standard_D4ds_v4": 0.284000, + "Standard_D4ds_v5": 0.284000, + "Standard_D4lds_v5": 0.252000, + "Standard_D4ls_v5": 0.222000, + "Standard_D4s_v3": 0.250000, + "Standard_D4s_v4": 0.240000, + "Standard_D4s_v5": 0.240000, + "Standard_D5_v2": 1.345000, + "Standard_D5_v2_Promo": 1.681000, + "Standard_D64-16s_v3": 4.000000, + "Standard_D64-32s_v3": 4.000000, + "Standard_D64_v3": 4.000000, + "Standard_D64_v4": 3.840000, + "Standard_D64_v5": 3.840000, + "Standard_D64a_v4": 3.840000, + "Standard_D64ads_v5": 7.104000, + "Standard_D64as_v4": 3.840000, + "Standard_D64as_v5": 3.456000, + "Standard_D64d_v4": 4.544000, + "Standard_D64d_v5": 4.544000, + "Standard_D64ds_v4": 4.544000, + "Standard_D64ds_v5": 4.544000, + "Standard_D64lds_v5": 4.032000, + "Standard_D64ls_v5": 3.552000, + "Standard_D64s_v3": 4.000000, + "Standard_D64s_v4": 3.840000, + "Standard_D64s_v5": 3.840000, + "Standard_D8_v3": 0.500000, + "Standard_D8_v4": 0.480000, + "Standard_D8_v5": 0.480000, + "Standard_D8a_v4": 0.480000, + "Standard_D8ads_v5": 0.888000, + "Standard_D8as_v4": 0.480000, + "Standard_D8as_v5": 0.800000, + "Standard_D8d_v4": 0.568000, + "Standard_D8d_v5": 0.568000, + "Standard_D8ds_v4": 0.568000, + "Standard_D8ds_v5": 0.568000, + "Standard_D8lds_v5": 0.504000, + "Standard_D8ls_v5": 0.444000, + "Standard_D8s_v3": 0.500000, + "Standard_D8s_v4": 0.480000, + "Standard_D8s_v5": 0.480000, + "Standard_D96_v5": 5.760000, + "Standard_D96a_v4": 5.760000, + "Standard_D96ads_v5": 10.656000, + "Standard_D96as_v4": 5.760000, + "Standard_D96as_v5": 9.600000, + "Standard_D96d_v5": 6.816000, + "Standard_D96ds_v5": 6.816000, + "Standard_D96lds_v5": 6.048000, + "Standard_D96ls_v5": 5.328000, + "Standard_D96s_v5": 5.760000, + "Standard_DS11-1_v2": 0.200000, + "Standard_DS11_v2": 0.200000, + "Standard_DS11_v2_Promo": 0.249000, + "Standard_DS12-1_v2": 0.399000, + "Standard_DS12-2_v2": 0.399000, + "Standard_DS12_v2": 0.399000, + "Standard_DS12_v2_Promo": 0.498750, + "Standard_DS13-2_v2": 0.798000, + "Standard_DS13-4_v2": 0.798000, + "Standard_DS13_v2": 0.798000, + "Standard_DS13_v2_Promo": 0.997000, + "Standard_DS14-4_v2": 1.596000, + "Standard_DS14-8_v2": 1.596000, + "Standard_DS14_v2": 1.596000, + "Standard_DS14_v2_Promo": 1.995000, + "Standard_DS15_v2": 1.995000, + "Standard_DS15i_v2": 1.995000, + "Standard_DS1_v2": 0.084000, + "Standard_DS2_v2": 0.168000, + "Standard_DS2_v2_Promo": 0.210000, + "Standard_DS3_v2": 0.336000, + "Standard_DS3_v2_Promo": 0.420000, + "Standard_DS4_v2": 0.673000, + "Standard_DS4_v2_Promo": 0.841000, + "Standard_DS5_v2": 1.345000, + "Standard_DS5_v2_Promo": 1.681000, + "Standard_E104i_v5": 8.637000, + "Standard_E104id_v5": 9.953000, + "Standard_E104ids_v5": 9.953000, + "Standard_E104is_v5": 8.637000, + "Standard_E112iads_v5": 9.794000, + "Standard_E112ias_v5": 8.378000, + "Standard_E112ibds_v5": 12.382000, + "Standard_E112ibs_v5": 10.965000, + "Standard_E16-4ads_v5": 1.272000, + "Standard_E16-4as_v4": 1.208000, + "Standard_E16-4as_v5": 1.088000, + "Standard_E16-4ds_v4": 1.392000, + "Standard_E16-4ds_v5": 1.392000, + "Standard_E16-4s_v3": 1.277000, + "Standard_E16-4s_v4": 1.208000, + "Standard_E16-4s_v5": 1.208000, + "Standard_E16-8ads_v5": 1.272000, + "Standard_E16-8as_v4": 1.208000, + "Standard_E16-8as_v5": 1.088000, + "Standard_E16-8ds_v4": 1.392000, + "Standard_E16-8ds_v5": 1.392000, + "Standard_E16-8s_v3": 1.277000, + "Standard_E16-8s_v4": 1.208000, + "Standard_E16-8s_v5": 1.208000, + "Standard_E16_v3": 1.277000, + "Standard_E16_v4": 1.208000, + "Standard_E16_v5": 1.208000, + "Standard_E16a_v4": 1.208000, + "Standard_E16ads_v5": 1.272000, + "Standard_E16as_v4": 1.208000, + "Standard_E16as_v5": 1.088000, + "Standard_E16bds_v5": 1.608000, + "Standard_E16bs_v5": 1.424000, + "Standard_E16d_v4": 1.392000, + "Standard_E16d_v5": 1.392000, + "Standard_E16ds_v4": 1.392000, + "Standard_E16ds_v5": 1.392000, + "Standard_E16s_v3": 1.277000, + "Standard_E16s_v4": 1.208000, + "Standard_E16s_v5": 1.208000, + "Standard_E20_v3": 1.509000, + "Standard_E20_v4": 1.510000, + "Standard_E20_v5": 1.510000, + "Standard_E20a_v4": 1.510000, + "Standard_E20ads_v5": 1.590000, + "Standard_E20as_v4": 1.510000, + "Standard_E20as_v5": 1.406000, + "Standard_E20d_v4": 1.740000, + "Standard_E20d_v5": 1.740000, + "Standard_E20ds_v4": 1.740000, + "Standard_E20ds_v5": 1.740000, + "Standard_E20s_v3": 1.509000, + "Standard_E20s_v4": 1.510000, + "Standard_E20s_v5": 1.510000, + "Standard_E2_v3": 0.160000, + "Standard_E2_v4": 0.151000, + "Standard_E2_v5": 0.151000, + "Standard_E2a_v4": 0.151000, + "Standard_E2ads_v5": 0.159000, + "Standard_E2as_v4": 0.151000, + "Standard_E2as_v5": 0.182000, + "Standard_E2bds_v5": 0.201000, + "Standard_E2bs_v5": 0.178000, + "Standard_E2d_v4": 0.174000, + "Standard_E2d_v5": 0.174000, + "Standard_E2ds_v4": 0.174000, + "Standard_E2ds_v5": 0.174000, + "Standard_E2s_v3": 0.160000, + "Standard_E2s_v4": 0.151000, + "Standard_E2s_v5": 0.151000, + "Standard_E32-16ads_v5": 4.016000, + "Standard_E32-16as_v4": 2.416000, + "Standard_E32-16as_v5": 2.176000, + "Standard_E32-16ds_v4": 2.784000, + "Standard_E32-16ds_v5": 2.784000, + "Standard_E32-16s_v3": 2.554000, + "Standard_E32-16s_v4": 2.416000, + "Standard_E32-16s_v5": 2.416000, + "Standard_E32-8ads_v5": 4.016000, + "Standard_E32-8as_v4": 2.416000, + "Standard_E32-8as_v5": 2.176000, + "Standard_E32-8ds_v4": 2.784000, + "Standard_E32-8ds_v5": 2.784000, + "Standard_E32-8s_v3": 2.554000, + "Standard_E32-8s_v4": 2.416000, + "Standard_E32-8s_v5": 2.416000, + "Standard_E32_v3": 2.554000, + "Standard_E32_v4": 2.416000, + "Standard_E32_v5": 2.416000, + "Standard_E32a_v4": 2.416000, + "Standard_E32ads_v5": 2.544000, + "Standard_E32as_v4": 2.416000, + "Standard_E32as_v5": 2.176000, + "Standard_E32bds_v5": 3.216000, + "Standard_E32bs_v5": 2.848000, + "Standard_E32d_v4": 2.784000, + "Standard_E32d_v5": 2.784000, + "Standard_E32ds_v4": 2.784000, + "Standard_E32ds_v5": 2.784000, + "Standard_E32s_v3": 2.554000, + "Standard_E32s_v4": 2.416000, + "Standard_E32s_v5": 2.416000, + "Standard_E4-2ads_v5": 0.318000, + "Standard_E4-2as_v4": 0.302000, + "Standard_E4-2as_v5": 0.272000, + "Standard_E4-2ds_v4": 0.348000, + "Standard_E4-2ds_v5": 0.348000, + "Standard_E4-2s_v3": 0.319000, + "Standard_E4-2s_v4": 0.302000, + "Standard_E4-2s_v5": 0.302000, + "Standard_E48_v3": 3.623000, + "Standard_E48_v4": 3.624000, + "Standard_E48_v5": 3.624000, + "Standard_E48a_v4": 3.624000, + "Standard_E48ads_v5": 3.862000, + "Standard_E48as_v4": 3.624000, + "Standard_E48as_v5": 3.264000, + "Standard_E48bds_v5": 4.824000, + "Standard_E48bs_v5": 4.272000, + "Standard_E48d_v4": 4.176000, + "Standard_E48d_v5": 4.176000, + "Standard_E48ds_v4": 4.176000, + "Standard_E48ds_v5": 4.176000, + "Standard_E48s_v3": 3.623000, + "Standard_E48s_v4": 3.624000, + "Standard_E48s_v5": 3.624000, + "Standard_E4_v3": 0.319000, + "Standard_E4_v4": 0.302000, + "Standard_E4_v5": 0.302000, + "Standard_E4a_v4": 0.302000, + "Standard_E4ads_v5": 0.318000, + "Standard_E4as_v4": 0.302000, + "Standard_E4as_v5": 0.272000, + "Standard_E4bds_v5": 0.402000, + "Standard_E4bs_v5": 0.356000, + "Standard_E4d_v4": 0.348000, + "Standard_E4d_v5": 0.348000, + "Standard_E4ds_v4": 0.348000, + "Standard_E4ds_v5": 0.348000, + "Standard_E4s_v3": 0.319000, + "Standard_E4s_v4": 0.302000, + "Standard_E4s_v5": 0.302000, + "Standard_E64-16ads_v5": 5.088000, + "Standard_E64-16as_v4": 4.832000, + "Standard_E64-16as_v5": 4.352000, + "Standard_E64-16ds_v4": 5.568000, + "Standard_E64-16ds_v5": 5.568000, + "Standard_E64-16s_v3": 4.347000, + "Standard_E64-16s_v4": 4.832000, + "Standard_E64-16s_v5": 4.832000, + "Standard_E64-32ads_v5": 5.088000, + "Standard_E64-32as_v4": 4.832000, + "Standard_E64-32as_v5": 7.296000, + "Standard_E64-32ds_v4": 5.568000, + "Standard_E64-32ds_v5": 5.568000, + "Standard_E64-32s_v3": 4.347000, + "Standard_E64-32s_v4": 4.832000, + "Standard_E64-32s_v5": 4.832000, + "Standard_E64_v3": 4.347000, + "Standard_E64_v4": 4.832000, + "Standard_E64_v5": 4.832000, + "Standard_E64a_v4": 4.832000, + "Standard_E64ads_v5": 5.134000, + "Standard_E64as_v4": 4.832000, + "Standard_E64as_v5": 4.352000, + "Standard_E64bds_v5": 6.432000, + "Standard_E64bs_v5": 5.696000, + "Standard_E64d_v4": 5.568000, + "Standard_E64d_v5": 5.568000, + "Standard_E64ds_v4": 5.568000, + "Standard_E64ds_v5": 5.568000, + "Standard_E64s_v3": 4.347000, + "Standard_E64s_v4": 4.832000, + "Standard_E64s_v5": 4.832000, + "Standard_E8-2ads_v5": 1.004000, + "Standard_E8-2as_v4": 0.604000, + "Standard_E8-2as_v5": 0.544000, + "Standard_E8-2ds_v4": 0.696000, + "Standard_E8-2ds_v5": 0.696000, + "Standard_E8-2s_v3": 0.638000, + "Standard_E8-2s_v4": 0.604000, + "Standard_E8-2s_v5": 0.604000, + "Standard_E8-4ads_v5": 1.004000, + "Standard_E8-4as_v4": 0.604000, + "Standard_E8-4as_v5": 0.544000, + "Standard_E8-4ds_v4": 0.696000, + "Standard_E8-4ds_v5": 0.696000, + "Standard_E8-4s_v3": 0.638000, + "Standard_E8-4s_v4": 0.604000, + "Standard_E8-4s_v5": 0.604000, + "Standard_E80ids_v4": 6.960000, + "Standard_E80is_v4": 6.040000, + "Standard_E8_v3": 0.638000, + "Standard_E8_v4": 0.604000, + "Standard_E8_v5": 0.604000, + "Standard_E8a_v4": 0.604000, + "Standard_E8ads_v5": 0.636000, + "Standard_E8as_v4": 0.604000, + "Standard_E8as_v5": 0.544000, + "Standard_E8bds_v5": 0.804000, + "Standard_E8bs_v5": 0.712000, + "Standard_E8d_v4": 0.696000, + "Standard_E8d_v5": 0.696000, + "Standard_E8ds_v4": 0.696000, + "Standard_E8ds_v5": 0.696000, + "Standard_E8s_v3": 0.638000, + "Standard_E8s_v4": 0.604000, + "Standard_E8s_v5": 0.604000, + "Standard_E96-24ads_v5": 7.632000, + "Standard_E96-24as_v4": 7.248000, + "Standard_E96-24as_v5": 10.944000, + "Standard_E96-24ds_v5": 8.352000, + "Standard_E96-24s_v5": 7.248000, + "Standard_E96-48ads_v5": 12.048000, + "Standard_E96-48as_v4": 7.248000, + "Standard_E96-48as_v5": 10.944000, + "Standard_E96-48ds_v5": 8.352000, + "Standard_E96-48s_v5": 7.248000, + "Standard_E96_v5": 7.248000, + "Standard_E96a_v4": 7.248000, + "Standard_E96ads_v5": 7.632000, + "Standard_E96as_v4": 7.248000, + "Standard_E96as_v5": 6.574000, + "Standard_E96bds_v5": 9.648000, + "Standard_E96bs_v5": 8.544000, + "Standard_E96d_v5": 8.352000, + "Standard_E96ds_v5": 8.352000, + "Standard_E96iads_v5": 8.395000, + "Standard_E96ias_v4": 7.248000, + "Standard_E96ias_v5": 7.181000, + "Standard_E96s_v5": 7.248000, + "Standard_F1": 0.065000, + "Standard_F16": 1.042000, + "Standard_F16s": 1.042000, + "Standard_F16s_v2": 0.888000, + "Standard_F1s": 0.065000, + "Standard_F2": 0.130000, + "Standard_F2s": 0.130000, + "Standard_F2s_v2": 0.111000, + "Standard_F32s_v2": 1.776000, + "Standard_F4": 0.260000, + "Standard_F48s_v2": 2.664000, + "Standard_F4s": 0.260000, + "Standard_F4s_v2": 0.222000, + "Standard_F64s_v2": 3.552000, + "Standard_F72s_v2": 3.996000, + "Standard_F8": 0.521000, + "Standard_F8s": 0.521000, + "Standard_F8s_v2": 0.444000, + "Standard_L16as_v3": 1.496000, + "Standard_L16s_v3": 1.672000, + "Standard_L32as_v3": 2.992000, + "Standard_L32s_v3": 3.344000, + "Standard_L48as_v3": 4.488000, + "Standard_L48s_v3": 5.016000, + "Standard_L64as_v3": 5.984000, + "Standard_L64s_v3": 6.688000, + "Standard_L80as_v3": 7.480000, + "Standard_L80s_v3": 8.360000, + "Standard_L8as_v3": 0.748000, + "Standard_L8s_v3": 0.836000, + "Standard_M128": 19.340000, + "Standard_M128-32ms": 38.697600, + "Standard_M128-64ms": 38.697600, + "Standard_M128m": 38.697600, + "Standard_M128ms": 38.697600, + "Standard_M128s": 19.340000, + "Standard_M16-4ms": 4.455900, + "Standard_M16-8ms": 4.455900, + "Standard_M16ms": 4.455900, + "Standard_M16s": 3.461000, + "Standard_M208-104ms_v2": 64.699000, + "Standard_M208-104s_v2": 32.350000, + "Standard_M208-52ms_v2": 64.699000, + "Standard_M208-52s_v2": 32.350000, + "Standard_M208ms_v2": 64.699000, + "Standard_M208s_v2": 32.350000, + "Standard_M32-16ms": 8.911700, + "Standard_M32-8ms": 8.911700, + "Standard_M32ls": 3.447600, + "Standard_M32ms": 8.911700, + "Standard_M32s": 4.835800, + "Standard_M32ts": 3.248000, + "Standard_M416-104ms_v2": 143.770000, + "Standard_M416-104s_v2": 71.891000, + "Standard_M416-208ms_v2": 143.770000, + "Standard_M416-208s_v2": 71.891000, + "Standard_M416is_v2": 71.891000, + "Standard_M416ms_v2": 143.770000, + "Standard_M416s_8_v2": 95.860000, + "Standard_M416s_v2": 71.891000, + "Standard_M64": 9.670000, + "Standard_M64-16ms": 14.989000, + "Standard_M64-32ms": 14.989000, + "Standard_M64ls": 6.498000, + "Standard_M64m": 14.989000, + "Standard_M64ms": 14.989000, + "Standard_M64s": 9.670000, + "Standard_M8-2ms": 2.227900, + "Standard_M8-4ms": 2.227900, + "Standard_M8ms": 2.227900, + "Standard_NC16as_T4_v3": 1.565000, + "Standard_NC4as_T4_v3": 0.684000, + "Standard_NC64as_T4_v3": 5.645000, + "Standard_NC8as_T4_v3": 0.977000, + } + // australiaeast + initialOnDemandPrices["australiaeast"] = map[string]float64{ + "Basic_A0": 0.024000, + "Basic_A1": 0.032000, + "Basic_A2": 0.098000, + "Basic_A3": 0.232000, + "Basic_A4": 0.464000, + "DCdsv3 Type1": 7.498000, + "DCsv2 Type 1": 1.056000, + "DCsv3 Type1": 6.336000, + "Dadsv5_Type1": 8.008000, + "Dasv4_Type1": 6.336000, + "Dasv4_Type2": 6.600000, + "Dasv5_Type1": 6.653000, + "Ddsv4_Type 1": 4.998000, + "Ddsv4_Type2": 5.936000, + "Ddsv5_Type1": 7.498000, + "Dsv3_Type3": 5.280000, + "Dsv3_Type4": 6.600000, + "Dsv4_Type1": 5.280000, + "Dsv4_Type2": 6.336000, + "Dsv5_Type1": 6.600000, + "Eadsv5_Type1": 8.395000, + "Easv4_Type1": 7.973000, + "Easv4_Type2": 7.973000, + "Easv5_Type1": 7.181000, + "Ebdsv5-Type1": 7.075000, + "Ebsv5-Type1": 6.266000, + "Edsv4_Type 1": 6.125000, + "Edsv4_Type2": 7.273000, + "Edsv5_Type1": 9.187000, + "Esv3_Type3": 5.148000, + "Esv3_Type4": 6.974000, + "Esv4_Type1": 5.315000, + "Esv4_Type2": 6.976000, + "Esv5_Type1": 7.973000, + "Fsv2 Type3": 5.128000, + "Fsv2_Type2": 4.396000, + "Fsv2_Type4": 5.861000, + "Lasv3_Type1": 8.228000, + "Lsv2_Type1": 8.228000, + "Lsv3_Type1": 9.196000, + "Mdmsv2MedMem _Type1": 42.571000, + "Mdsv2MedMem_Type1": 21.277000, + "Ms_Type1": 21.270000, + "Msm_Type1": 42.577000, + "Msmv2MedMem Type1": 41.899000, + "Msmv2_Type1": 158.140000, + "Msv2MedMem Type1": 20.606000, + "Msv2_Type1": 79.080000, + "NVasv4_Type1": 10.406000, + "NVsv3_Type1": 6.972000, + "Standard_A0": 0.029000, + "Standard_A1": 0.071000, + "Standard_A1_v2": 0.050000, + "Standard_A2": 0.142000, + "Standard_A2_v2": 0.106000, + "Standard_A2m_v2": 0.149000, + "Standard_A3": 0.284000, + "Standard_A4": 0.568000, + "Standard_A4_v2": 0.222000, + "Standard_A4m_v2": 0.312000, + "Standard_A5": 0.278000, + "Standard_A6": 0.556000, + "Standard_A7": 1.112000, + "Standard_A8_v2": 0.467000, + "Standard_A8m_v2": 0.655000, + "Standard_B12ms": 0.634000, + "Standard_B16als_v2": 0.673000, + "Standard_B16as_v2": 0.760000, + "Standard_B16ls_v2": 0.822000, + "Standard_B16ms": 0.845000, + "Standard_B16pls_v2": 0.601000, + "Standard_B16ps_v2": 0.678000, + "Standard_B16s_v2": 0.918000, + "Standard_B1ls": 0.006600, + "Standard_B1ms": 0.026400, + "Standard_B1s": 0.013200, + "Standard_B20ms": 1.056000, + "Standard_B2als_v2": 0.056700, + "Standard_B2as_v2": 0.104000, + "Standard_B2ats_v2": 0.011900, + "Standard_B2ls_v2": 0.052800, + "Standard_B2ms": 0.106000, + "Standard_B2pls_v2": 0.042400, + "Standard_B2ps_v2": 0.094000, + "Standard_B2pts_v2": 0.010600, + "Standard_B2s": 0.052800, + "Standard_B2s_v2": 0.106000, + "Standard_B2ts_v2": 0.022400, + "Standard_B32als_v2": 1.346000, + "Standard_B32as_v2": 1.520000, + "Standard_B32ls_v2": 1.496000, + "Standard_B32s_v2": 1.837000, + "Standard_B4als_v2": 0.168000, + "Standard_B4as_v2": 0.190000, + "Standard_B4ls_v2": 0.206000, + "Standard_B4ms": 0.211000, + "Standard_B4pls_v2": 0.150000, + "Standard_B4ps_v2": 0.188000, + "Standard_B4s_v2": 0.230000, + "Standard_B8als_v2": 0.373000, + "Standard_B8as_v2": 0.380000, + "Standard_B8ls_v2": 0.374000, + "Standard_B8ms": 0.422000, + "Standard_B8pls_v2": 0.300000, + "Standard_B8ps_v2": 0.339000, + "Standard_B8s_v2": 0.459000, + "Standard_D1": 0.093000, + "Standard_D11": 0.210000, + "Standard_D11_v2": 0.200000, + "Standard_D11_v2_Promo": 0.200000, + "Standard_D12": 0.420000, + "Standard_D12_v2": 0.399000, + "Standard_D12_v2_Promo": 0.399000, + "Standard_D13": 0.840000, + "Standard_D13_v2": 0.798000, + "Standard_D13_v2_Promo": 0.798000, + "Standard_D14": 1.680000, + "Standard_D14_v2": 1.596000, + "Standard_D14_v2_Promo": 1.596000, + "Standard_D15_v2": 1.995000, + "Standard_D15i_v2": 1.995000, + "Standard_D16_v3": 1.000000, + "Standard_D16_v4": 0.960000, + "Standard_D16_v5": 0.960000, + "Standard_D16a_v4": 0.960000, + "Standard_D16ads_v5": 1.776000, + "Standard_D16as_v4": 0.960000, + "Standard_D16as_v5": 1.600000, + "Standard_D16d_v4": 1.136000, + "Standard_D16d_v5": 1.136000, + "Standard_D16ds_v4": 1.136000, + "Standard_D16ds_v5": 1.136000, + "Standard_D16lds_v5": 1.008000, + "Standard_D16ls_v5": 0.888000, + "Standard_D16pds_v5": 0.912000, + "Standard_D16plds_v5": 0.808000, + "Standard_D16pls_v5": 0.710000, + "Standard_D16ps_v5": 0.768000, + "Standard_D16s_v3": 1.000000, + "Standard_D16s_v4": 0.960000, + "Standard_D16s_v5": 0.960000, + "Standard_D1_v2": 0.084000, + "Standard_D2": 0.186000, + "Standard_D2_v2": 0.168000, + "Standard_D2_v2_Promo": 0.168000, + "Standard_D2_v3": 0.125000, + "Standard_D2_v4": 0.120000, + "Standard_D2_v5": 0.120000, + "Standard_D2a_v4": 0.120000, + "Standard_D2ads_v5": 0.130000, + "Standard_D2as_v4": 0.120000, + "Standard_D2as_v5": 0.108000, + "Standard_D2d_v4": 0.142000, + "Standard_D2d_v5": 0.142000, + "Standard_D2ds_v4": 0.142000, + "Standard_D2ds_v5": 0.142000, + "Standard_D2lds_v5": 0.126000, + "Standard_D2ls_v5": 0.111000, + "Standard_D2pds_v5": 0.114000, + "Standard_D2plds_v5": 0.101000, + "Standard_D2pls_v5": 0.088800, + "Standard_D2ps_v5": 0.096000, + "Standard_D2s_v3": 0.125000, + "Standard_D2s_v4": 0.120000, + "Standard_D2s_v5": 0.120000, + "Standard_D3": 0.372000, + "Standard_D32-16s_v3": 2.000000, + "Standard_D32-8s_v3": 2.000000, + "Standard_D32_v3": 2.000000, + "Standard_D32_v4": 1.920000, + "Standard_D32_v5": 1.920000, + "Standard_D32a_v4": 1.920000, + "Standard_D32ads_v5": 3.552000, + "Standard_D32as_v4": 1.920000, + "Standard_D32as_v5": 1.728000, + "Standard_D32d_v4": 2.272000, + "Standard_D32d_v5": 2.272000, + "Standard_D32ds_v4": 2.272000, + "Standard_D32ds_v5": 2.272000, + "Standard_D32lds_v5": 2.016000, + "Standard_D32ls_v5": 1.776000, + "Standard_D32pds_v5": 1.824000, + "Standard_D32plds_v5": 1.616000, + "Standard_D32pls_v5": 1.421000, + "Standard_D32ps_v5": 1.536000, + "Standard_D32s_v3": 2.000000, + "Standard_D32s_v4": 1.920000, + "Standard_D32s_v5": 1.920000, + "Standard_D3_v2": 0.336000, + "Standard_D3_v2_Promo": 0.336000, + "Standard_D4": 0.744000, + "Standard_D48_v3": 3.000000, + "Standard_D48_v4": 2.880000, + "Standard_D48_v5": 2.880000, + "Standard_D48a_v4": 2.880000, + "Standard_D48ads_v5": 5.328000, + "Standard_D48as_v4": 2.880000, + "Standard_D48as_v5": 2.592000, + "Standard_D48d_v4": 3.408000, + "Standard_D48d_v5": 3.408000, + "Standard_D48ds_v4": 3.408000, + "Standard_D48ds_v5": 3.408000, + "Standard_D48lds_v5": 3.024000, + "Standard_D48ls_v5": 2.664000, + "Standard_D48pds_v5": 2.736000, + "Standard_D48plds_v5": 2.424000, + "Standard_D48pls_v5": 2.131000, + "Standard_D48ps_v5": 2.304000, + "Standard_D48s_v3": 3.000000, + "Standard_D48s_v4": 2.880000, + "Standard_D48s_v5": 2.880000, + "Standard_D4_v2": 0.673000, + "Standard_D4_v2_Promo": 0.673000, + "Standard_D4_v3": 0.250000, + "Standard_D4_v4": 0.240000, + "Standard_D4_v5": 0.240000, + "Standard_D4a_v4": 0.240000, + "Standard_D4ads_v5": 0.444000, + "Standard_D4as_v4": 0.240000, + "Standard_D4as_v5": 0.400000, + "Standard_D4d_v4": 0.284000, + "Standard_D4d_v5": 0.284000, + "Standard_D4ds_v4": 0.284000, + "Standard_D4ds_v5": 0.284000, + "Standard_D4lds_v5": 0.252000, + "Standard_D4ls_v5": 0.222000, + "Standard_D4pds_v5": 0.228000, + "Standard_D4plds_v5": 0.202000, + "Standard_D4pls_v5": 0.178000, + "Standard_D4ps_v5": 0.192000, + "Standard_D4s_v3": 0.250000, + "Standard_D4s_v4": 0.240000, + "Standard_D4s_v5": 0.240000, + "Standard_D5_v2": 1.345000, + "Standard_D5_v2_Promo": 1.345000, + "Standard_D64-16s_v3": 4.000000, + "Standard_D64-32s_v3": 4.000000, + "Standard_D64_v3": 4.000000, + "Standard_D64_v4": 3.840000, + "Standard_D64_v5": 3.840000, + "Standard_D64a_v4": 3.840000, + "Standard_D64ads_v5": 4.160000, + "Standard_D64as_v4": 3.840000, + "Standard_D64as_v5": 6.400000, + "Standard_D64d_v4": 4.544000, + "Standard_D64d_v5": 4.544000, + "Standard_D64ds_v4": 4.544000, + "Standard_D64ds_v5": 4.544000, + "Standard_D64lds_v5": 4.032000, + "Standard_D64ls_v5": 3.552000, + "Standard_D64pds_v5": 3.648000, + "Standard_D64plds_v5": 3.232000, + "Standard_D64pls_v5": 2.842000, + "Standard_D64ps_v5": 3.072000, + "Standard_D64s_v3": 4.000000, + "Standard_D64s_v4": 3.840000, + "Standard_D64s_v5": 3.840000, + "Standard_D8_v3": 0.500000, + "Standard_D8_v4": 0.480000, + "Standard_D8_v5": 0.480000, + "Standard_D8a_v4": 0.480000, + "Standard_D8ads_v5": 0.888000, + "Standard_D8as_v4": 0.480000, + "Standard_D8as_v5": 0.800000, + "Standard_D8d_v4": 0.568000, + "Standard_D8d_v5": 0.568000, + "Standard_D8ds_v4": 0.568000, + "Standard_D8ds_v5": 0.568000, + "Standard_D8lds_v5": 0.504000, + "Standard_D8ls_v5": 0.444000, + "Standard_D8pds_v5": 0.456000, + "Standard_D8plds_v5": 0.404000, + "Standard_D8pls_v5": 0.355000, + "Standard_D8ps_v5": 0.384000, + "Standard_D8s_v3": 0.500000, + "Standard_D8s_v4": 0.480000, + "Standard_D8s_v5": 0.480000, + "Standard_D96_v5": 5.760000, + "Standard_D96a_v4": 5.760000, + "Standard_D96ads_v5": 6.240000, + "Standard_D96as_v4": 5.760000, + "Standard_D96as_v5": 9.600000, + "Standard_D96d_v5": 6.816000, + "Standard_D96ds_v5": 6.816000, + "Standard_D96lds_v5": 6.048000, + "Standard_D96ls_v5": 5.328000, + "Standard_D96s_v5": 5.760000, + "Standard_DC16ds_v3": 2.272000, + "Standard_DC16s_v3": 1.920000, + "Standard_DC1ds_v3": 0.142000, + "Standard_DC1s_v2": 0.120000, + "Standard_DC1s_v3": 0.120000, + "Standard_DC24ds_v3": 3.408000, + "Standard_DC24s_v3": 2.880000, + "Standard_DC2ds_v3": 0.284000, + "Standard_DC2s_v2": 0.240000, + "Standard_DC2s_v3": 0.240000, + "Standard_DC32ds_v3": 4.544000, + "Standard_DC32s_v3": 3.840000, + "Standard_DC48ds_v3": 6.816000, + "Standard_DC48s_v3": 5.760000, + "Standard_DC4ds_v3": 0.568000, + "Standard_DC4s_v2": 0.480000, + "Standard_DC4s_v3": 0.480000, + "Standard_DC8_v2": 0.960000, + "Standard_DC8ds_v3": 1.136000, + "Standard_DC8s_v3": 0.960000, + "Standard_DS1": 0.093000, + "Standard_DS11": 0.210000, + "Standard_DS11-1_v2": 0.200000, + "Standard_DS11_v2": 0.200000, + "Standard_DS11_v2_Promo": 0.200000, + "Standard_DS12": 0.420000, + "Standard_DS12-1_v2": 0.399000, + "Standard_DS12-2_v2": 0.399000, + "Standard_DS12_v2": 0.399000, + "Standard_DS12_v2_Promo": 0.399000, + "Standard_DS13": 0.840000, + "Standard_DS13-2_v2": 0.798000, + "Standard_DS13-4_v2": 0.798000, + "Standard_DS13_v2": 0.798000, + "Standard_DS13_v2_Promo": 0.798000, + "Standard_DS14": 1.680000, + "Standard_DS14-4_v2": 1.596000, + "Standard_DS14-8_v2": 1.596000, + "Standard_DS14_v2": 1.596000, + "Standard_DS14_v2_Promo": 1.596000, + "Standard_DS15_v2": 1.995000, + "Standard_DS15i_v2": 1.995000, + "Standard_DS1_v2": 0.084000, + "Standard_DS2": 0.186000, + "Standard_DS2_v2": 0.168000, + "Standard_DS2_v2_Promo": 0.168000, + "Standard_DS3": 0.372000, + "Standard_DS3_v2": 0.336000, + "Standard_DS3_v2_Promo": 0.336000, + "Standard_DS4": 0.744000, + "Standard_DS4_v2": 0.673000, + "Standard_DS4_v2_Promo": 0.673000, + "Standard_DS5_v2": 1.345000, + "Standard_DS5_v2_Promo": 1.345000, + "Standard_E104i_v5": 8.637000, + "Standard_E104id_v5": 9.953000, + "Standard_E104ids_v5": 9.953000, + "Standard_E104is_v5": 8.637000, + "Standard_E112iads_v5": 9.794000, + "Standard_E112ias_v5": 8.378000, + "Standard_E112ibds_v5": 12.382000, + "Standard_E112ibs_v5": 10.965000, + "Standard_E16-4ads_v5": 2.008000, + "Standard_E16-4as_v4": 1.208000, + "Standard_E16-4as_v5": 1.824000, + "Standard_E16-4ds_v4": 1.392000, + "Standard_E16-4ds_v5": 1.392000, + "Standard_E16-4s_v3": 1.277000, + "Standard_E16-4s_v4": 1.208000, + "Standard_E16-4s_v5": 1.208000, + "Standard_E16-8ads_v5": 1.272000, + "Standard_E16-8as_v4": 1.208000, + "Standard_E16-8as_v5": 1.088000, + "Standard_E16-8ds_v4": 1.392000, + "Standard_E16-8ds_v5": 1.392000, + "Standard_E16-8s_v3": 1.277000, + "Standard_E16-8s_v4": 1.208000, + "Standard_E16-8s_v5": 1.208000, + "Standard_E16_v3": 1.277000, + "Standard_E16_v4": 1.208000, + "Standard_E16_v5": 1.208000, + "Standard_E16a_v4": 1.208000, + "Standard_E16ads_v5": 1.318000, + "Standard_E16as_v4": 1.208000, + "Standard_E16as_v5": 1.134000, + "Standard_E16bds_v5": 1.608000, + "Standard_E16bs_v5": 1.424000, + "Standard_E16d_v4": 1.392000, + "Standard_E16d_v5": 1.392000, + "Standard_E16ds_v4": 1.392000, + "Standard_E16ds_v5": 1.392000, + "Standard_E16pds_v5": 1.112000, + "Standard_E16ps_v5": 0.966000, + "Standard_E16s_v3": 1.277000, + "Standard_E16s_v4": 1.208000, + "Standard_E16s_v5": 1.208000, + "Standard_E20_v3": 1.509000, + "Standard_E20_v4": 1.510000, + "Standard_E20_v5": 1.510000, + "Standard_E20a_v4": 1.510000, + "Standard_E20ads_v5": 1.590000, + "Standard_E20as_v4": 1.510000, + "Standard_E20as_v5": 1.360000, + "Standard_E20d_v4": 1.740000, + "Standard_E20d_v5": 1.740000, + "Standard_E20ds_v4": 1.740000, + "Standard_E20ds_v5": 1.740000, + "Standard_E20pds_v5": 1.390000, + "Standard_E20ps_v5": 1.208000, + "Standard_E20s_v3": 1.509000, + "Standard_E20s_v4": 1.510000, + "Standard_E20s_v5": 1.510000, + "Standard_E2_v3": 0.160000, + "Standard_E2_v4": 0.151000, + "Standard_E2_v5": 0.151000, + "Standard_E2a_v4": 0.151000, + "Standard_E2ads_v5": 0.205000, + "Standard_E2as_v4": 0.151000, + "Standard_E2as_v5": 0.182000, + "Standard_E2bds_v5": 0.201000, + "Standard_E2bs_v5": 0.178000, + "Standard_E2d_v4": 0.174000, + "Standard_E2d_v5": 0.174000, + "Standard_E2ds_v4": 0.174000, + "Standard_E2ds_v5": 0.174000, + "Standard_E2pds_v5": 0.139000, + "Standard_E2ps_v5": 0.121000, + "Standard_E2s_v3": 0.160000, + "Standard_E2s_v4": 0.151000, + "Standard_E2s_v5": 0.151000, + "Standard_E32-16ads_v5": 2.544000, + "Standard_E32-16as_v4": 2.416000, + "Standard_E32-16as_v5": 3.648000, + "Standard_E32-16ds_v4": 2.784000, + "Standard_E32-16ds_v5": 2.784000, + "Standard_E32-16s_v3": 2.554000, + "Standard_E32-16s_v4": 2.416000, + "Standard_E32-16s_v5": 2.416000, + "Standard_E32-8ads_v5": 4.016000, + "Standard_E32-8as_v4": 2.416000, + "Standard_E32-8as_v5": 3.648000, + "Standard_E32-8ds_v4": 2.784000, + "Standard_E32-8ds_v5": 2.784000, + "Standard_E32-8s_v3": 2.554000, + "Standard_E32-8s_v4": 2.416000, + "Standard_E32-8s_v5": 2.416000, + "Standard_E32_v3": 2.554000, + "Standard_E32_v4": 2.416000, + "Standard_E32_v5": 2.416000, + "Standard_E32a_v4": 2.416000, + "Standard_E32ads_v5": 2.544000, + "Standard_E32as_v4": 2.416000, + "Standard_E32as_v5": 2.176000, + "Standard_E32bds_v5": 3.216000, + "Standard_E32bs_v5": 2.848000, + "Standard_E32d_v4": 2.784000, + "Standard_E32d_v5": 2.784000, + "Standard_E32ds_v4": 2.784000, + "Standard_E32ds_v5": 2.784000, + "Standard_E32pds_v5": 2.224000, + "Standard_E32ps_v5": 1.933000, + "Standard_E32s_v3": 2.554000, + "Standard_E32s_v4": 2.416000, + "Standard_E32s_v5": 2.416000, + "Standard_E4-2ads_v5": 0.318000, + "Standard_E4-2as_v4": 0.302000, + "Standard_E4-2as_v5": 0.272000, + "Standard_E4-2ds_v4": 0.348000, + "Standard_E4-2ds_v5": 0.348000, + "Standard_E4-2s_v3": 0.319000, + "Standard_E4-2s_v4": 0.302000, + "Standard_E4-2s_v5": 0.302000, + "Standard_E48_v3": 3.623000, + "Standard_E48_v4": 3.624000, + "Standard_E48_v5": 3.624000, + "Standard_E48a_v4": 3.624000, + "Standard_E48ads_v5": 3.862000, + "Standard_E48as_v4": 3.624000, + "Standard_E48as_v5": 3.310000, + "Standard_E48bds_v5": 4.824000, + "Standard_E48bs_v5": 4.272000, + "Standard_E48d_v4": 4.176000, + "Standard_E48d_v5": 4.176000, + "Standard_E48ds_v4": 4.176000, + "Standard_E48ds_v5": 4.176000, + "Standard_E48s_v3": 3.623000, + "Standard_E48s_v4": 3.624000, + "Standard_E48s_v5": 3.624000, + "Standard_E4_v3": 0.319000, + "Standard_E4_v4": 0.302000, + "Standard_E4_v5": 0.302000, + "Standard_E4a_v4": 0.302000, + "Standard_E4ads_v5": 0.318000, + "Standard_E4as_v4": 0.302000, + "Standard_E4as_v5": 0.318000, + "Standard_E4bds_v5": 0.402000, + "Standard_E4bs_v5": 0.356000, + "Standard_E4d_v4": 0.348000, + "Standard_E4d_v5": 0.348000, + "Standard_E4ds_v4": 0.348000, + "Standard_E4ds_v5": 0.348000, + "Standard_E4pds_v5": 0.278000, + "Standard_E4ps_v5": 0.242000, + "Standard_E4s_v3": 0.319000, + "Standard_E4s_v4": 0.302000, + "Standard_E4s_v5": 0.302000, + "Standard_E64-16ads_v5": 5.088000, + "Standard_E64-16as_v4": 4.832000, + "Standard_E64-16as_v5": 7.296000, + "Standard_E64-16ds_v4": 5.568000, + "Standard_E64-16ds_v5": 5.568000, + "Standard_E64-16s_v3": 4.347000, + "Standard_E64-16s_v4": 4.832000, + "Standard_E64-16s_v5": 4.832000, + "Standard_E64-32ads_v5": 5.088000, + "Standard_E64-32as_v4": 4.832000, + "Standard_E64-32as_v5": 4.352000, + "Standard_E64-32ds_v4": 5.568000, + "Standard_E64-32ds_v5": 5.568000, + "Standard_E64-32s_v3": 4.347000, + "Standard_E64-32s_v4": 4.832000, + "Standard_E64-32s_v5": 4.832000, + "Standard_E64_v3": 4.347000, + "Standard_E64_v4": 4.832000, + "Standard_E64_v5": 4.832000, + "Standard_E64a_v4": 4.832000, + "Standard_E64ads_v5": 5.088000, + "Standard_E64as_v4": 4.832000, + "Standard_E64as_v5": 4.352000, + "Standard_E64bds_v5": 6.432000, + "Standard_E64bs_v5": 5.696000, + "Standard_E64d_v4": 5.568000, + "Standard_E64d_v5": 5.568000, + "Standard_E64ds_v4": 5.568000, + "Standard_E64ds_v5": 5.568000, + "Standard_E64i_v3": 4.347000, + "Standard_E64is_v3": 4.347000, + "Standard_E64s_v3": 4.347000, + "Standard_E64s_v4": 4.832000, + "Standard_E64s_v5": 4.832000, + "Standard_E8-2ads_v5": 0.636000, + "Standard_E8-2as_v4": 0.604000, + "Standard_E8-2as_v5": 0.544000, + "Standard_E8-2ds_v4": 0.696000, + "Standard_E8-2ds_v5": 0.696000, + "Standard_E8-2s_v3": 0.638000, + "Standard_E8-2s_v4": 0.604000, + "Standard_E8-2s_v5": 0.604000, + "Standard_E8-4ads_v5": 1.004000, + "Standard_E8-4as_v4": 0.604000, + "Standard_E8-4as_v5": 0.912000, + "Standard_E8-4ds_v4": 0.696000, + "Standard_E8-4ds_v5": 0.696000, + "Standard_E8-4s_v3": 0.638000, + "Standard_E8-4s_v4": 0.604000, + "Standard_E8-4s_v5": 0.604000, + "Standard_E80ids_v4": 6.960000, + "Standard_E80is_v4": 6.040000, + "Standard_E8_v3": 0.638000, + "Standard_E8_v4": 0.604000, + "Standard_E8_v5": 0.604000, + "Standard_E8a_v4": 0.604000, + "Standard_E8ads_v5": 0.682000, + "Standard_E8as_v4": 0.604000, + "Standard_E8as_v5": 0.590000, + "Standard_E8bds_v5": 0.804000, + "Standard_E8bs_v5": 0.712000, + "Standard_E8d_v4": 0.696000, + "Standard_E8d_v5": 0.696000, + "Standard_E8ds_v4": 0.696000, + "Standard_E8ds_v5": 0.696000, + "Standard_E8pds_v5": 0.556000, + "Standard_E8ps_v5": 0.483000, + "Standard_E8s_v3": 0.638000, + "Standard_E8s_v4": 0.604000, + "Standard_E8s_v5": 0.604000, + "Standard_E96-24ads_v5": 12.048000, + "Standard_E96-24as_v4": 7.248000, + "Standard_E96-24as_v5": 6.528000, + "Standard_E96-24ds_v5": 8.352000, + "Standard_E96-24s_v5": 7.248000, + "Standard_E96-48ads_v5": 7.632000, + "Standard_E96-48as_v4": 7.248000, + "Standard_E96-48as_v5": 10.944000, + "Standard_E96-48ds_v5": 8.352000, + "Standard_E96-48s_v5": 7.248000, + "Standard_E96_v5": 7.248000, + "Standard_E96a_v4": 7.248000, + "Standard_E96ads_v5": 7.678000, + "Standard_E96as_v4": 7.248000, + "Standard_E96as_v5": 6.528000, + "Standard_E96bds_v5": 9.648000, + "Standard_E96bs_v5": 8.544000, + "Standard_E96d_v5": 8.352000, + "Standard_E96ds_v5": 8.352000, + "Standard_E96iads_v5": 8.395000, + "Standard_E96ias_v5": 7.181000, + "Standard_E96s_v5": 7.248000, + "Standard_F1": 0.065000, + "Standard_F16": 1.042000, + "Standard_F16s": 1.042000, + "Standard_F16s_v2": 0.888000, + "Standard_F1s": 0.065000, + "Standard_F2": 0.130000, + "Standard_F2s": 0.130000, + "Standard_F2s_v2": 0.111000, + "Standard_F32s_v2": 1.776000, + "Standard_F4": 0.260000, + "Standard_F48s_v2": 2.664000, + "Standard_F4s": 0.260000, + "Standard_F4s_v2": 0.222000, + "Standard_F64s_v2": 3.552000, + "Standard_F72s_v2": 3.996000, + "Standard_F8": 0.521000, + "Standard_F8s": 0.521000, + "Standard_F8s_v2": 0.444000, + "Standard_FX12mds": 1.356000, + "Standard_FX24mds": 2.712000, + "Standard_FX36mds": 4.068000, + "Standard_FX48mds": 5.424000, + "Standard_FX4mds": 0.452000, + "Standard_G1": 0.638000, + "Standard_G2": 1.276000, + "Standard_G3": 2.552000, + "Standard_G4": 5.104000, + "Standard_G5": 10.208000, + "Standard_GS1": 0.638000, + "Standard_GS2": 1.276000, + "Standard_GS3": 2.552000, + "Standard_GS4": 5.104000, + "Standard_GS4-4": 5.104000, + "Standard_GS4-8": 5.104000, + "Standard_GS5": 10.208000, + "Standard_GS5-16": 10.208000, + "Standard_GS5-8": 10.208000, + "Standard_H16": 2.369000, + "Standard_H16m": 3.175000, + "Standard_H16mr": 3.492000, + "Standard_H16r": 2.606000, + "Standard_H8": 1.185000, + "Standard_H8m": 1.587000, + "Standard_HB120-16rs_v2": 5.220000, + "Standard_HB120-32rs_v2": 5.220000, + "Standard_HB120-64rs_v2": 5.220000, + "Standard_HB120-96rs_v2": 5.220000, + "Standard_HB120rs_v2": 5.220000, + "Standard_HC44-16rs": 4.594000, + "Standard_HC44-32rs": 4.594000, + "Standard_HC44rs": 4.594000, + "Standard_L16as_v3": 1.496000, + "Standard_L16s": 1.496000, + "Standard_L16s_v2": 1.496000, + "Standard_L16s_v3": 1.672000, + "Standard_L32as_v3": 2.992000, + "Standard_L32s": 2.992000, + "Standard_L32s_v2": 2.992000, + "Standard_L32s_v3": 3.344000, + "Standard_L48as_v3": 4.488000, + "Standard_L48s_v2": 4.488000, + "Standard_L48s_v3": 5.016000, + "Standard_L4s": 0.374000, + "Standard_L64as_v3": 5.984000, + "Standard_L64s_v2": 5.984000, + "Standard_L64s_v3": 6.688000, + "Standard_L80as_v3": 7.480000, + "Standard_L80s_v2": 7.480000, + "Standard_L80s_v3": 8.360000, + "Standard_L88is_v2": 8.228000, + "Standard_L8as_v3": 0.748000, + "Standard_L8s": 0.748000, + "Standard_L8s_v2": 0.748000, + "Standard_L8s_v3": 0.836000, + "Standard_L96s_v2": 8.976000, + "Standard_M128": 19.340000, + "Standard_M128-32ms": 38.697600, + "Standard_M128-64ms": 38.697600, + "Standard_M128dms_v2": 38.701000, + "Standard_M128ds_v2": 19.343000, + "Standard_M128m": 38.697600, + "Standard_M128ms": 38.697600, + "Standard_M128ms_v2": 38.090000, + "Standard_M128s": 19.340000, + "Standard_M128s_v2": 18.733000, + "Standard_M16-4ms": 4.455900, + "Standard_M16-8ms": 4.455900, + "Standard_M16ms": 4.455900, + "Standard_M16s": 3.461000, + "Standard_M192idms_v2": 46.493000, + "Standard_M192ids_v2": 23.246000, + "Standard_M192ims_v2": 45.882000, + "Standard_M192is_v2": 22.636000, + "Standard_M208-104ms_v2": 64.699000, + "Standard_M208-104s_v2": 32.350000, + "Standard_M208-52ms_v2": 64.699000, + "Standard_M208-52s_v2": 32.350000, + "Standard_M208ms_v2": 64.699000, + "Standard_M208s_v2": 32.350000, + "Standard_M32-16ms": 8.911700, + "Standard_M32-8ms": 8.911700, + "Standard_M32dms_v2": 8.912000, + "Standard_M32ls": 3.447600, + "Standard_M32ms": 8.911700, + "Standard_M32ms_v2": 8.759000, + "Standard_M32s": 4.835800, + "Standard_M32ts": 3.248000, + "Standard_M416-104ms_v2": 143.770000, + "Standard_M416-104s_v2": 71.891000, + "Standard_M416-208ms_v2": 143.770000, + "Standard_M416-208s_v2": 71.891000, + "Standard_M416is_v2": 71.891000, + "Standard_M416ms_v2": 143.770000, + "Standard_M416s_8_v2": 95.860000, + "Standard_M416s_v2": 71.891000, + "Standard_M64": 9.670000, + "Standard_M64-16ms": 14.989000, + "Standard_M64-32ms": 14.989000, + "Standard_M64dms_v2": 14.993000, + "Standard_M64ds_v2": 9.670000, + "Standard_M64ls": 6.498000, + "Standard_M64m": 14.989000, + "Standard_M64ms": 14.989000, + "Standard_M64ms_v2": 14.688000, + "Standard_M64s": 9.670000, + "Standard_M64s_v2": 9.365000, + "Standard_M8-2ms": 2.227900, + "Standard_M8-4ms": 2.227900, + "Standard_M8ms": 2.227900, + "Standard_NC12": 3.100000, + "Standard_NC12s_v3": 8.468000, + "Standard_NC16as_T4_v3": 1.565000, + "Standard_NC24": 6.199000, + "Standard_NC24ads_A100_v4": 5.326000, + "Standard_NC24r": 6.842000, + "Standard_NC24rs_v3": 18.630000, + "Standard_NC24s_v3": 16.936000, + "Standard_NC48ads_A100_v4": 10.652000, + "Standard_NC4as_T4_v3": 0.684000, + "Standard_NC6": 1.550000, + "Standard_NC64as_T4_v3": 5.658000, + "Standard_NC6s_v3": 4.234000, + "Standard_NC8as_T4_v3": 0.978000, + "Standard_NC96ads_A100_v4": 21.303000, + "Standard_ND96ams_A100_v4": 47.516000, + "Standard_ND96amsr_A100_v4": 47.516000, + "Standard_NV12": 3.179000, + "Standard_NV12ads_A10_v5": 1.317000, + "Standard_NV12s_v3": 1.590000, + "Standard_NV16as_v4": 1.351000, + "Standard_NV18ads_A10_v5": 2.320000, + "Standard_NV24": 6.358000, + "Standard_NV24s_v3": 3.179000, + "Standard_NV32as_v4": 2.703000, + "Standard_NV36adms_A10_v5": 6.554000, + "Standard_NV36ads_A10_v5": 4.640000, + "Standard_NV48s_v3": 6.358000, + "Standard_NV4as_v4": 0.338000, + "Standard_NV6": 1.590000, + "Standard_NV6ads_A10_v5": 0.658000, + "Standard_NV72ads_A10_v5": 9.454000, + "Standard_NV8as_v4": 0.676000, + } + // australiasoutheast + initialOnDemandPrices["australiasoutheast"] = map[string]float64{ + "Basic_A0": 0.024000, + "Basic_A1": 0.032000, + "Basic_A2": 0.104000, + "Basic_A3": 0.232000, + "Basic_A4": 0.464000, + "DCsv2 Type 1": 1.098000, + "Dadsv5_Type1": 8.248000, + "Dasv4_Type1": 6.589000, + "Dasv4_Type2": 6.589000, + "Dasv5_Type1": 6.887000, + "Ddsv4_Type 1": 5.171000, + "Ddsv4_Type2": 6.140000, + "Ddsv5_Type1": 7.756000, + "Dsv3_Type3": 5.280000, + "Dsv3_Type4": 6.600000, + "Dsv4_Type1": 5.491000, + "Dsv4_Type2": 6.864000, + "Dsv5_Type1": 6.864000, + "Eadsv5_Type1": 8.992000, + "Easv5_Type1": 7.756000, + "Ebdsv5-Type1": 7.642000, + "Ebsv5-Type1": 6.818000, + "Edsv4_Type 1": 6.589000, + "Edsv4_Type2": 7.825000, + "Edsv5_Type1": 9.884000, + "Esv3_Type3": 5.148000, + "Esv3_Type4": 6.974000, + "Esv4_Type1": 5.766000, + "Esv4_Type2": 7.568000, + "Esv5_Type1": 8.649000, + "Fsv2 Type3": 5.642000, + "Fsv2_Type2": 4.376000, + "Fsv2_Type4": 6.078000, + "Lasv3_Type1": 8.923000, + "Lsv3_Type1": 9.981000, + "Mdmsv2MedMem _Type1": 38.167000, + "Mdsv2MedMem_Type1": 19.076000, + "Msmv2MedMem Type1": 37.565000, + "Msmv2_Type1": 158.140000, + "Msv2MedMem Type1": 18.474000, + "Msv2_Type1": 79.080000, + "Standard_A0": 0.029000, + "Standard_A1": 0.071000, + "Standard_A1_v2": 0.050000, + "Standard_A2": 0.142000, + "Standard_A2_v2": 0.106000, + "Standard_A2m_v2": 0.178000, + "Standard_A3": 0.284000, + "Standard_A4": 0.568000, + "Standard_A4_v2": 0.222000, + "Standard_A4m_v2": 0.374000, + "Standard_A5": 0.278000, + "Standard_A6": 0.556000, + "Standard_A7": 1.112000, + "Standard_A8_v2": 0.467000, + "Standard_A8m_v2": 0.786000, + "Standard_B12ms": 0.634000, + "Standard_B16als_v2": 0.766000, + "Standard_B16as_v2": 0.856000, + "Standard_B16ls_v2": 0.840000, + "Standard_B16ms": 0.845000, + "Standard_B16s_v2": 0.939000, + "Standard_B1ls": 0.006600, + "Standard_B1ms": 0.026400, + "Standard_B1s": 0.013200, + "Standard_B20ms": 1.057000, + "Standard_B2als_v2": 0.048900, + "Standard_B2as_v2": 0.107000, + "Standard_B2ats_v2": 0.012200, + "Standard_B2ls_v2": 0.063300, + "Standard_B2ms": 0.106000, + "Standard_B2s": 0.052800, + "Standard_B2s_v2": 0.117000, + "Standard_B2ts_v2": 0.013500, + "Standard_B32als_v2": 1.385000, + "Standard_B32as_v2": 1.711000, + "Standard_B32ls_v2": 1.680000, + "Standard_B32s_v2": 1.731000, + "Standard_B4als_v2": 0.192000, + "Standard_B4as_v2": 0.214000, + "Standard_B4ls_v2": 0.210000, + "Standard_B4ms": 0.211000, + "Standard_B4s_v2": 0.235000, + "Standard_B8als_v2": 0.383000, + "Standard_B8as_v2": 0.391000, + "Standard_B8ls_v2": 0.420000, + "Standard_B8ms": 0.423000, + "Standard_B8s_v2": 0.433000, + "Standard_D1": 0.093000, + "Standard_D11": 0.210000, + "Standard_D11_v2": 0.200000, + "Standard_D11_v2_Promo": 0.200000, + "Standard_D12": 0.420000, + "Standard_D12_v2": 0.399000, + "Standard_D12_v2_Promo": 0.399000, + "Standard_D13": 0.840000, + "Standard_D13_v2": 0.798000, + "Standard_D13_v2_Promo": 0.798000, + "Standard_D14": 1.680000, + "Standard_D14_v2": 1.596000, + "Standard_D14_v2_Promo": 1.596000, + "Standard_D15_v2": 1.995000, + "Standard_D15i_v2": 1.995000, + "Standard_D16_v3": 1.000000, + "Standard_D16_v4": 0.998000, + "Standard_D16_v5": 0.998000, + "Standard_D16a_v4": 0.998000, + "Standard_D16ads_v5": 1.071000, + "Standard_D16as_v4": 0.998000, + "Standard_D16as_v5": 0.894000, + "Standard_D16d_v4": 1.085000, + "Standard_D16d_v5": 1.175000, + "Standard_D16ds_v4": 1.085000, + "Standard_D16ds_v5": 1.175000, + "Standard_D16lds_v5": 0.998000, + "Standard_D16ls_v5": 0.884000, + "Standard_D16s_v3": 1.000000, + "Standard_D16s_v4": 0.998000, + "Standard_D16s_v5": 0.998000, + "Standard_D1_v2": 0.077800, + "Standard_D2": 0.186000, + "Standard_D2_v2": 0.155760, + "Standard_D2_v2_Promo": 0.155760, + "Standard_D2_v3": 0.125000, + "Standard_D2_v4": 0.125000, + "Standard_D2_v5": 0.125000, + "Standard_D2a_v4": 0.125000, + "Standard_D2ads_v5": 0.134000, + "Standard_D2as_v4": 0.125000, + "Standard_D2as_v5": 0.112000, + "Standard_D2d_v4": 0.136000, + "Standard_D2d_v5": 0.147000, + "Standard_D2ds_v4": 0.136000, + "Standard_D2ds_v5": 0.147000, + "Standard_D2lds_v5": 0.125000, + "Standard_D2ls_v5": 0.111000, + "Standard_D2s_v3": 0.125000, + "Standard_D2s_v4": 0.125000, + "Standard_D2s_v5": 0.125000, + "Standard_D3": 0.372000, + "Standard_D32-16s_v3": 2.000000, + "Standard_D32-8s_v3": 2.000000, + "Standard_D32_v3": 2.000000, + "Standard_D32_v4": 1.997000, + "Standard_D32_v5": 1.997000, + "Standard_D32a_v4": 1.997000, + "Standard_D32ads_v5": 3.614000, + "Standard_D32as_v4": 1.997000, + "Standard_D32as_v5": 1.789000, + "Standard_D32d_v4": 2.170000, + "Standard_D32d_v5": 2.350000, + "Standard_D32ds_v4": 2.170000, + "Standard_D32ds_v5": 2.350000, + "Standard_D32lds_v5": 1.997000, + "Standard_D32ls_v5": 1.768000, + "Standard_D32s_v3": 2.000000, + "Standard_D32s_v4": 1.997000, + "Standard_D32s_v5": 1.997000, + "Standard_D3_v2": 0.311000, + "Standard_D3_v2_Promo": 0.311000, + "Standard_D4": 0.744000, + "Standard_D48_v3": 3.000000, + "Standard_D48_v4": 2.995000, + "Standard_D48_v5": 2.995000, + "Standard_D48a_v4": 2.995000, + "Standard_D48ads_v5": 5.422000, + "Standard_D48as_v4": 2.995000, + "Standard_D48as_v5": 2.683000, + "Standard_D48d_v4": 3.254000, + "Standard_D48d_v5": 3.526000, + "Standard_D48ds_v4": 3.254000, + "Standard_D48ds_v5": 3.526000, + "Standard_D48lds_v5": 2.995000, + "Standard_D48ls_v5": 2.652000, + "Standard_D48s_v3": 3.000000, + "Standard_D48s_v4": 2.995000, + "Standard_D48s_v5": 2.995000, + "Standard_D4_v2": 0.623000, + "Standard_D4_v2_Promo": 0.623000, + "Standard_D4_v3": 0.250000, + "Standard_D4_v4": 0.250000, + "Standard_D4_v5": 0.250000, + "Standard_D4a_v4": 0.250000, + "Standard_D4ads_v5": 0.268000, + "Standard_D4ahs_v4": 0.250000, + "Standard_D4as_v4": 0.250000, + "Standard_D4as_v5": 0.224000, + "Standard_D4d_v4": 0.271000, + "Standard_D4d_v5": 0.294000, + "Standard_D4ds_v4": 0.271000, + "Standard_D4ds_v5": 0.294000, + "Standard_D4lds_v5": 0.250000, + "Standard_D4ls_v5": 0.221000, + "Standard_D4s_v3": 0.250000, + "Standard_D4s_v4": 0.250000, + "Standard_D4s_v5": 0.250000, + "Standard_D5_v2": 1.246000, + "Standard_D5_v2_Promo": 1.246000, + "Standard_D64-16s_v3": 4.000000, + "Standard_D64-32s_v3": 4.000000, + "Standard_D64_v3": 4.000000, + "Standard_D64_v4": 3.994000, + "Standard_D64_v5": 3.994000, + "Standard_D64a_v4": 3.994000, + "Standard_D64ads_v5": 4.285000, + "Standard_D64as_v4": 3.994000, + "Standard_D64as_v5": 6.522000, + "Standard_D64d_v4": 4.339000, + "Standard_D64d_v5": 4.701000, + "Standard_D64ds_v4": 4.339000, + "Standard_D64ds_v5": 4.701000, + "Standard_D64lds_v5": 3.994000, + "Standard_D64ls_v5": 3.536000, + "Standard_D64s_v3": 4.000000, + "Standard_D64s_v4": 3.994000, + "Standard_D64s_v5": 3.994000, + "Standard_D8_v3": 0.500000, + "Standard_D8_v4": 0.499000, + "Standard_D8_v5": 0.499000, + "Standard_D8a_v4": 0.499000, + "Standard_D8ads_v5": 0.536000, + "Standard_D8as_v4": 0.499000, + "Standard_D8as_v5": 0.447000, + "Standard_D8d_v4": 0.542000, + "Standard_D8d_v5": 0.588000, + "Standard_D8ds_v4": 0.542000, + "Standard_D8ds_v5": 0.588000, + "Standard_D8lds_v5": 0.499000, + "Standard_D8ls_v5": 0.442000, + "Standard_D8s_v3": 0.500000, + "Standard_D8s_v4": 0.499000, + "Standard_D8s_v5": 0.499000, + "Standard_D96_v5": 5.990000, + "Standard_D96a_v4": 5.990000, + "Standard_D96ads_v5": 10.843000, + "Standard_D96as_v4": 5.990000, + "Standard_D96as_v5": 9.782000, + "Standard_D96d_v5": 7.051000, + "Standard_D96ds_v5": 7.051000, + "Standard_D96lds_v5": 5.990000, + "Standard_D96ls_v5": 5.304000, + "Standard_D96s_v5": 5.990000, + "Standard_DC1s_v2": 0.125000, + "Standard_DC2s_v2": 0.250000, + "Standard_DC4s_v2": 0.499000, + "Standard_DC8_v2": 0.998000, + "Standard_DS1": 0.093000, + "Standard_DS11": 0.210000, + "Standard_DS11-1_v2": 0.200000, + "Standard_DS11_v2": 0.200000, + "Standard_DS11_v2_Promo": 0.200000, + "Standard_DS12": 0.420000, + "Standard_DS12-1_v2": 0.399000, + "Standard_DS12-2_v2": 0.399000, + "Standard_DS12_v2": 0.399000, + "Standard_DS12_v2_Promo": 0.399000, + "Standard_DS13": 0.840000, + "Standard_DS13-2_v2": 0.798000, + "Standard_DS13-4_v2": 0.798000, + "Standard_DS13_v2": 0.798000, + "Standard_DS13_v2_Promo": 0.798000, + "Standard_DS14": 1.680000, + "Standard_DS14-4_v2": 1.596000, + "Standard_DS14-8_v2": 1.596000, + "Standard_DS14_v2": 1.596000, + "Standard_DS14_v2_Promo": 1.596000, + "Standard_DS15_v2": 1.995000, + "Standard_DS15i_v2": 1.995000, + "Standard_DS1_v2": 0.077800, + "Standard_DS2": 0.186000, + "Standard_DS2_v2": 0.155760, + "Standard_DS2_v2_Promo": 0.155760, + "Standard_DS3": 0.372000, + "Standard_DS3_v2": 0.311000, + "Standard_DS3_v2_Promo": 0.311000, + "Standard_DS4": 0.744000, + "Standard_DS4_v2": 0.623000, + "Standard_DS4_v2_Promo": 0.623000, + "Standard_DS5_v2": 1.246000, + "Standard_DS5_v2_Promo": 1.246000, + "Standard_E104i_v5": 9.369000, + "Standard_E104id_v5": 10.708000, + "Standard_E104ids_v5": 10.708000, + "Standard_E104is_v5": 9.369000, + "Standard_E112iads_v5": 10.490000, + "Standard_E112ias_v5": 9.049000, + "Standard_E112ibds_v5": 13.373000, + "Standard_E112ibs_v5": 11.932000, + "Standard_E16-4ads_v5": 2.098000, + "Standard_E16-4as_v4": 1.310000, + "Standard_E16-4as_v5": 1.175000, + "Standard_E16-4ds_v4": 1.382000, + "Standard_E16-4ds_v5": 1.498000, + "Standard_E16-4s_v3": 1.277000, + "Standard_E16-4s_v4": 1.310000, + "Standard_E16-4s_v5": 1.310000, + "Standard_E16-8ads_v5": 2.098000, + "Standard_E16-8as_v4": 1.310000, + "Standard_E16-8as_v5": 1.175000, + "Standard_E16-8ds_v4": 1.382000, + "Standard_E16-8ds_v5": 1.498000, + "Standard_E16-8s_v3": 1.277000, + "Standard_E16-8s_v4": 1.310000, + "Standard_E16-8s_v5": 1.310000, + "Standard_E16_v3": 1.277000, + "Standard_E16_v4": 1.310000, + "Standard_E16_v5": 1.310000, + "Standard_E16a_v4": 1.310000, + "Standard_E16ads_v5": 1.362000, + "Standard_E16as_v4": 1.310000, + "Standard_E16as_v5": 1.175000, + "Standard_E16bds_v5": 1.737000, + "Standard_E16bs_v5": 1.550000, + "Standard_E16d_v4": 1.382000, + "Standard_E16d_v5": 1.498000, + "Standard_E16ds_v4": 1.382000, + "Standard_E16ds_v5": 1.498000, + "Standard_E16s_v3": 1.277000, + "Standard_E16s_v4": 1.310000, + "Standard_E16s_v5": 1.310000, + "Standard_E20_v3": 1.509000, + "Standard_E20_v4": 1.638000, + "Standard_E20_v5": 1.638000, + "Standard_E20a_v4": 1.638000, + "Standard_E20ads_v5": 1.749000, + "Standard_E20as_v4": 1.638000, + "Standard_E20as_v5": 1.469000, + "Standard_E20d_v4": 1.728000, + "Standard_E20d_v5": 1.872000, + "Standard_E20ds_v4": 1.728000, + "Standard_E20ds_v5": 1.872000, + "Standard_E20s_v3": 1.509000, + "Standard_E20s_v4": 1.638000, + "Standard_E20s_v5": 1.638000, + "Standard_E2_v3": 0.160000, + "Standard_E2_v4": 0.164000, + "Standard_E2_v5": 0.164000, + "Standard_E2a_v4": 0.164000, + "Standard_E2ads_v5": 0.170000, + "Standard_E2as_v4": 0.164000, + "Standard_E2as_v5": 0.147000, + "Standard_E2bds_v5": 0.217000, + "Standard_E2bs_v5": 0.194000, + "Standard_E2d_v4": 0.173000, + "Standard_E2d_v5": 0.187000, + "Standard_E2ds_v4": 0.173000, + "Standard_E2ds_v5": 0.187000, + "Standard_E2s_v3": 0.160000, + "Standard_E2s_v4": 0.164000, + "Standard_E2s_v5": 0.164000, + "Standard_E32-16ads_v5": 2.725000, + "Standard_E32-16as_v4": 2.621000, + "Standard_E32-16as_v5": 2.350000, + "Standard_E32-16ds_v4": 2.765000, + "Standard_E32-16ds_v5": 2.995000, + "Standard_E32-16s_v3": 2.554000, + "Standard_E32-16s_v4": 2.621000, + "Standard_E32-16s_v5": 2.621000, + "Standard_E32-8ads_v5": 2.725000, + "Standard_E32-8as_v4": 2.621000, + "Standard_E32-8as_v5": 3.822000, + "Standard_E32-8ds_v4": 2.765000, + "Standard_E32-8ds_v5": 2.995000, + "Standard_E32-8s_v3": 2.554000, + "Standard_E32-8s_v4": 2.621000, + "Standard_E32-8s_v5": 2.621000, + "Standard_E32_v3": 2.554000, + "Standard_E32_v4": 2.621000, + "Standard_E32_v5": 2.621000, + "Standard_E32a_v4": 2.621000, + "Standard_E32ads_v5": 2.771000, + "Standard_E32as_v4": 2.621000, + "Standard_E32as_v5": 2.350000, + "Standard_E32bds_v5": 3.474000, + "Standard_E32bs_v5": 3.099000, + "Standard_E32d_v4": 2.765000, + "Standard_E32d_v5": 2.995000, + "Standard_E32ds_v4": 2.765000, + "Standard_E32ds_v5": 2.995000, + "Standard_E32s_v3": 2.554000, + "Standard_E32s_v4": 2.621000, + "Standard_E32s_v5": 2.621000, + "Standard_E4-2ads_v5": 0.525000, + "Standard_E4-2as_v4": 0.328000, + "Standard_E4-2as_v5": 0.294000, + "Standard_E4-2ds_v4": 0.346000, + "Standard_E4-2ds_v5": 0.374000, + "Standard_E4-2s_v3": 0.319000, + "Standard_E4-2s_v4": 0.328000, + "Standard_E4-2s_v5": 0.328000, + "Standard_E48_v3": 3.623000, + "Standard_E48_v4": 3.931000, + "Standard_E48_v5": 3.931000, + "Standard_E48a_v4": 3.931000, + "Standard_E48ads_v5": 4.133000, + "Standard_E48as_v4": 3.931000, + "Standard_E48as_v5": 3.526000, + "Standard_E48bds_v5": 5.210000, + "Standard_E48bs_v5": 4.649000, + "Standard_E48d_v4": 4.147000, + "Standard_E48d_v5": 4.493000, + "Standard_E48ds_v4": 4.147000, + "Standard_E48ds_v5": 4.493000, + "Standard_E48s_v3": 3.623000, + "Standard_E48s_v4": 3.931000, + "Standard_E48s_v5": 3.931000, + "Standard_E4_v3": 0.319000, + "Standard_E4_v4": 0.328000, + "Standard_E4_v5": 0.328000, + "Standard_E4a_v4": 0.328000, + "Standard_E4ads_v5": 0.387000, + "Standard_E4as_v4": 0.328000, + "Standard_E4as_v5": 0.294000, + "Standard_E4bds_v5": 0.434000, + "Standard_E4bs_v5": 0.387000, + "Standard_E4d_v4": 0.346000, + "Standard_E4d_v5": 0.374000, + "Standard_E4ds_v4": 0.346000, + "Standard_E4ds_v5": 0.374000, + "Standard_E4s_v3": 0.319000, + "Standard_E4s_v4": 0.328000, + "Standard_E4s_v5": 0.328000, + "Standard_E64-16ads_v5": 8.394000, + "Standard_E64-16as_v4": 5.242000, + "Standard_E64-16as_v5": 4.701000, + "Standard_E64-16ds_v4": 5.530000, + "Standard_E64-16ds_v5": 5.990000, + "Standard_E64-16s_v3": 4.347000, + "Standard_E64-16s_v4": 5.242000, + "Standard_E64-16s_v5": 5.242000, + "Standard_E64-32ads_v5": 8.394000, + "Standard_E64-32as_v4": 5.242000, + "Standard_E64-32as_v5": 4.701000, + "Standard_E64-32ds_v4": 5.530000, + "Standard_E64-32ds_v5": 5.990000, + "Standard_E64-32s_v3": 4.347000, + "Standard_E64-32s_v4": 5.242000, + "Standard_E64-32s_v5": 5.242000, + "Standard_E64_v3": 4.347000, + "Standard_E64_v4": 5.242000, + "Standard_E64_v5": 5.242000, + "Standard_E64a_v4": 5.242000, + "Standard_E64ads_v5": 5.450000, + "Standard_E64as_v4": 5.242000, + "Standard_E64as_v5": 4.747000, + "Standard_E64bds_v5": 6.947000, + "Standard_E64bs_v5": 6.198000, + "Standard_E64d_v4": 5.530000, + "Standard_E64d_v5": 5.990000, + "Standard_E64ds_v4": 5.530000, + "Standard_E64ds_v5": 5.990000, + "Standard_E64i_v3": 4.347000, + "Standard_E64is_v3": 4.347000, + "Standard_E64s_v3": 4.347000, + "Standard_E64s_v4": 5.242000, + "Standard_E64s_v5": 5.242000, + "Standard_E8-2ads_v5": 1.049000, + "Standard_E8-2as_v4": 0.655000, + "Standard_E8-2as_v5": 0.588000, + "Standard_E8-2ds_v4": 0.691000, + "Standard_E8-2ds_v5": 0.749000, + "Standard_E8-2s_v3": 0.638000, + "Standard_E8-2s_v4": 0.655000, + "Standard_E8-2s_v5": 0.655000, + "Standard_E8-4ads_v5": 1.049000, + "Standard_E8-4as_v4": 0.655000, + "Standard_E8-4as_v5": 0.588000, + "Standard_E8-4ds_v4": 0.691000, + "Standard_E8-4ds_v5": 0.749000, + "Standard_E8-4s_v3": 0.638000, + "Standard_E8-4s_v4": 0.655000, + "Standard_E8-4s_v5": 0.655000, + "Standard_E80ids_v4": 6.912000, + "Standard_E80is_v4": 6.552000, + "Standard_E8_v3": 0.638000, + "Standard_E8_v4": 0.655000, + "Standard_E8_v5": 0.655000, + "Standard_E8a_v4": 0.655000, + "Standard_E8ads_v5": 0.681000, + "Standard_E8as_v4": 0.655000, + "Standard_E8as_v5": 0.634000, + "Standard_E8bds_v5": 0.868000, + "Standard_E8bs_v5": 0.775000, + "Standard_E8d_v4": 0.691000, + "Standard_E8d_v5": 0.749000, + "Standard_E8ds_v4": 0.691000, + "Standard_E8ds_v5": 0.749000, + "Standard_E8s_v3": 0.638000, + "Standard_E8s_v4": 0.655000, + "Standard_E8s_v5": 0.655000, + "Standard_E96-24ads_v5": 12.590000, + "Standard_E96-24as_v4": 7.862000, + "Standard_E96-24as_v5": 7.051000, + "Standard_E96-24ds_v5": 8.986000, + "Standard_E96-24s_v5": 7.862000, + "Standard_E96-48ads_v5": 8.174000, + "Standard_E96-48as_v4": 7.862000, + "Standard_E96-48as_v5": 11.467000, + "Standard_E96-48ds_v5": 8.986000, + "Standard_E96-48s_v5": 7.862000, + "Standard_E96_v5": 7.862000, + "Standard_E96a_v4": 7.862000, + "Standard_E96ads_v5": 8.174000, + "Standard_E96as_v4": 7.862000, + "Standard_E96as_v5": 7.051000, + "Standard_E96bds_v5": 10.421000, + "Standard_E96bs_v5": 9.298000, + "Standard_E96d_v5": 8.986000, + "Standard_E96ds_v5": 8.986000, + "Standard_E96iads_v5": 8.992000, + "Standard_E96ias_v4": 7.862000, + "Standard_E96ias_v5": 7.756000, + "Standard_E96s_v5": 7.862000, + "Standard_F1": 0.065000, + "Standard_F16": 1.042000, + "Standard_F16s": 1.042000, + "Standard_F16s_v2": 0.891000, + "Standard_F1s": 0.065000, + "Standard_F2": 0.130000, + "Standard_F2s": 0.130000, + "Standard_F2s_v2": 0.111000, + "Standard_F32s_v2": 1.782000, + "Standard_F4": 0.260000, + "Standard_F48s_v2": 2.672000, + "Standard_F4s": 0.260000, + "Standard_F4s_v2": 0.223000, + "Standard_F64s_v2": 3.563000, + "Standard_F72s_v2": 4.009000, + "Standard_F8": 0.521000, + "Standard_F8s": 0.521000, + "Standard_F8s_v2": 0.445000, + "Standard_L16as_v3": 1.622000, + "Standard_L16s_v3": 1.815000, + "Standard_L32as_v3": 3.245000, + "Standard_L32s_v3": 3.630000, + "Standard_L48as_v3": 4.867000, + "Standard_L48s_v3": 5.444000, + "Standard_L64as_v3": 6.490000, + "Standard_L64s_v3": 7.259000, + "Standard_L80as_v3": 8.112000, + "Standard_L80s_v3": 9.074000, + "Standard_L8as_v3": 0.811000, + "Standard_L8s_v3": 0.907000, + "Standard_M128": 19.340000, + "Standard_M128-32ms": 38.698000, + "Standard_M128-64ms": 38.698000, + "Standard_M128dms_v2": 34.697000, + "Standard_M128ds_v2": 17.342000, + "Standard_M128m": 38.698000, + "Standard_M128ms": 38.698000, + "Standard_M128ms_v2": 34.150000, + "Standard_M128s": 19.340000, + "Standard_M128s_v2": 16.795000, + "Standard_M16-4ms": 4.455900, + "Standard_M16-8ms": 4.455900, + "Standard_M16ms": 4.455900, + "Standard_M16s": 3.461000, + "Standard_M192idms_v2": 41.683000, + "Standard_M192ids_v2": 20.842000, + "Standard_M192ims_v2": 41.136000, + "Standard_M192is_v2": 20.294000, + "Standard_M208-104ms_v2": 64.699000, + "Standard_M208-104s_v2": 32.350000, + "Standard_M208-52ms_v2": 64.699000, + "Standard_M208-52s_v2": 32.350000, + "Standard_M208ms_v2": 64.699000, + "Standard_M208s_v2": 32.350000, + "Standard_M32-16ms": 8.911700, + "Standard_M32-8ms": 8.911700, + "Standard_M32dms_v2": 7.990000, + "Standard_M32ls": 3.447600, + "Standard_M32ms": 8.911700, + "Standard_M32ms_v2": 7.853000, + "Standard_M32s": 4.835800, + "Standard_M32ts": 3.248000, + "Standard_M416-104ms_v2": 143.770000, + "Standard_M416-104s_v2": 71.891000, + "Standard_M416-208ms_v2": 143.770000, + "Standard_M416-208s_v2": 71.891000, + "Standard_M416is_v2": 71.891000, + "Standard_M416ms_v2": 143.770000, + "Standard_M416s_8_v2": 92.550000, + "Standard_M416s_v2": 71.891000, + "Standard_M64": 9.670000, + "Standard_M64-16ms": 14.989000, + "Standard_M64-32ms": 14.989000, + "Standard_M64dms_v2": 13.442000, + "Standard_M64ds_v2": 8.670000, + "Standard_M64ls": 6.498000, + "Standard_M64m": 14.989000, + "Standard_M64ms": 14.989000, + "Standard_M64ms_v2": 13.169000, + "Standard_M64s": 9.670000, + "Standard_M64s_v2": 8.397000, + "Standard_M8-2ms": 2.227900, + "Standard_M8-4ms": 2.227900, + "Standard_M8ms": 2.227900, + } + // brazilsouth + initialOnDemandPrices["brazilsouth"] = map[string]float64{ + "Basic_A0": 0.022000, + "Basic_A1": 0.041000, + "Basic_A2": 0.112000, + "Basic_A3": 0.232000, + "Basic_A4": 0.464000, + "Dadsv5_Type1": 10.164000, + "Dasv4_Type1": 8.078000, + "Dasv4_Type2": 8.415000, + "Dasv5_Type1": 8.501000, + "Ddsv4_Type 1": 6.336000, + "Ddsv4_Type2": 7.524000, + "Ddsv5_Type1": 9.504000, + "Dsv3_Type3": 6.996000, + "Dsv3_Type4": 8.413000, + "Dsv4_Type1": 6.731000, + "Dsv4_Type2": 8.078000, + "Dsv5_Type1": 8.415000, + "Eadsv5_Type1": 10.982000, + "Easv4_Type1": 10.613000, + "Easv4_Type2": 10.613000, + "Easv5_Type1": 9.557000, + "Ebdsv5-Type1": 9.381000, + "Ebsv5-Type1": 8.430000, + "Edsv4_Type 1": 8.025000, + "Edsv4_Type2": 9.551000, + "Edsv5_Type1": 12.038000, + "Esv3_Type3": 8.010000, + "Esv3_Type4": 10.852000, + "Esv4_Type1": 7.075000, + "Esv4_Type2": 9.286000, + "Esv5_Type1": 10.613000, + "FXmds Type1": 7.339000, + "Fsv2 Type3": 6.052000, + "Fsv2_Type2": 5.188000, + "Fsv2_Type4": 6.917000, + "Lasv3_Type1": 10.956000, + "Lsv3_Type1": 12.232000, + "Mdmsv2MedMem _Type1": 57.250000, + "Mdsv2MedMem_Type1": 28.614000, + "Ms_Type1": 28.604000, + "Msm_Type1": 57.258000, + "Msmv2MedMem Type1": 56.347000, + "Msmv2_Type1": 212.680000, + "Msv2MedMem Type1": 27.712000, + "Msv2_Type1": 106.350000, + "NVsv3_Type1": 8.527000, + "Standard_A0": 0.024000, + "Standard_A1": 0.080000, + "Standard_A1_v2": 0.061000, + "Standard_A2": 0.160000, + "Standard_A2_v2": 0.129000, + "Standard_A2m_v2": 0.208000, + "Standard_A3": 0.320000, + "Standard_A4": 0.640000, + "Standard_A4_v2": 0.270000, + "Standard_A4m_v2": 0.437000, + "Standard_A5": 0.291000, + "Standard_A6": 0.582000, + "Standard_A7": 1.164000, + "Standard_A8_v2": 0.567000, + "Standard_A8m_v2": 0.917000, + "Standard_B12ms": 0.806000, + "Standard_B16als_v2": 0.857000, + "Standard_B16as_v2": 0.968000, + "Standard_B16ls_v2": 0.952000, + "Standard_B16ms": 1.075000, + "Standard_B16s_v2": 1.149000, + "Standard_B1ls": 0.008400, + "Standard_B1ms": 0.033600, + "Standard_B1s": 0.016800, + "Standard_B20ms": 1.344000, + "Standard_B2als_v2": 0.069700, + "Standard_B2as_v2": 0.130000, + "Standard_B2ats_v2": 0.015100, + "Standard_B2ls_v2": 0.076400, + "Standard_B2ms": 0.134000, + "Standard_B2s": 0.067200, + "Standard_B2s_v2": 0.134000, + "Standard_B2ts_v2": 0.016800, + "Standard_B32als_v2": 1.714000, + "Standard_B32as_v2": 1.936000, + "Standard_B32ls_v2": 2.051000, + "Standard_B32s_v2": 2.298000, + "Standard_B4als_v2": 0.214000, + "Standard_B4as_v2": 0.242000, + "Standard_B4ls_v2": 0.257000, + "Standard_B4ms": 0.269000, + "Standard_B4s_v2": 0.269000, + "Standard_B8als_v2": 0.465000, + "Standard_B8as_v2": 0.484000, + "Standard_B8ls_v2": 0.513000, + "Standard_B8ms": 0.538000, + "Standard_B8s_v2": 0.574000, + "Standard_D1": 0.095000, + "Standard_D11": 0.235000, + "Standard_D11_v2": 0.235000, + "Standard_D11_v2_Promo": 0.235000, + "Standard_D12": 0.470000, + "Standard_D12_v2": 0.470000, + "Standard_D12_v2_Promo": 0.470000, + "Standard_D13": 0.939000, + "Standard_D13_v2": 0.940000, + "Standard_D13_v2_Promo": 0.940000, + "Standard_D14": 1.879000, + "Standard_D14_v2": 1.879000, + "Standard_D14_v2_Promo": 1.879000, + "Standard_D15_v2": 2.349000, + "Standard_D15i_v2": 2.349000, + "Standard_D16_v3": 1.272000, + "Standard_D16_v4": 1.224000, + "Standard_D16_v5": 1.224000, + "Standard_D16a_v4": 1.224000, + "Standard_D16ads_v5": 2.056000, + "Standard_D16as_v4": 1.224000, + "Standard_D16as_v5": 1.104000, + "Standard_D16as_v5_Promo": 0.979000, + "Standard_D16d_v4": 1.440000, + "Standard_D16d_v5": 1.440000, + "Standard_D16ds_v4": 1.440000, + "Standard_D16ds_v5": 1.440000, + "Standard_D16lds_v5": 1.192000, + "Standard_D16ls_v5": 1.048000, + "Standard_D16s_v3": 1.272000, + "Standard_D16s_v4": 1.224000, + "Standard_D16s_v5": 1.224000, + "Standard_D1_v2": 0.085600, + "Standard_D2": 0.190000, + "Standard_D2_v2": 0.171000, + "Standard_D2_v2_Promo": 0.171000, + "Standard_D2_v3": 0.159000, + "Standard_D2_v4": 0.153000, + "Standard_D2_v5": 0.153000, + "Standard_D2a_v4": 0.153000, + "Standard_D2ads_v5": 0.257000, + "Standard_D2as_v4": 0.153000, + "Standard_D2as_v5": 0.138000, + "Standard_D2as_v5_Promo": 0.122000, + "Standard_D2d_v4": 0.180000, + "Standard_D2d_v5": 0.180000, + "Standard_D2ds_v4": 0.180000, + "Standard_D2ds_v5": 0.180000, + "Standard_D2lds_v5": 0.149000, + "Standard_D2ls_v5": 0.131000, + "Standard_D2s_v3": 0.159000, + "Standard_D2s_v4": 0.153000, + "Standard_D2s_v5": 0.153000, + "Standard_D3": 0.380000, + "Standard_D32-16s_v3": 2.544000, + "Standard_D32-8s_v3": 2.544000, + "Standard_D32_v3": 2.544000, + "Standard_D32_v4": 2.448000, + "Standard_D32_v5": 2.448000, + "Standard_D32a_v4": 2.448000, + "Standard_D32ads_v5": 2.640000, + "Standard_D32as_v4": 2.448000, + "Standard_D32as_v5": 2.208000, + "Standard_D32as_v5_Promo": 1.958000, + "Standard_D32d_v4": 2.880000, + "Standard_D32d_v5": 2.880000, + "Standard_D32ds_v4": 2.880000, + "Standard_D32ds_v5": 2.880000, + "Standard_D32lds_v5": 2.384000, + "Standard_D32ls_v5": 2.096000, + "Standard_D32s_v3": 2.544000, + "Standard_D32s_v4": 2.448000, + "Standard_D32s_v5": 2.448000, + "Standard_D3_v2": 0.343000, + "Standard_D3_v2_Promo": 0.343000, + "Standard_D4": 0.760000, + "Standard_D48_v3": 3.816000, + "Standard_D48_v4": 3.672000, + "Standard_D48_v5": 3.672000, + "Standard_D48a_v4": 3.672000, + "Standard_D48ads_v5": 3.960000, + "Standard_D48as_v4": 3.672000, + "Standard_D48as_v5": 3.312000, + "Standard_D48as_v5_Promo": 2.938000, + "Standard_D48d_v4": 4.320000, + "Standard_D48d_v5": 4.320000, + "Standard_D48ds_v4": 4.320000, + "Standard_D48ds_v5": 4.320000, + "Standard_D48lds_v5": 3.576000, + "Standard_D48ls_v5": 3.144000, + "Standard_D48s_v3": 3.816000, + "Standard_D48s_v4": 3.672000, + "Standard_D48s_v5": 3.672000, + "Standard_D4_v2": 0.685000, + "Standard_D4_v2_Promo": 0.685000, + "Standard_D4_v3": 0.318000, + "Standard_D4_v4": 0.306000, + "Standard_D4_v5": 0.306000, + "Standard_D4a_v4": 0.306000, + "Standard_D4ads_v5": 0.330000, + "Standard_D4as_v4": 0.306000, + "Standard_D4as_v5": 0.276000, + "Standard_D4as_v5_Promo": 0.245000, + "Standard_D4d_v4": 0.360000, + "Standard_D4d_v5": 0.360000, + "Standard_D4ds_v4": 0.360000, + "Standard_D4ds_v5": 0.360000, + "Standard_D4lds_v5": 0.298000, + "Standard_D4ls_v5": 0.262000, + "Standard_D4s_v3": 0.318000, + "Standard_D4s_v4": 0.306000, + "Standard_D4s_v5": 0.306000, + "Standard_D5_v2": 1.370000, + "Standard_D5_v2_Promo": 1.370000, + "Standard_D64-16s_v3": 5.088000, + "Standard_D64-32s_v3": 5.088000, + "Standard_D64_v3": 5.088000, + "Standard_D64_v4": 4.896000, + "Standard_D64_v5": 4.896000, + "Standard_D64a_v4": 4.896000, + "Standard_D64ads_v5": 5.280000, + "Standard_D64as_v4": 4.896000, + "Standard_D64as_v5": 7.360000, + "Standard_D64as_v5_Promo": 3.917000, + "Standard_D64d_v4": 5.760000, + "Standard_D64d_v5": 5.760000, + "Standard_D64ds_v4": 5.760000, + "Standard_D64ds_v5": 5.760000, + "Standard_D64lds_v5": 4.768000, + "Standard_D64ls_v5": 4.192000, + "Standard_D64s_v3": 5.088000, + "Standard_D64s_v4": 4.896000, + "Standard_D64s_v5": 4.896000, + "Standard_D8_v3": 0.636000, + "Standard_D8_v4": 0.612000, + "Standard_D8_v5": 0.612000, + "Standard_D8a_v4": 0.612000, + "Standard_D8ads_v5": 1.028000, + "Standard_D8as_v4": 0.612000, + "Standard_D8as_v5": 0.552000, + "Standard_D8as_v5_Promo": 0.490000, + "Standard_D8d_v4": 0.720000, + "Standard_D8d_v5": 0.720000, + "Standard_D8ds_v4": 0.720000, + "Standard_D8ds_v5": 0.720000, + "Standard_D8lds_v5": 0.596000, + "Standard_D8ls_v5": 0.524000, + "Standard_D8s_v3": 0.636000, + "Standard_D8s_v4": 0.612000, + "Standard_D8s_v5": 0.612000, + "Standard_D96_v5": 7.344000, + "Standard_D96a_v4": 7.344000, + "Standard_D96ads_v5": 12.336000, + "Standard_D96as_v4": 7.344000, + "Standard_D96as_v5": 6.624000, + "Standard_D96as_v5_Promo": 5.875000, + "Standard_D96d_v5": 8.640000, + "Standard_D96ds_v5": 8.640000, + "Standard_D96lds_v5": 7.152000, + "Standard_D96ls_v5": 6.288000, + "Standard_D96s_v5": 7.344000, + "Standard_DS1": 0.095000, + "Standard_DS11": 0.235000, + "Standard_DS11-1_v2": 0.235000, + "Standard_DS11_v2": 0.235000, + "Standard_DS11_v2_Promo": 0.235000, + "Standard_DS12": 0.470000, + "Standard_DS12-1_v2": 0.470000, + "Standard_DS12-2_v2": 0.470000, + "Standard_DS12_v2": 0.470000, + "Standard_DS12_v2_Promo": 0.470000, + "Standard_DS13": 0.939000, + "Standard_DS13-2_v2": 0.940000, + "Standard_DS13-4_v2": 0.940000, + "Standard_DS13_v2": 0.940000, + "Standard_DS13_v2_Promo": 0.940000, + "Standard_DS14": 1.879000, + "Standard_DS14-4_v2": 1.879000, + "Standard_DS14-8_v2": 1.879000, + "Standard_DS14_v2": 1.879000, + "Standard_DS14_v2_Promo": 1.879000, + "Standard_DS15_v2": 2.349000, + "Standard_DS15i_v2": 2.349000, + "Standard_DS1_v2": 0.085600, + "Standard_DS2": 0.190000, + "Standard_DS2_v2": 0.171000, + "Standard_DS2_v2_Promo": 0.171000, + "Standard_DS3": 0.380000, + "Standard_DS3_v2": 0.343000, + "Standard_DS3_v2_Promo": 0.343000, + "Standard_DS4": 0.760000, + "Standard_DS4_v2": 0.685000, + "Standard_DS4_v2_Promo": 0.685000, + "Standard_DS5_v2": 1.370000, + "Standard_DS5_v2_Promo": 1.370000, + "Standard_E104i_v5": 11.497000, + "Standard_E104id_v5": 13.042000, + "Standard_E104ids_v5": 13.042000, + "Standard_E104is_v5": 11.497000, + "Standard_E112iads_v5": 12.813000, + "Standard_E112ias_v5": 11.150000, + "Standard_E112ibds_v5": 16.416000, + "Standard_E112ibs_v5": 14.753000, + "Standard_E16-4ads_v5": 1.664000, + "Standard_E16-4as_v4": 1.608000, + "Standard_E16-4as_v5": 1.448000, + "Standard_E16-4ds_v4": 1.824000, + "Standard_E16-4ds_v5": 1.824000, + "Standard_E16-4s_v3": 1.879000, + "Standard_E16-4s_v4": 1.608000, + "Standard_E16-4s_v5": 1.608000, + "Standard_E16-8ads_v5": 2.400000, + "Standard_E16-8as_v4": 1.608000, + "Standard_E16-8as_v5": 1.448000, + "Standard_E16-8ds_v4": 1.824000, + "Standard_E16-8ds_v5": 1.824000, + "Standard_E16-8s_v3": 1.879000, + "Standard_E16-8s_v4": 1.608000, + "Standard_E16-8s_v5": 1.608000, + "Standard_E16_v3": 1.879000, + "Standard_E16_v4": 1.608000, + "Standard_E16_v5": 1.608000, + "Standard_E16a_v4": 1.608000, + "Standard_E16ads_v5": 1.710000, + "Standard_E16as_v4": 1.608000, + "Standard_E16as_v5": 1.494000, + "Standard_E16bds_v5": 2.132000, + "Standard_E16bs_v5": 1.916000, + "Standard_E16d_v4": 1.824000, + "Standard_E16d_v5": 1.824000, + "Standard_E16ds_v4": 1.824000, + "Standard_E16ds_v5": 1.824000, + "Standard_E16s_v3": 1.879000, + "Standard_E16s_v4": 1.608000, + "Standard_E16s_v5": 1.608000, + "Standard_E20_v3": 2.800000, + "Standard_E20_v4": 2.010000, + "Standard_E20_v5": 2.010000, + "Standard_E20a_v4": 2.010000, + "Standard_E20ads_v5": 2.126000, + "Standard_E20as_v4": 2.010000, + "Standard_E20as_v5": 1.856000, + "Standard_E20d_v4": 2.280000, + "Standard_E20d_v5": 2.280000, + "Standard_E20ds_v4": 2.280000, + "Standard_E20ds_v5": 2.280000, + "Standard_E20s_v3": 2.800000, + "Standard_E20s_v4": 2.010000, + "Standard_E20s_v5": 2.010000, + "Standard_E2_v3": 0.235000, + "Standard_E2_v4": 0.201000, + "Standard_E2_v5": 0.201000, + "Standard_E2a_v4": 0.201000, + "Standard_E2ads_v5": 0.208000, + "Standard_E2as_v4": 0.201000, + "Standard_E2as_v5": 0.181000, + "Standard_E2bds_v5": 0.267000, + "Standard_E2bs_v5": 0.240000, + "Standard_E2d_v4": 0.228000, + "Standard_E2d_v5": 0.228000, + "Standard_E2ds_v4": 0.228000, + "Standard_E2ds_v5": 0.228000, + "Standard_E2s_v3": 0.235000, + "Standard_E2s_v4": 0.201000, + "Standard_E2s_v5": 0.201000, + "Standard_E32-16ads_v5": 4.800000, + "Standard_E32-16as_v4": 3.216000, + "Standard_E32-16as_v5": 2.896000, + "Standard_E32-16ds_v4": 3.648000, + "Standard_E32-16ds_v5": 3.648000, + "Standard_E32-16s_v3": 3.758000, + "Standard_E32-16s_v4": 3.216000, + "Standard_E32-16s_v5": 3.216000, + "Standard_E32-8ads_v5": 4.800000, + "Standard_E32-8as_v4": 3.216000, + "Standard_E32-8as_v5": 2.896000, + "Standard_E32-8ds_v4": 3.648000, + "Standard_E32-8ds_v5": 3.648000, + "Standard_E32-8s_v3": 3.758000, + "Standard_E32-8s_v4": 3.216000, + "Standard_E32-8s_v5": 3.216000, + "Standard_E32_v3": 3.758000, + "Standard_E32_v4": 3.216000, + "Standard_E32_v5": 3.216000, + "Standard_E32a_v4": 3.216000, + "Standard_E32ads_v5": 3.374000, + "Standard_E32as_v4": 3.216000, + "Standard_E32as_v5": 2.896000, + "Standard_E32bds_v5": 4.264000, + "Standard_E32bs_v5": 3.832000, + "Standard_E32d_v4": 3.648000, + "Standard_E32d_v5": 3.648000, + "Standard_E32ds_v4": 3.648000, + "Standard_E32ds_v5": 3.648000, + "Standard_E32s_v3": 3.758000, + "Standard_E32s_v4": 3.216000, + "Standard_E32s_v5": 3.216000, + "Standard_E4-2ads_v5": 0.600000, + "Standard_E4-2as_v4": 0.402000, + "Standard_E4-2as_v5": 0.546000, + "Standard_E4-2ds_v4": 0.456000, + "Standard_E4-2ds_v5": 0.456000, + "Standard_E4-2s_v3": 0.470000, + "Standard_E4-2s_v4": 0.402000, + "Standard_E4-2s_v5": 0.402000, + "Standard_E48_v3": 5.637000, + "Standard_E48_v4": 4.824000, + "Standard_E48_v5": 4.824000, + "Standard_E48a_v4": 4.824000, + "Standard_E48ads_v5": 4.992000, + "Standard_E48as_v4": 4.824000, + "Standard_E48as_v5": 4.344000, + "Standard_E48bds_v5": 6.396000, + "Standard_E48bs_v5": 5.748000, + "Standard_E48d_v4": 5.472000, + "Standard_E48d_v5": 5.472000, + "Standard_E48ds_v4": 5.472000, + "Standard_E48ds_v5": 5.472000, + "Standard_E48s_v3": 5.637000, + "Standard_E48s_v4": 4.824000, + "Standard_E48s_v5": 4.824000, + "Standard_E4_v3": 0.470000, + "Standard_E4_v4": 0.402000, + "Standard_E4_v5": 0.402000, + "Standard_E4a_v4": 0.402000, + "Standard_E4ads_v5": 0.462000, + "Standard_E4as_v4": 0.402000, + "Standard_E4as_v5": 0.362000, + "Standard_E4bds_v5": 0.533000, + "Standard_E4bs_v5": 0.479000, + "Standard_E4d_v4": 0.456000, + "Standard_E4d_v5": 0.456000, + "Standard_E4ds_v4": 0.456000, + "Standard_E4ds_v5": 0.456000, + "Standard_E4s_v3": 0.470000, + "Standard_E4s_v4": 0.402000, + "Standard_E4s_v5": 0.402000, + "Standard_E64-16ads_v5": 9.600000, + "Standard_E64-16as_v4": 6.432000, + "Standard_E64-16as_v5": 8.736000, + "Standard_E64-16ds_v4": 7.296000, + "Standard_E64-16ds_v5": 7.296000, + "Standard_E64-16s_v3": 6.764000, + "Standard_E64-16s_v4": 6.432000, + "Standard_E64-16s_v5": 6.432000, + "Standard_E64-32ads_v5": 9.600000, + "Standard_E64-32as_v4": 6.432000, + "Standard_E64-32as_v5": 5.792000, + "Standard_E64-32ds_v4": 7.296000, + "Standard_E64-32ds_v5": 7.296000, + "Standard_E64-32s_v3": 6.764000, + "Standard_E64-32s_v4": 6.432000, + "Standard_E64-32s_v5": 6.432000, + "Standard_E64_v3": 6.764000, + "Standard_E64_v4": 6.432000, + "Standard_E64_v5": 6.432000, + "Standard_E64a_v4": 6.432000, + "Standard_E64ads_v5": 6.702000, + "Standard_E64as_v4": 6.432000, + "Standard_E64as_v5": 5.792000, + "Standard_E64bds_v5": 8.528000, + "Standard_E64bs_v5": 7.664000, + "Standard_E64d_v4": 7.296000, + "Standard_E64d_v5": 7.296000, + "Standard_E64ds_v4": 7.296000, + "Standard_E64ds_v5": 7.296000, + "Standard_E64i_v3": 6.764000, + "Standard_E64is_v3": 6.764000, + "Standard_E64s_v3": 6.764000, + "Standard_E64s_v4": 6.432000, + "Standard_E64s_v5": 6.432000, + "Standard_E8-2ads_v5": 1.200000, + "Standard_E8-2as_v4": 0.804000, + "Standard_E8-2as_v5": 0.724000, + "Standard_E8-2ds_v4": 0.912000, + "Standard_E8-2ds_v5": 0.912000, + "Standard_E8-2s_v3": 0.940000, + "Standard_E8-2s_v4": 0.804000, + "Standard_E8-2s_v5": 0.804000, + "Standard_E8-4ads_v5": 1.200000, + "Standard_E8-4as_v4": 0.804000, + "Standard_E8-4as_v5": 1.092000, + "Standard_E8-4ds_v4": 0.912000, + "Standard_E8-4ds_v5": 0.912000, + "Standard_E8-4s_v3": 0.940000, + "Standard_E8-4s_v4": 0.804000, + "Standard_E8-4s_v5": 0.804000, + "Standard_E80ids_v4": 9.120000, + "Standard_E80is_v4": 8.040000, + "Standard_E8_v3": 0.940000, + "Standard_E8_v4": 0.804000, + "Standard_E8_v5": 0.804000, + "Standard_E8a_v4": 0.804000, + "Standard_E8ads_v5": 0.878000, + "Standard_E8as_v4": 0.804000, + "Standard_E8as_v5": 0.770000, + "Standard_E8bds_v5": 1.066000, + "Standard_E8bs_v5": 0.958000, + "Standard_E8d_v4": 0.912000, + "Standard_E8d_v5": 0.912000, + "Standard_E8ds_v4": 0.912000, + "Standard_E8ds_v5": 0.912000, + "Standard_E8s_v3": 0.940000, + "Standard_E8s_v4": 0.804000, + "Standard_E8s_v5": 0.804000, + "Standard_E96-24ads_v5": 9.984000, + "Standard_E96-24as_v4": 9.648000, + "Standard_E96-24as_v5": 13.104000, + "Standard_E96-24ds_v5": 10.944000, + "Standard_E96-24s_v5": 9.648000, + "Standard_E96-48ads_v5": 9.984000, + "Standard_E96-48as_v4": 9.648000, + "Standard_E96-48as_v5": 13.104000, + "Standard_E96-48ds_v5": 10.944000, + "Standard_E96-48s_v5": 9.648000, + "Standard_E96_v5": 9.648000, + "Standard_E96a_v4": 9.648000, + "Standard_E96ads_v5": 9.984000, + "Standard_E96as_v4": 9.648000, + "Standard_E96as_v5": 8.688000, + "Standard_E96bds_v5": 12.792000, + "Standard_E96bs_v5": 11.496000, + "Standard_E96d_v5": 10.944000, + "Standard_E96ds_v5": 10.944000, + "Standard_E96iads_v5": 10.982000, + "Standard_E96ias_v5": 9.557000, + "Standard_E96s_v5": 9.648000, + "Standard_F1": 0.071800, + "Standard_F16": 1.148000, + "Standard_F16s": 1.148000, + "Standard_F16s_v2": 1.048000, + "Standard_F1s": 0.071800, + "Standard_F2": 0.144000, + "Standard_F2s": 0.144000, + "Standard_F2s_v2": 0.131000, + "Standard_F32s_v2": 2.096000, + "Standard_F4": 0.287000, + "Standard_F48s_v2": 3.144000, + "Standard_F4s": 0.287000, + "Standard_F4s_v2": 0.262000, + "Standard_F64s_v2": 4.192000, + "Standard_F72s_v2": 4.716000, + "Standard_F8": 0.574000, + "Standard_F8s": 0.574000, + "Standard_F8s_v2": 0.524000, + "Standard_FX12mds": 1.668000, + "Standard_FX24mds": 3.336000, + "Standard_FX36mds": 5.004000, + "Standard_FX48mds": 6.672000, + "Standard_FX4mds": 0.556000, + "Standard_HB120-16rs_v2": 7.200000, + "Standard_HB120-32rs_v2": 7.200000, + "Standard_HB120-64rs_v2": 7.200000, + "Standard_HB120-96rs_v2": 7.200000, + "Standard_HB120rs_v2": 7.200000, + "Standard_HC44-16rs": 6.336000, + "Standard_HC44-32rs": 6.336000, + "Standard_HC44rs": 6.336000, + "Standard_L16as_v3": 1.992000, + "Standard_L16s_v3": 2.224000, + "Standard_L32as_v3": 3.984000, + "Standard_L32s_v3": 4.448000, + "Standard_L48as_v3": 5.976000, + "Standard_L48s_v3": 6.672000, + "Standard_L64as_v3": 7.968000, + "Standard_L64s_v3": 8.896000, + "Standard_L80as_v3": 9.960000, + "Standard_L80s_v3": 11.120000, + "Standard_L8as_v3": 0.996000, + "Standard_L8s_v3": 1.112000, + "Standard_M128": 26.009000, + "Standard_M128-32ms": 52.042000, + "Standard_M128-64ms": 52.042000, + "Standard_M128dms_v2": 52.046000, + "Standard_M128ds_v2": 26.013000, + "Standard_M128m": 52.042000, + "Standard_M128ms": 52.042000, + "Standard_M128ms_v2": 51.225000, + "Standard_M128s": 26.009000, + "Standard_M128s_v2": 25.192000, + "Standard_M16-4ms": 5.992000, + "Standard_M16-8ms": 5.992000, + "Standard_M16ms": 5.992000, + "Standard_M16s": 4.655000, + "Standard_M192idms_v2": 62.524000, + "Standard_M192ids_v2": 31.262000, + "Standard_M192ims_v2": 61.704000, + "Standard_M192is_v2": 30.442000, + "Standard_M208-104ms_v2": 87.009000, + "Standard_M208-104s_v2": 43.504000, + "Standard_M208-52ms_v2": 87.009000, + "Standard_M208-52s_v2": 43.504000, + "Standard_M208ms_v2": 87.009000, + "Standard_M208s_v2": 43.504000, + "Standard_M32-16ms": 11.985000, + "Standard_M32-8ms": 11.985000, + "Standard_M32dms_v2": 11.985000, + "Standard_M32ls": 4.582435, + "Standard_M32ms": 11.985000, + "Standard_M32ms_v2": 11.780000, + "Standard_M32s": 6.503000, + "Standard_M32ts": 4.317665, + "Standard_M416-104ms_v2": 193.340000, + "Standard_M416-104s_v2": 96.681000, + "Standard_M416-208ms_v2": 193.340000, + "Standard_M416-208s_v2": 96.681000, + "Standard_M416is_v2": 96.681000, + "Standard_M416ms_v2": 193.340000, + "Standard_M416s_8_v2": 105.120000, + "Standard_M416s_v2": 96.681000, + "Standard_M64": 13.005000, + "Standard_M64-16ms": 20.158000, + "Standard_M64-32ms": 20.158000, + "Standard_M64dms_v2": 20.163000, + "Standard_M64ds_v2": 13.005000, + "Standard_M64ls": 8.636925, + "Standard_M64m": 20.158000, + "Standard_M64ms": 20.158000, + "Standard_M64ms_v2": 19.753000, + "Standard_M64s": 13.005000, + "Standard_M64s_v2": 12.594000, + "Standard_M8-2ms": 2.996000, + "Standard_M8-4ms": 2.996000, + "Standard_M8ms": 2.996000, + "Standard_NC12s_v3": 12.240000, + "Standard_NC16as_T4_v3": 2.046000, + "Standard_NC24rs_v3": 26.928000, + "Standard_NC24s_v3": 24.480000, + "Standard_NC4as_T4_v3": 0.894000, + "Standard_NC64as_T4_v3": 7.383000, + "Standard_NC6s_v3": 6.120000, + "Standard_NC8as_T4_v3": 1.277000, + "Standard_ND96ams_A100_v4": 65.540000, + "Standard_ND96amsr_A100_v4": 65.540000, + "Standard_NV12ads_A10_v5": 1.816000, + "Standard_NV12s_v3": 1.938000, + "Standard_NV18ads_A10_v5": 3.200000, + "Standard_NV24s_v3": 3.876000, + "Standard_NV36adms_A10_v5": 9.040000, + "Standard_NV36ads_A10_v5": 6.400000, + "Standard_NV48s_v3": 7.752000, + "Standard_NV6ads_A10_v5": 0.908000, + "Standard_NV72ads_A10_v5": 13.040000, + } + // brazilsoutheast + initialOnDemandPrices["brazilsoutheast"] = map[string]float64{ + "Basic_A0": 0.028600, + "Basic_A1": 0.053300, + "Basic_A2": 0.146000, + "Basic_A3": 0.302000, + "Basic_A4": 0.603000, + "Dadsv5_Type1": 13.213000, + "Dasv4_Type1": 10.940000, + "Dasv5_Type1": 11.051000, + "Ddsv4_Type 1": 8.240000, + "Ddsv4_Type2": 9.785000, + "Ddsv5_Type1": 12.355000, + "Dsv3_Type3": 4.225000, + "Dsv3_Type4": 5.281000, + "Dsv4_Type1": 8.751000, + "Dsv4_Type2": 10.939000, + "Dsv5_Type1": 10.940000, + "Eadsv5_Type1": 14.277000, + "Easv4_Type1": 13.797000, + "Easv4_Type2": 13.797000, + "Easv5_Type1": 12.424000, + "Ebdsv5-Type1": 12.195000, + "Ebsv5-Type1": 10.960000, + "Edsv4_Type 1": 10.512000, + "Edsv4_Type2": 12.483000, + "Edsv5_Type1": 15.650000, + "Esv3_Type3": 4.297000, + "Esv3_Type4": 5.822000, + "Esv4_Type1": 9.198000, + "Esv4_Type2": 12.072000, + "Esv5_Type1": 13.797000, + "FXmds Type1": 9.541000, + "Fsv2 Type3": 8.992000, + "Fsv2_Type2": 8.992000, + "Fsv2_Type4": 8.992000, + "Lasv3_Type1": 14.243000, + "Lsv3_Type1": 15.902000, + "Mdmsv2MedMem _Type1": 66.058000, + "Mdsv2MedMem_Type1": 33.017000, + "Msmv2MedMem Type1": 65.016000, + "Msmv2_Type1": 283.560000, + "Msv2MedMem Type1": 31.975000, + "Msv2_Type1": 141.800000, + "Standard_A1_v2": 0.079300, + "Standard_A2_v2": 0.168000, + "Standard_A2m_v2": 0.270000, + "Standard_A4_v2": 0.351000, + "Standard_A4m_v2": 0.568000, + "Standard_A8_v2": 0.737000, + "Standard_A8m_v2": 1.192000, + "Standard_B12ms": 1.048000, + "Standard_B16als_v2": 1.188000, + "Standard_B16as_v2": 1.332000, + "Standard_B16ls_v2": 1.238000, + "Standard_B16ms": 1.398000, + "Standard_B16s_v2": 1.471000, + "Standard_B1ls": 0.010900, + "Standard_B1ms": 0.043700, + "Standard_B1s": 0.021800, + "Standard_B20ms": 1.747000, + "Standard_B2als_v2": 0.078700, + "Standard_B2as_v2": 0.157000, + "Standard_B2ats_v2": 0.019700, + "Standard_B2ls_v2": 0.087400, + "Standard_B2ms": 0.175000, + "Standard_B2s": 0.087400, + "Standard_B2s_v2": 0.175000, + "Standard_B2ts_v2": 0.031000, + "Standard_B32als_v2": 2.376000, + "Standard_B32as_v2": 2.664000, + "Standard_B32ls_v2": 2.475000, + "Standard_B32s_v2": 2.796000, + "Standard_B4als_v2": 0.279000, + "Standard_B4as_v2": 0.333000, + "Standard_B4ls_v2": 0.328000, + "Standard_B4ms": 0.349000, + "Standard_B4s_v2": 0.349000, + "Standard_B8als_v2": 0.594000, + "Standard_B8as_v2": 0.629000, + "Standard_B8ls_v2": 0.656000, + "Standard_B8ms": 0.699000, + "Standard_B8s_v2": 0.736000, + "Standard_D1": 0.124000, + "Standard_D11": 0.306000, + "Standard_D11_v2": 0.306000, + "Standard_D12": 0.611000, + "Standard_D12_v2": 0.611000, + "Standard_D13": 1.221000, + "Standard_D13_v2": 1.222000, + "Standard_D14": 2.443000, + "Standard_D14_v2": 2.443000, + "Standard_D15_v2": 3.054000, + "Standard_D15i_v2": 3.054000, + "Standard_D16_v3": 1.654000, + "Standard_D16_v4": 1.591000, + "Standard_D16_v5": 1.591000, + "Standard_D16a_v4": 1.591000, + "Standard_D16ads_v5": 1.716000, + "Standard_D16as_v4": 1.591000, + "Standard_D16as_v5": 2.171000, + "Standard_D16d_v4": 1.873000, + "Standard_D16d_v5": 1.872000, + "Standard_D16ds_v4": 1.873000, + "Standard_D16ds_v5": 1.872000, + "Standard_D16lds_v5": 1.550000, + "Standard_D16ls_v5": 1.362000, + "Standard_D16s_v3": 1.654000, + "Standard_D16s_v4": 1.591000, + "Standard_D16s_v5": 1.591000, + "Standard_D1_v2": 0.111000, + "Standard_D2": 0.247000, + "Standard_D2_v2": 0.223000, + "Standard_D2_v3": 0.207000, + "Standard_D2_v4": 0.199000, + "Standard_D2_v5": 0.199000, + "Standard_D2a_v4": 0.199000, + "Standard_D2ads_v5": 0.307000, + "Standard_D2as_v4": 0.199000, + "Standard_D2as_v5": 0.271000, + "Standard_D2d_v4": 0.234000, + "Standard_D2d_v5": 0.234000, + "Standard_D2ds_v4": 0.234000, + "Standard_D2ds_v5": 0.234000, + "Standard_D2lds_v5": 0.194000, + "Standard_D2ls_v5": 0.170000, + "Standard_D2s_v3": 0.207000, + "Standard_D2s_v4": 0.199000, + "Standard_D2s_v5": 0.199000, + "Standard_D3": 0.494000, + "Standard_D32-16s_v3": 3.307000, + "Standard_D32-8s_v3": 3.307000, + "Standard_D32_v3": 3.307000, + "Standard_D32_v4": 3.182000, + "Standard_D32_v5": 3.182000, + "Standard_D32a_v4": 3.182000, + "Standard_D32ads_v5": 3.432000, + "Standard_D32as_v4": 3.182000, + "Standard_D32as_v5": 4.342000, + "Standard_D32d_v4": 3.746000, + "Standard_D32d_v5": 3.744000, + "Standard_D32ds_v4": 3.746000, + "Standard_D32ds_v5": 3.744000, + "Standard_D32lds_v5": 3.099000, + "Standard_D32ls_v5": 2.725000, + "Standard_D32s_v3": 3.307000, + "Standard_D32s_v4": 3.182000, + "Standard_D32s_v5": 3.182000, + "Standard_D3_v2": 0.445000, + "Standard_D4": 0.988000, + "Standard_D48_v3": 4.961000, + "Standard_D48_v4": 4.773000, + "Standard_D48_v5": 4.774000, + "Standard_D48a_v4": 4.774000, + "Standard_D48ads_v5": 5.148000, + "Standard_D48as_v4": 4.774000, + "Standard_D48as_v5": 4.306000, + "Standard_D48d_v4": 5.618000, + "Standard_D48d_v5": 5.616000, + "Standard_D48ds_v4": 5.618000, + "Standard_D48ds_v5": 5.616000, + "Standard_D48lds_v5": 4.649000, + "Standard_D48ls_v5": 4.087000, + "Standard_D48s_v3": 4.961000, + "Standard_D48s_v4": 4.773000, + "Standard_D48s_v5": 4.774000, + "Standard_D4_v2": 0.890000, + "Standard_D4_v3": 0.413000, + "Standard_D4_v4": 0.398000, + "Standard_D4_v5": 0.398000, + "Standard_D4a_v4": 0.398000, + "Standard_D4ads_v5": 0.429000, + "Standard_D4as_v4": 0.398000, + "Standard_D4as_v5": 0.543000, + "Standard_D4d_v4": 0.468000, + "Standard_D4d_v5": 0.468000, + "Standard_D4ds_v4": 0.468000, + "Standard_D4ds_v5": 0.468000, + "Standard_D4lds_v5": 0.387000, + "Standard_D4ls_v5": 0.341000, + "Standard_D4s_v3": 0.413000, + "Standard_D4s_v4": 0.398000, + "Standard_D4s_v5": 0.398000, + "Standard_D5_v2": 1.780000, + "Standard_D64-16s_v3": 6.614000, + "Standard_D64-32s_v3": 6.614000, + "Standard_D64_v3": 6.614000, + "Standard_D64_v4": 6.364000, + "Standard_D64_v5": 6.365000, + "Standard_D64a_v4": 6.365000, + "Standard_D64ads_v5": 6.864000, + "Standard_D64as_v4": 6.365000, + "Standard_D64as_v5": 8.685000, + "Standard_D64d_v4": 7.491000, + "Standard_D64d_v5": 7.488000, + "Standard_D64ds_v4": 7.491000, + "Standard_D64ds_v5": 7.488000, + "Standard_D64lds_v5": 6.198000, + "Standard_D64ls_v5": 5.450000, + "Standard_D64s_v3": 6.614000, + "Standard_D64s_v4": 6.364000, + "Standard_D64s_v5": 6.365000, + "Standard_D8_v3": 0.827000, + "Standard_D8_v4": 0.796000, + "Standard_D8_v5": 0.796000, + "Standard_D8a_v4": 0.796000, + "Standard_D8ads_v5": 0.858000, + "Standard_D8as_v4": 0.796000, + "Standard_D8as_v5": 1.086000, + "Standard_D8d_v4": 0.936000, + "Standard_D8d_v5": 0.936000, + "Standard_D8ds_v4": 0.936000, + "Standard_D8ds_v5": 0.936000, + "Standard_D8lds_v5": 0.775000, + "Standard_D8ls_v5": 0.681000, + "Standard_D8s_v3": 0.827000, + "Standard_D8s_v4": 0.796000, + "Standard_D8s_v5": 0.796000, + "Standard_D96_v5": 9.547000, + "Standard_D96a_v4": 9.547000, + "Standard_D96ads_v5": 14.712000, + "Standard_D96as_v4": 9.547000, + "Standard_D96as_v5": 13.027000, + "Standard_D96d_v5": 11.232000, + "Standard_D96ds_v5": 11.232000, + "Standard_D96lds_v5": 9.298000, + "Standard_D96ls_v5": 8.174000, + "Standard_D96s_v5": 9.547000, + "Standard_DS1": 0.124000, + "Standard_DS11": 0.306000, + "Standard_DS11-1_v2": 0.306000, + "Standard_DS11_v2": 0.306000, + "Standard_DS12": 0.611000, + "Standard_DS12-1_v2": 0.611000, + "Standard_DS12-2_v2": 0.611000, + "Standard_DS12_v2": 0.611000, + "Standard_DS13": 1.221000, + "Standard_DS13-2_v2": 1.222000, + "Standard_DS13-4_v2": 1.222000, + "Standard_DS13_v2": 1.222000, + "Standard_DS14": 2.443000, + "Standard_DS14-4_v2": 2.443000, + "Standard_DS14-8_v2": 2.443000, + "Standard_DS14_v2": 2.443000, + "Standard_DS15_v2": 3.054000, + "Standard_DS15i_v2": 3.054000, + "Standard_DS1_v2": 0.111000, + "Standard_DS2": 0.247000, + "Standard_DS2_v2": 0.223000, + "Standard_DS3": 0.494000, + "Standard_DS3_v2": 0.445000, + "Standard_DS4": 0.988000, + "Standard_DS4_v2": 0.890000, + "Standard_DS5_v2": 1.780000, + "Standard_E104i_v5": 14.946000, + "Standard_E104id_v5": 16.954000, + "Standard_E104ids_v5": 16.954000, + "Standard_E104is_v5": 14.946000, + "Standard_E112iads_v5": 16.657000, + "Standard_E112ias_v5": 14.494000, + "Standard_E112ibds_v5": 21.341000, + "Standard_E112ibs_v5": 19.179000, + "Standard_E16-4ads_v5": 2.899000, + "Standard_E16-4as_v4": 2.090000, + "Standard_E16-4as_v5": 1.882000, + "Standard_E16-4ds_v4": 2.389000, + "Standard_E16-4ds_v5": 2.371000, + "Standard_E16-4s_v3": 2.444000, + "Standard_E16-4s_v4": 2.090000, + "Standard_E16-4s_v5": 2.090000, + "Standard_E16-8ads_v5": 2.163000, + "Standard_E16-8as_v4": 2.090000, + "Standard_E16-8as_v5": 1.882000, + "Standard_E16-8ds_v4": 2.389000, + "Standard_E16-8ds_v5": 2.371000, + "Standard_E16-8s_v3": 2.444000, + "Standard_E16-8s_v4": 2.090000, + "Standard_E16-8s_v5": 2.090000, + "Standard_E16_v3": 2.444000, + "Standard_E16_v4": 2.090000, + "Standard_E16_v5": 2.090000, + "Standard_E16a_v4": 2.090000, + "Standard_E16ads_v5": 2.209000, + "Standard_E16as_v4": 2.090000, + "Standard_E16as_v5": 1.928000, + "Standard_E16bds_v5": 2.772000, + "Standard_E16bs_v5": 2.491000, + "Standard_E16d_v4": 2.389000, + "Standard_E16d_v5": 2.371000, + "Standard_E16ds_v4": 2.389000, + "Standard_E16ds_v5": 2.371000, + "Standard_E16s_v3": 2.444000, + "Standard_E16s_v4": 2.090000, + "Standard_E16s_v5": 2.090000, + "Standard_E20_v3": 3.640000, + "Standard_E20_v4": 2.613000, + "Standard_E20_v5": 2.613000, + "Standard_E20a_v4": 2.613000, + "Standard_E20ads_v5": 2.750000, + "Standard_E20as_v4": 2.613000, + "Standard_E20as_v5": 2.353000, + "Standard_E20d_v4": 2.986000, + "Standard_E20d_v5": 2.964000, + "Standard_E20ds_v4": 2.986000, + "Standard_E20ds_v5": 2.964000, + "Standard_E20s_v3": 3.640000, + "Standard_E20s_v4": 2.613000, + "Standard_E20s_v5": 2.613000, + "Standard_E2_v3": 0.306000, + "Standard_E2_v4": 0.261000, + "Standard_E2_v5": 0.261000, + "Standard_E2a_v4": 0.261000, + "Standard_E2ads_v5": 0.316000, + "Standard_E2as_v4": 0.261000, + "Standard_E2as_v5": 0.235000, + "Standard_E2bds_v5": 0.346000, + "Standard_E2bs_v5": 0.311000, + "Standard_E2d_v4": 0.299000, + "Standard_E2d_v5": 0.296000, + "Standard_E2ds_v4": 0.299000, + "Standard_E2ds_v5": 0.296000, + "Standard_E2s_v3": 0.306000, + "Standard_E2s_v4": 0.261000, + "Standard_E2s_v5": 0.261000, + "Standard_E32-16ads_v5": 5.798000, + "Standard_E32-16as_v4": 4.181000, + "Standard_E32-16as_v5": 5.237000, + "Standard_E32-16ds_v4": 4.778000, + "Standard_E32-16ds_v5": 4.742000, + "Standard_E32-16s_v3": 4.888000, + "Standard_E32-16s_v4": 4.181000, + "Standard_E32-16s_v5": 4.181000, + "Standard_E32-8ads_v5": 4.326000, + "Standard_E32-8as_v4": 4.181000, + "Standard_E32-8as_v5": 3.765000, + "Standard_E32-8ds_v4": 4.778000, + "Standard_E32-8ds_v5": 4.742000, + "Standard_E32-8s_v3": 4.888000, + "Standard_E32-8s_v4": 4.181000, + "Standard_E32-8s_v5": 4.181000, + "Standard_E32_v3": 4.888000, + "Standard_E32_v4": 4.181000, + "Standard_E32_v5": 4.181000, + "Standard_E32a_v4": 4.181000, + "Standard_E32ads_v5": 4.372000, + "Standard_E32as_v4": 4.181000, + "Standard_E32as_v5": 3.765000, + "Standard_E32bds_v5": 5.543000, + "Standard_E32bs_v5": 4.982000, + "Standard_E32d_v4": 4.778000, + "Standard_E32d_v5": 4.742000, + "Standard_E32ds_v4": 4.778000, + "Standard_E32ds_v5": 4.742000, + "Standard_E32s_v3": 4.888000, + "Standard_E32s_v4": 4.181000, + "Standard_E32s_v5": 4.181000, + "Standard_E4-2ads_v5": 0.725000, + "Standard_E4-2as_v4": 0.523000, + "Standard_E4-2as_v5": 0.655000, + "Standard_E4-2ds_v4": 0.597000, + "Standard_E4-2ds_v5": 0.593000, + "Standard_E4-2s_v3": 0.611000, + "Standard_E4-2s_v4": 0.523000, + "Standard_E4-2s_v5": 0.523000, + "Standard_E48_v3": 7.332000, + "Standard_E48_v4": 6.271000, + "Standard_E48_v5": 6.271000, + "Standard_E48a_v4": 6.271000, + "Standard_E48ads_v5": 6.536000, + "Standard_E48as_v4": 6.271000, + "Standard_E48as_v5": 5.693000, + "Standard_E48bds_v5": 8.315000, + "Standard_E48bs_v5": 7.472000, + "Standard_E48d_v4": 7.167000, + "Standard_E48d_v5": 7.114000, + "Standard_E48ds_v4": 7.167000, + "Standard_E48ds_v5": 7.114000, + "Standard_E48s_v3": 7.332000, + "Standard_E48s_v4": 6.271000, + "Standard_E48s_v5": 6.271000, + "Standard_E4_v3": 0.611000, + "Standard_E4_v4": 0.523000, + "Standard_E4_v5": 0.523000, + "Standard_E4a_v4": 0.523000, + "Standard_E4ads_v5": 0.587000, + "Standard_E4as_v4": 0.523000, + "Standard_E4as_v5": 0.517000, + "Standard_E4bds_v5": 0.693000, + "Standard_E4bs_v5": 0.623000, + "Standard_E4d_v4": 0.597000, + "Standard_E4d_v5": 0.593000, + "Standard_E4ds_v4": 0.597000, + "Standard_E4ds_v5": 0.593000, + "Standard_E4s_v3": 0.611000, + "Standard_E4s_v4": 0.523000, + "Standard_E4s_v5": 0.523000, + "Standard_E64-16ads_v5": 8.653000, + "Standard_E64-16as_v4": 8.362000, + "Standard_E64-16as_v5": 10.474000, + "Standard_E64-16ds_v4": 9.556000, + "Standard_E64-16ds_v5": 9.485000, + "Standard_E64-16s_v3": 8.798000, + "Standard_E64-16s_v4": 8.362000, + "Standard_E64-16s_v5": 8.362000, + "Standard_E64-32ads_v5": 11.597000, + "Standard_E64-32as_v4": 8.362000, + "Standard_E64-32as_v5": 10.474000, + "Standard_E64-32ds_v4": 9.556000, + "Standard_E64-32ds_v5": 9.485000, + "Standard_E64-32s_v3": 8.798000, + "Standard_E64-32s_v4": 8.362000, + "Standard_E64-32s_v5": 8.362000, + "Standard_E64_v3": 8.798000, + "Standard_E64_v4": 8.362000, + "Standard_E64_v5": 8.362000, + "Standard_E64a_v4": 8.362000, + "Standard_E64ads_v5": 8.653000, + "Standard_E64as_v4": 8.362000, + "Standard_E64as_v5": 7.576000, + "Standard_E64bds_v5": 11.086000, + "Standard_E64bs_v5": 9.963000, + "Standard_E64d_v4": 9.556000, + "Standard_E64d_v5": 9.485000, + "Standard_E64ds_v4": 9.556000, + "Standard_E64ds_v5": 9.485000, + "Standard_E64i_v3": 8.798000, + "Standard_E64is_v3": 8.798000, + "Standard_E64s_v3": 8.798000, + "Standard_E64s_v4": 8.362000, + "Standard_E64s_v5": 8.362000, + "Standard_E8-2ads_v5": 1.450000, + "Standard_E8-2as_v4": 1.045000, + "Standard_E8-2as_v5": 0.941000, + "Standard_E8-2ds_v4": 1.195000, + "Standard_E8-2ds_v5": 1.186000, + "Standard_E8-2s_v3": 1.222000, + "Standard_E8-2s_v4": 1.045000, + "Standard_E8-2s_v5": 1.045000, + "Standard_E8-4ads_v5": 1.082000, + "Standard_E8-4as_v4": 1.045000, + "Standard_E8-4as_v5": 1.309000, + "Standard_E8-4ds_v4": 1.195000, + "Standard_E8-4ds_v5": 1.186000, + "Standard_E8-4s_v3": 1.222000, + "Standard_E8-4s_v4": 1.045000, + "Standard_E8-4s_v5": 1.045000, + "Standard_E80ids_v4": 11.945000, + "Standard_E80is_v4": 10.452000, + "Standard_E8_v3": 1.222000, + "Standard_E8_v4": 1.045000, + "Standard_E8_v5": 1.045000, + "Standard_E8a_v4": 1.045000, + "Standard_E8ads_v5": 1.128000, + "Standard_E8as_v4": 1.045000, + "Standard_E8as_v5": 0.987000, + "Standard_E8bds_v5": 1.386000, + "Standard_E8bs_v5": 1.245000, + "Standard_E8d_v4": 1.195000, + "Standard_E8d_v5": 1.186000, + "Standard_E8ds_v4": 1.195000, + "Standard_E8ds_v5": 1.186000, + "Standard_E8s_v3": 1.222000, + "Standard_E8s_v4": 1.045000, + "Standard_E8s_v5": 1.045000, + "Standard_E96-24ads_v5": 12.979000, + "Standard_E96-24as_v4": 12.542000, + "Standard_E96-24as_v5": 11.294000, + "Standard_E96-24ds_v5": 14.227000, + "Standard_E96-24s_v5": 12.542000, + "Standard_E96-48ads_v5": 12.979000, + "Standard_E96-48as_v4": 12.542000, + "Standard_E96-48as_v5": 15.710000, + "Standard_E96-48ds_v5": 14.227000, + "Standard_E96-48s_v5": 12.542000, + "Standard_E96_v5": 12.542000, + "Standard_E96a_v4": 12.542000, + "Standard_E96ads_v5": 12.979000, + "Standard_E96as_v4": 12.542000, + "Standard_E96as_v5": 11.294000, + "Standard_E96bds_v5": 16.630000, + "Standard_E96bs_v5": 14.945000, + "Standard_E96d_v5": 14.227000, + "Standard_E96ds_v5": 14.227000, + "Standard_E96iads_v5": 14.277000, + "Standard_E96ias_v4": 12.542000, + "Standard_E96ias_v5": 12.424000, + "Standard_E96s_v5": 12.542000, + "Standard_F1": 0.093300, + "Standard_F16": 1.493000, + "Standard_F16s": 1.493000, + "Standard_F16s_v2": 1.362000, + "Standard_F1s": 0.093300, + "Standard_F2": 0.187000, + "Standard_F2s": 0.187000, + "Standard_F2s_v2": 0.170000, + "Standard_F32s_v2": 2.725000, + "Standard_F4": 0.373000, + "Standard_F48s_v2": 4.087000, + "Standard_F4s": 0.373000, + "Standard_F4s_v2": 0.341000, + "Standard_F64s_v2": 5.450000, + "Standard_F72s_v2": 6.131000, + "Standard_F8": 0.747000, + "Standard_F8s": 0.747000, + "Standard_F8s_v2": 0.681000, + "Standard_FX12mds": 2.168000, + "Standard_FX24mds": 4.337000, + "Standard_FX36mds": 6.505000, + "Standard_FX48mds": 8.674000, + "Standard_FX4mds": 0.723000, + "Standard_L16as_v3": 2.590000, + "Standard_L16s_v3": 2.891000, + "Standard_L32as_v3": 5.179000, + "Standard_L32s_v3": 5.782000, + "Standard_L48as_v3": 7.769000, + "Standard_L48s_v3": 8.674000, + "Standard_L64as_v3": 10.358000, + "Standard_L64s_v3": 11.565000, + "Standard_L80as_v3": 12.948000, + "Standard_L80s_v3": 14.456000, + "Standard_L8as_v3": 1.295000, + "Standard_L8s_v3": 1.446000, + "Standard_M128": 30.010000, + "Standard_M128-32ms": 60.048000, + "Standard_M128-64ms": 60.048000, + "Standard_M128dms_v2": 60.053000, + "Standard_M128ds_v2": 30.015000, + "Standard_M128m": 60.048000, + "Standard_M128ms": 60.048000, + "Standard_M128ms_v2": 59.106000, + "Standard_M128s": 30.010000, + "Standard_M128s_v2": 29.068000, + "Standard_M16-4ms": 6.914000, + "Standard_M16-8ms": 6.914000, + "Standard_M16ms": 6.914000, + "Standard_M16s": 5.371000, + "Standard_M192idms_v2": 72.144000, + "Standard_M192ids_v2": 36.072000, + "Standard_M192ims_v2": 71.197000, + "Standard_M192is_v2": 35.125000, + "Standard_M208-104ms_v2": 116.010000, + "Standard_M208-104s_v2": 58.004000, + "Standard_M208-52ms_v2": 116.010000, + "Standard_M208-52s_v2": 58.004000, + "Standard_M208ms_v2": 116.010000, + "Standard_M208s_v2": 58.004000, + "Standard_M32-16ms": 13.828000, + "Standard_M32-8ms": 13.828000, + "Standard_M32dms_v2": 13.829000, + "Standard_M32ls": 5.444335, + "Standard_M32ms": 13.828000, + "Standard_M32ms_v2": 13.592000, + "Standard_M32s": 7.503500, + "Standard_M32ts": 5.129765, + "Standard_M416-104ms_v2": 257.780000, + "Standard_M416-104s_v2": 128.910000, + "Standard_M416-208ms_v2": 257.780000, + "Standard_M416-208s_v2": 128.910000, + "Standard_M416is_v2": 128.910000, + "Standard_M416ms_v2": 257.780000, + "Standard_M416s_8_v2": 136.660000, + "Standard_M416s_v2": 128.910000, + "Standard_M64": 15.005000, + "Standard_M64-16ms": 23.259100, + "Standard_M64-32ms": 23.259100, + "Standard_M64dms_v2": 23.265000, + "Standard_M64ds_v2": 15.005000, + "Standard_M64ls": 10.261425, + "Standard_M64m": 23.259100, + "Standard_M64ms": 23.259100, + "Standard_M64ms_v2": 22.792000, + "Standard_M64s": 15.005000, + "Standard_M64s_v2": 14.532000, + "Standard_M8-2ms": 3.457000, + "Standard_M8-4ms": 3.457000, + "Standard_M8ms": 3.457000, + "Standard_PB12s": 1.660000, + "Standard_PB24s": 3.320000, + "Standard_PB6s": 0.830000, + } + // canadacentral + initialOnDemandPrices["canadacentral"] = map[string]float64{ + "Basic_A0": 0.020000, + "Basic_A1": 0.026000, + "Basic_A2": 0.087000, + "Basic_A3": 0.194000, + "Basic_A4": 0.387000, + "DCdsv3 Type1": 6.653000, + "DCsv2 Type 1": 0.942000, + "DCsv3 Type1": 5.650000, + "Dadsv5_Type1": 7.084000, + "Dasv4_Type1": 5.885000, + "Dasv4_Type2": 5.885000, + "Dasv5_Type1": 5.914000, + "Ddsv4_Type 1": 4.435000, + "Ddsv4_Type2": 5.267000, + "Ddsv5_Type1": 6.653000, + "Dsv3_Type3": 4.884000, + "Dsv3_Type4": 5.886000, + "Dsv4_Type1": 4.708000, + "Dsv4_Type2": 5.650000, + "Dsv5_Type1": 5.885000, + "Eadsv5_Type1": 7.603000, + "Easv4_Type1": 7.286000, + "Easv4_Type2": 7.286000, + "Easv5_Type1": 6.547000, + "Ebdsv5-Type1": 6.494000, + "Ebsv5-Type1": 5.790000, + "Edsv4_Type 1": 5.561000, + "Edsv4_Type2": 6.604000, + "Edsv5_Type1": 8.342000, + "Esv3_Type3": 4.705000, + "Esv3_Type4": 6.375000, + "Esv4_Type1": 4.857000, + "Esv4_Type2": 6.376000, + "Esv5_Type1": 7.286000, + "FXmds Type1": 5.069000, + "Fsv2 Type3": 4.297000, + "Fsv2_Type2": 3.683000, + "Fsv2_Type4": 4.897000, + "Lsv3_Type1": 8.448000, + "Mdmsv2MedMem _Type1": 32.295000, + "Mdsv2MedMem_Type1": 16.141000, + "Ms_Type1": 16.136000, + "Msm_Type1": 32.300000, + "Msmv2MedMem Type1": 31.786000, + "Msmv2_Type1": 130.880000, + "Msv2MedMem Type1": 15.632000, + "Msv2_Type1": 65.446000, + "NVasv4_Type1": 8.612000, + "Standard_A0": 0.022000, + "Standard_A1": 0.056000, + "Standard_A1_v2": 0.043000, + "Standard_A2": 0.111000, + "Standard_A2_v2": 0.091000, + "Standard_A2m_v2": 0.129000, + "Standard_A3": 0.222000, + "Standard_A4": 0.446000, + "Standard_A4_v2": 0.191000, + "Standard_A4m_v2": 0.270000, + "Standard_A5": 0.242000, + "Standard_A6": 0.484000, + "Standard_A7": 0.968000, + "Standard_A8_v2": 0.400000, + "Standard_A8m_v2": 0.568000, + "Standard_B12ms": 0.556000, + "Standard_B16als_v2": 0.666000, + "Standard_B16as_v2": 0.742000, + "Standard_B16ls_v2": 0.731000, + "Standard_B16ms": 0.742000, + "Standard_B16pls_v2": 0.595000, + "Standard_B16ps_v2": 0.662000, + "Standard_B16s_v2": 0.816000, + "Standard_B1ls": 0.005800, + "Standard_B1ms": 0.023000, + "Standard_B1s": 0.011600, + "Standard_B20ms": 0.927000, + "Standard_B2als_v2": 0.051000, + "Standard_B2as_v2": 0.083600, + "Standard_B2ats_v2": 0.010500, + "Standard_B2ls_v2": 0.046400, + "Standard_B2ms": 0.092800, + "Standard_B2pls_v2": 0.046000, + "Standard_B2ps_v2": 0.082800, + "Standard_B2pts_v2": 0.018400, + "Standard_B2s": 0.046400, + "Standard_B2s_v2": 0.102000, + "Standard_B2ts_v2": 0.011600, + "Standard_B32als_v2": 1.332000, + "Standard_B32as_v2": 1.338000, + "Standard_B32ls_v2": 1.315000, + "Standard_B32s_v2": 1.485000, + "Standard_B4als_v2": 0.148000, + "Standard_B4as_v2": 0.186000, + "Standard_B4ls_v2": 0.183000, + "Standard_B4ms": 0.185000, + "Standard_B4pls_v2": 0.149000, + "Standard_B4ps_v2": 0.166000, + "Standard_B4s_v2": 0.186000, + "Standard_B8als_v2": 0.333000, + "Standard_B8as_v2": 0.371000, + "Standard_B8ls_v2": 0.366000, + "Standard_B8ms": 0.371000, + "Standard_B8pls_v2": 0.297000, + "Standard_B8ps_v2": 0.331000, + "Standard_B8s_v2": 0.408000, + "Standard_D1": 0.074700, + "Standard_D11": 0.193000, + "Standard_D11_v2": 0.183000, + "Standard_D11_v2_Promo": 0.199000, + "Standard_D12": 0.387000, + "Standard_D12_v2": 0.366000, + "Standard_D12_v2_Promo": 0.399000, + "Standard_D13": 0.773000, + "Standard_D13_v2": 0.732000, + "Standard_D13_v2_Promo": 0.798000, + "Standard_D14": 1.546000, + "Standard_D14_v2": 1.463000, + "Standard_D14_v2_Promo": 1.596000, + "Standard_D15_v2": 1.829000, + "Standard_D15i_v2": 1.829000, + "Standard_D16_v3": 0.888000, + "Standard_D16_v4": 0.856000, + "Standard_D16_v5": 0.856000, + "Standard_D16a_v4": 0.856000, + "Standard_D16ads_v5": 0.920000, + "Standard_D16as_v4": 0.856000, + "Standard_D16as_v5": 1.504000, + "Standard_D16as_v5_Promo": 0.685000, + "Standard_D16d_v4": 1.008000, + "Standard_D16d_v5": 1.008000, + "Standard_D16ds_v4": 1.008000, + "Standard_D16ds_v5": 1.008000, + "Standard_D16lds_v5": 0.848000, + "Standard_D16ls_v5": 0.744000, + "Standard_D16pds_v5": 0.808000, + "Standard_D16plds_v5": 0.678000, + "Standard_D16pls_v5": 0.595000, + "Standard_D16ps_v5": 0.685000, + "Standard_D16s_v3": 0.888000, + "Standard_D16s_v4": 0.856000, + "Standard_D16s_v5": 0.856000, + "Standard_D1_v2": 0.070000, + "Standard_D2": 0.149000, + "Standard_D2_v2": 0.140000, + "Standard_D2_v2_Promo": 0.152000, + "Standard_D2_v3": 0.111000, + "Standard_D2_v4": 0.107000, + "Standard_D2_v5": 0.107000, + "Standard_D2a_v4": 0.107000, + "Standard_D2ads_v5": 0.207000, + "Standard_D2as_v4": 0.107000, + "Standard_D2as_v5": 0.188000, + "Standard_D2as_v5_Promo": 0.085600, + "Standard_D2d_v4": 0.126000, + "Standard_D2d_v5": 0.126000, + "Standard_D2ds_v4": 0.126000, + "Standard_D2ds_v5": 0.126000, + "Standard_D2lds_v5": 0.106000, + "Standard_D2ls_v5": 0.093000, + "Standard_D2pds_v5": 0.101000, + "Standard_D2plds_v5": 0.084800, + "Standard_D2pls_v5": 0.074400, + "Standard_D2ps_v5": 0.085600, + "Standard_D2s_v3": 0.111000, + "Standard_D2s_v4": 0.107000, + "Standard_D2s_v5": 0.107000, + "Standard_D3": 0.299000, + "Standard_D32-16s_v3": 1.776000, + "Standard_D32-8s_v3": 1.776000, + "Standard_D32_v3": 1.776000, + "Standard_D32_v4": 1.712000, + "Standard_D32_v5": 1.712000, + "Standard_D32a_v4": 1.712000, + "Standard_D32ads_v5": 3.312000, + "Standard_D32as_v4": 1.712000, + "Standard_D32as_v5": 1.536000, + "Standard_D32as_v5_Promo": 1.370000, + "Standard_D32d_v4": 2.016000, + "Standard_D32d_v5": 2.016000, + "Standard_D32ds_v4": 2.016000, + "Standard_D32ds_v5": 2.016000, + "Standard_D32lds_v5": 1.696000, + "Standard_D32ls_v5": 1.488000, + "Standard_D32pds_v5": 1.616000, + "Standard_D32plds_v5": 1.357000, + "Standard_D32pls_v5": 1.190000, + "Standard_D32ps_v5": 1.370000, + "Standard_D32s_v3": 1.776000, + "Standard_D32s_v4": 1.712000, + "Standard_D32s_v5": 1.712000, + "Standard_D3_v2": 0.280000, + "Standard_D3_v2_Promo": 0.305000, + "Standard_D4": 0.597000, + "Standard_D48_v3": 2.664000, + "Standard_D48_v4": 2.568000, + "Standard_D48_v5": 2.568000, + "Standard_D48a_v4": 2.568000, + "Standard_D48ads_v5": 2.760000, + "Standard_D48as_v4": 2.568000, + "Standard_D48as_v5": 2.304000, + "Standard_D48as_v5_Promo": 2.054000, + "Standard_D48d_v4": 3.024000, + "Standard_D48d_v5": 3.024000, + "Standard_D48ds_v4": 3.024000, + "Standard_D48ds_v5": 3.024000, + "Standard_D48lds_v5": 2.544000, + "Standard_D48ls_v5": 2.232000, + "Standard_D48pds_v5": 2.424000, + "Standard_D48plds_v5": 2.035000, + "Standard_D48pls_v5": 1.786000, + "Standard_D48ps_v5": 2.054000, + "Standard_D48s_v3": 2.664000, + "Standard_D48s_v4": 2.568000, + "Standard_D48s_v5": 2.568000, + "Standard_D4_v2": 0.560000, + "Standard_D4_v2_Promo": 0.610000, + "Standard_D4_v3": 0.222000, + "Standard_D4_v4": 0.214000, + "Standard_D4_v5": 0.214000, + "Standard_D4a_v4": 0.214000, + "Standard_D4ads_v5": 0.414000, + "Standard_D4as_v4": 0.214000, + "Standard_D4as_v5": 0.192000, + "Standard_D4as_v5_Promo": 0.171000, + "Standard_D4d_v4": 0.252000, + "Standard_D4d_v5": 0.252000, + "Standard_D4ds_v4": 0.252000, + "Standard_D4ds_v5": 0.252000, + "Standard_D4lds_v5": 0.212000, + "Standard_D4ls_v5": 0.186000, + "Standard_D4pds_v5": 0.202000, + "Standard_D4plds_v5": 0.170000, + "Standard_D4pls_v5": 0.149000, + "Standard_D4ps_v5": 0.171000, + "Standard_D4s_v3": 0.222000, + "Standard_D4s_v4": 0.214000, + "Standard_D4s_v5": 0.214000, + "Standard_D5_v2": 1.120000, + "Standard_D5_v2_Promo": 1.220000, + "Standard_D64-16s_v3": 3.552000, + "Standard_D64-32s_v3": 3.552000, + "Standard_D64_v3": 3.552000, + "Standard_D64_v4": 3.424000, + "Standard_D64_v5": 3.424000, + "Standard_D64a_v4": 3.424000, + "Standard_D64ads_v5": 6.624000, + "Standard_D64as_v4": 3.424000, + "Standard_D64as_v5": 3.072000, + "Standard_D64as_v5_Promo": 2.739000, + "Standard_D64d_v4": 4.032000, + "Standard_D64d_v5": 4.032000, + "Standard_D64ds_v4": 4.032000, + "Standard_D64ds_v5": 4.032000, + "Standard_D64lds_v5": 3.392000, + "Standard_D64ls_v5": 2.976000, + "Standard_D64pds_v5": 3.232000, + "Standard_D64plds_v5": 2.714000, + "Standard_D64pls_v5": 2.381000, + "Standard_D64ps_v5": 2.739000, + "Standard_D64s_v3": 3.552000, + "Standard_D64s_v4": 3.424000, + "Standard_D64s_v5": 3.424000, + "Standard_D8_v3": 0.444000, + "Standard_D8_v4": 0.428000, + "Standard_D8_v5": 0.428000, + "Standard_D8a_v4": 0.428000, + "Standard_D8ads_v5": 0.828000, + "Standard_D8as_v4": 0.428000, + "Standard_D8as_v5": 0.752000, + "Standard_D8as_v5_Promo": 0.342000, + "Standard_D8d_v4": 0.504000, + "Standard_D8d_v5": 0.504000, + "Standard_D8ds_v4": 0.504000, + "Standard_D8ds_v5": 0.504000, + "Standard_D8lds_v5": 0.424000, + "Standard_D8ls_v5": 0.372000, + "Standard_D8pds_v5": 0.404000, + "Standard_D8plds_v5": 0.339000, + "Standard_D8pls_v5": 0.298000, + "Standard_D8ps_v5": 0.342000, + "Standard_D8s_v3": 0.444000, + "Standard_D8s_v4": 0.428000, + "Standard_D8s_v5": 0.428000, + "Standard_D96_v5": 5.136000, + "Standard_D96a_v4": 5.136000, + "Standard_D96ads_v5": 9.936000, + "Standard_D96as_v4": 5.136000, + "Standard_D96as_v5": 4.608000, + "Standard_D96as_v5_Promo": 4.109000, + "Standard_D96d_v5": 6.048000, + "Standard_D96ds_v5": 6.048000, + "Standard_D96lds_v5": 5.088000, + "Standard_D96ls_v5": 4.464000, + "Standard_D96s_v5": 5.136000, + "Standard_DC16ds_v3": 2.016000, + "Standard_DC16s_v3": 1.712000, + "Standard_DC1ds_v3": 0.126000, + "Standard_DC1s_v2": 0.107000, + "Standard_DC1s_v3": 0.107000, + "Standard_DC24ds_v3": 3.024000, + "Standard_DC24s_v3": 2.568000, + "Standard_DC2ds_v3": 0.252000, + "Standard_DC2s_v2": 0.214000, + "Standard_DC2s_v3": 0.214000, + "Standard_DC32ds_v3": 4.032000, + "Standard_DC32s_v3": 3.424000, + "Standard_DC48ds_v3": 6.048000, + "Standard_DC48s_v3": 5.136000, + "Standard_DC4ds_v3": 0.504000, + "Standard_DC4s_v2": 0.428000, + "Standard_DC4s_v3": 0.428000, + "Standard_DC8_v2": 0.856000, + "Standard_DC8ds_v3": 1.008000, + "Standard_DC8s_v3": 0.856000, + "Standard_DS1": 0.074700, + "Standard_DS11": 0.193000, + "Standard_DS11-1_v2": 0.183000, + "Standard_DS11_v2": 0.183000, + "Standard_DS11_v2_Promo": 0.199000, + "Standard_DS12": 0.387000, + "Standard_DS12-1_v2": 0.366000, + "Standard_DS12-2_v2": 0.366000, + "Standard_DS12_v2": 0.366000, + "Standard_DS12_v2_Promo": 0.399000, + "Standard_DS13": 0.773000, + "Standard_DS13-2_v2": 0.732000, + "Standard_DS13-4_v2": 0.732000, + "Standard_DS13_v2": 0.732000, + "Standard_DS13_v2_Promo": 0.798000, + "Standard_DS14": 1.546000, + "Standard_DS14-4_v2": 1.463000, + "Standard_DS14-8_v2": 1.463000, + "Standard_DS14_v2": 1.463000, + "Standard_DS14_v2_Promo": 1.596000, + "Standard_DS15_v2": 1.829000, + "Standard_DS15i_v2": 1.829000, + "Standard_DS1_v2": 0.070000, + "Standard_DS2": 0.149000, + "Standard_DS2_v2": 0.140000, + "Standard_DS2_v2_Promo": 0.152000, + "Standard_DS3": 0.299000, + "Standard_DS3_v2": 0.280000, + "Standard_DS3_v2_Promo": 0.305000, + "Standard_DS4": 0.597000, + "Standard_DS4_v2": 0.560000, + "Standard_DS4_v2_Promo": 0.610000, + "Standard_DS5_v2": 1.120000, + "Standard_DS5_v2_Promo": 1.220000, + "Standard_E104i_v5": 7.894000, + "Standard_E104id_v5": 9.038000, + "Standard_E104ids_v5": 9.038000, + "Standard_E104is_v5": 7.894000, + "Standard_E112iads_v5": 8.870000, + "Standard_E112ias_v5": 7.638000, + "Standard_E112ibds_v5": 11.365000, + "Standard_E112ibs_v5": 10.133000, + "Standard_E16-4ads_v5": 1.152000, + "Standard_E16-4as_v4": 1.104000, + "Standard_E16-4as_v5": 1.728000, + "Standard_E16-4ds_v4": 1.264000, + "Standard_E16-4ds_v5": 1.264000, + "Standard_E16-4s_v3": 1.168000, + "Standard_E16-4s_v4": 1.104000, + "Standard_E16-4s_v5": 1.104000, + "Standard_E16-8ads_v5": 1.152000, + "Standard_E16-8as_v4": 1.104000, + "Standard_E16-8as_v5": 0.992000, + "Standard_E16-8ds_v4": 1.264000, + "Standard_E16-8ds_v5": 1.264000, + "Standard_E16-8s_v3": 1.168000, + "Standard_E16-8s_v4": 1.104000, + "Standard_E16-8s_v5": 1.104000, + "Standard_E16_v3": 1.168000, + "Standard_E16_v4": 1.104000, + "Standard_E16_v5": 1.104000, + "Standard_E16a_v4": 1.104000, + "Standard_E16ads_v5": 1.198000, + "Standard_E16as_v4": 1.104000, + "Standard_E16as_v5": 1.038000, + "Standard_E16bds_v5": 1.476000, + "Standard_E16bs_v5": 1.316000, + "Standard_E16d_v4": 1.264000, + "Standard_E16d_v5": 1.264000, + "Standard_E16ds_v4": 1.264000, + "Standard_E16ds_v5": 1.264000, + "Standard_E16pds_v5": 1.012000, + "Standard_E16ps_v5": 0.883000, + "Standard_E16s_v3": 1.168000, + "Standard_E16s_v4": 1.104000, + "Standard_E16s_v5": 1.104000, + "Standard_E20_v3": 1.460000, + "Standard_E20_v4": 1.380000, + "Standard_E20_v5": 1.380000, + "Standard_E20a_v4": 1.380000, + "Standard_E20ads_v5": 1.440000, + "Standard_E20as_v4": 1.380000, + "Standard_E20as_v5": 1.286000, + "Standard_E20d_v4": 1.580000, + "Standard_E20d_v5": 1.580000, + "Standard_E20ds_v4": 1.580000, + "Standard_E20ds_v5": 1.580000, + "Standard_E20pds_v5": 1.265000, + "Standard_E20ps_v5": 1.104000, + "Standard_E20s_v3": 1.460000, + "Standard_E20s_v4": 1.380000, + "Standard_E20s_v5": 1.380000, + "Standard_E2_v3": 0.146000, + "Standard_E2_v4": 0.138000, + "Standard_E2_v5": 0.138000, + "Standard_E2a_v4": 0.138000, + "Standard_E2ads_v5": 0.144000, + "Standard_E2as_v4": 0.138000, + "Standard_E2as_v5": 0.124000, + "Standard_E2bds_v5": 0.185000, + "Standard_E2bs_v5": 0.165000, + "Standard_E2d_v4": 0.158000, + "Standard_E2d_v5": 0.158000, + "Standard_E2ds_v4": 0.158000, + "Standard_E2ds_v5": 0.158000, + "Standard_E2pds_v5": 0.127000, + "Standard_E2ps_v5": 0.110000, + "Standard_E2s_v3": 0.146000, + "Standard_E2s_v4": 0.138000, + "Standard_E2s_v5": 0.138000, + "Standard_E32-16ads_v5": 3.776000, + "Standard_E32-16as_v4": 2.208000, + "Standard_E32-16as_v5": 3.456000, + "Standard_E32-16ds_v4": 2.528000, + "Standard_E32-16ds_v5": 2.528000, + "Standard_E32-16s_v3": 2.336000, + "Standard_E32-16s_v4": 2.208000, + "Standard_E32-16s_v5": 2.208000, + "Standard_E32-8ads_v5": 3.776000, + "Standard_E32-8as_v4": 2.208000, + "Standard_E32-8as_v5": 3.456000, + "Standard_E32-8ds_v4": 2.528000, + "Standard_E32-8ds_v5": 2.528000, + "Standard_E32-8s_v3": 2.336000, + "Standard_E32-8s_v4": 2.208000, + "Standard_E32-8s_v5": 2.208000, + "Standard_E32_v3": 2.336000, + "Standard_E32_v4": 2.208000, + "Standard_E32_v5": 2.208000, + "Standard_E32a_v4": 2.208000, + "Standard_E32ads_v5": 2.350000, + "Standard_E32as_v4": 2.208000, + "Standard_E32as_v5": 2.030000, + "Standard_E32bds_v5": 2.952000, + "Standard_E32bs_v5": 2.632000, + "Standard_E32d_v4": 2.528000, + "Standard_E32d_v5": 2.528000, + "Standard_E32ds_v4": 2.528000, + "Standard_E32ds_v5": 2.528000, + "Standard_E32pds_v5": 2.024000, + "Standard_E32ps_v5": 1.766000, + "Standard_E32s_v3": 2.336000, + "Standard_E32s_v4": 2.208000, + "Standard_E32s_v5": 2.208000, + "Standard_E4-2ads_v5": 0.288000, + "Standard_E4-2as_v4": 0.276000, + "Standard_E4-2as_v5": 0.248000, + "Standard_E4-2ds_v4": 0.316000, + "Standard_E4-2ds_v5": 0.316000, + "Standard_E4-2s_v3": 0.292000, + "Standard_E4-2s_v4": 0.276000, + "Standard_E4-2s_v5": 0.276000, + "Standard_E48_v3": 3.311000, + "Standard_E48_v4": 3.312000, + "Standard_E48_v5": 3.312000, + "Standard_E48a_v4": 3.312000, + "Standard_E48ads_v5": 3.502000, + "Standard_E48as_v4": 3.312000, + "Standard_E48as_v5": 3.022000, + "Standard_E48bds_v5": 4.428000, + "Standard_E48bs_v5": 3.948000, + "Standard_E48d_v4": 3.792000, + "Standard_E48d_v5": 3.792000, + "Standard_E48ds_v4": 3.792000, + "Standard_E48ds_v5": 3.792000, + "Standard_E48s_v3": 3.311000, + "Standard_E48s_v4": 3.312000, + "Standard_E48s_v5": 3.312000, + "Standard_E4_v3": 0.292000, + "Standard_E4_v4": 0.276000, + "Standard_E4_v5": 0.276000, + "Standard_E4a_v4": 0.276000, + "Standard_E4ads_v5": 0.288000, + "Standard_E4as_v4": 0.276000, + "Standard_E4as_v5": 0.248000, + "Standard_E4bds_v5": 0.369000, + "Standard_E4bs_v5": 0.329000, + "Standard_E4d_v4": 0.316000, + "Standard_E4d_v5": 0.316000, + "Standard_E4ds_v4": 0.316000, + "Standard_E4ds_v5": 0.316000, + "Standard_E4pds_v5": 0.253000, + "Standard_E4ps_v5": 0.221000, + "Standard_E4s_v3": 0.292000, + "Standard_E4s_v4": 0.276000, + "Standard_E4s_v5": 0.276000, + "Standard_E64-16ads_v5": 4.608000, + "Standard_E64-16as_v4": 4.416000, + "Standard_E64-16as_v5": 3.968000, + "Standard_E64-16ds_v4": 5.056000, + "Standard_E64-16ds_v5": 5.056000, + "Standard_E64-16s_v3": 3.974000, + "Standard_E64-16s_v4": 4.416000, + "Standard_E64-16s_v5": 4.416000, + "Standard_E64-32ads_v5": 4.608000, + "Standard_E64-32as_v4": 4.416000, + "Standard_E64-32as_v5": 6.912000, + "Standard_E64-32ds_v4": 5.056000, + "Standard_E64-32ds_v5": 5.056000, + "Standard_E64-32s_v3": 3.974000, + "Standard_E64-32s_v4": 4.416000, + "Standard_E64-32s_v5": 4.416000, + "Standard_E64_v3": 3.974000, + "Standard_E64_v4": 4.416000, + "Standard_E64_v5": 4.416000, + "Standard_E64a_v4": 4.416000, + "Standard_E64ads_v5": 4.654000, + "Standard_E64as_v4": 4.416000, + "Standard_E64as_v5": 4.014000, + "Standard_E64bds_v5": 5.904000, + "Standard_E64bs_v5": 5.264000, + "Standard_E64d_v4": 5.056000, + "Standard_E64d_v5": 5.056000, + "Standard_E64ds_v4": 5.056000, + "Standard_E64ds_v5": 5.056000, + "Standard_E64i_v3": 3.974000, + "Standard_E64is_v3": 3.974000, + "Standard_E64s_v3": 3.974000, + "Standard_E64s_v4": 4.416000, + "Standard_E64s_v5": 4.416000, + "Standard_E8-2ads_v5": 0.576000, + "Standard_E8-2as_v4": 0.552000, + "Standard_E8-2as_v5": 0.496000, + "Standard_E8-2ds_v4": 0.632000, + "Standard_E8-2ds_v5": 0.632000, + "Standard_E8-2s_v3": 0.584000, + "Standard_E8-2s_v4": 0.552000, + "Standard_E8-2s_v5": 0.552000, + "Standard_E8-4ads_v5": 0.576000, + "Standard_E8-4as_v4": 0.552000, + "Standard_E8-4as_v5": 0.496000, + "Standard_E8-4ds_v4": 0.632000, + "Standard_E8-4ds_v5": 0.632000, + "Standard_E8-4s_v3": 0.584000, + "Standard_E8-4s_v4": 0.552000, + "Standard_E8-4s_v5": 0.552000, + "Standard_E80ids_v4": 6.320000, + "Standard_E80is_v4": 5.520000, + "Standard_E8_v3": 0.584000, + "Standard_E8_v4": 0.552000, + "Standard_E8_v5": 0.552000, + "Standard_E8a_v4": 0.552000, + "Standard_E8ads_v5": 0.576000, + "Standard_E8as_v4": 0.552000, + "Standard_E8as_v5": 0.542000, + "Standard_E8bds_v5": 0.738000, + "Standard_E8bs_v5": 0.658000, + "Standard_E8d_v4": 0.632000, + "Standard_E8d_v5": 0.632000, + "Standard_E8ds_v4": 0.632000, + "Standard_E8ds_v5": 0.632000, + "Standard_E8pds_v5": 0.506000, + "Standard_E8ps_v5": 0.442000, + "Standard_E8s_v3": 0.584000, + "Standard_E8s_v4": 0.552000, + "Standard_E8s_v5": 0.552000, + "Standard_E96-24ads_v5": 11.328000, + "Standard_E96-24as_v4": 6.624000, + "Standard_E96-24as_v5": 10.368000, + "Standard_E96-24ds_v5": 7.584000, + "Standard_E96-24s_v5": 6.624000, + "Standard_E96-48ads_v5": 6.912000, + "Standard_E96-48as_v4": 6.624000, + "Standard_E96-48as_v5": 10.368000, + "Standard_E96-48ds_v5": 7.584000, + "Standard_E96-48s_v5": 6.624000, + "Standard_E96_v5": 6.624000, + "Standard_E96a_v4": 6.624000, + "Standard_E96ads_v5": 6.912000, + "Standard_E96as_v4": 6.624000, + "Standard_E96as_v5": 5.952000, + "Standard_E96bds_v5": 8.856000, + "Standard_E96bs_v5": 7.896000, + "Standard_E96d_v5": 7.584000, + "Standard_E96ds_v5": 7.584000, + "Standard_E96iads_v5": 7.603000, + "Standard_E96ias_v4": 6.624000, + "Standard_E96ias_v5": 6.547000, + "Standard_E96s_v5": 6.624000, + "Standard_F1": 0.051900, + "Standard_F16": 0.830000, + "Standard_F16s": 0.830000, + "Standard_F16s_v2": 0.741000, + "Standard_F1s": 0.051900, + "Standard_F2": 0.104000, + "Standard_F2s": 0.104000, + "Standard_F2s_v2": 0.092700, + "Standard_F32s_v2": 1.482000, + "Standard_F4": 0.207000, + "Standard_F48s_v2": 2.224000, + "Standard_F4s": 0.207000, + "Standard_F4s_v2": 0.185000, + "Standard_F64s_v2": 2.965000, + "Standard_F72s_v2": 3.335000, + "Standard_F8": 0.415000, + "Standard_F8s": 0.415000, + "Standard_F8s_v2": 0.371000, + "Standard_FX12mds": 1.152000, + "Standard_FX24mds": 2.304000, + "Standard_FX36mds": 3.456000, + "Standard_FX48mds": 4.608000, + "Standard_FX4mds": 0.384000, + "Standard_G1": 0.484000, + "Standard_G2": 0.968000, + "Standard_G3": 1.936000, + "Standard_G4": 3.872000, + "Standard_G5": 7.744000, + "Standard_GS1": 0.484000, + "Standard_GS2": 0.968000, + "Standard_GS3": 1.936000, + "Standard_GS4": 3.872000, + "Standard_GS4-4": 3.872000, + "Standard_GS4-8": 3.872000, + "Standard_GS5": 7.744000, + "Standard_GS5-16": 7.744000, + "Standard_GS5-8": 7.744000, + "Standard_HB120-16rs_v2": 4.320000, + "Standard_HB120-32rs_v2": 4.320000, + "Standard_HB120-64rs_v2": 4.320000, + "Standard_HB120-96rs_v2": 4.320000, + "Standard_HB120rs_v2": 4.320000, + "Standard_HC44-16rs": 3.802000, + "Standard_HC44-32rs": 3.802000, + "Standard_HC44rs": 3.802000, + "Standard_L112ias_v3": 10.595000, + "Standard_L16as_v3": 1.376000, + "Standard_L16s": 1.376000, + "Standard_L16s_v3": 1.536000, + "Standard_L32as_v3": 2.752000, + "Standard_L32s": 2.752000, + "Standard_L32s_v3": 3.072000, + "Standard_L48as_v3": 4.128000, + "Standard_L48s_v3": 4.608000, + "Standard_L4s": 0.344000, + "Standard_L64as_v3": 5.504000, + "Standard_L64s_v3": 6.144000, + "Standard_L80as_v3": 6.880000, + "Standard_L80s_v3": 7.680000, + "Standard_L8as_v3": 0.688000, + "Standard_L8s": 0.688000, + "Standard_L8s_v3": 0.768000, + "Standard_L96ias_v3": 9.082000, + "Standard_M128": 14.671800, + "Standard_M128-32ms": 29.356800, + "Standard_M128-64ms": 29.356800, + "Standard_M128dms_v2": 29.359000, + "Standard_M128ds_v2": 14.674000, + "Standard_M128m": 29.356800, + "Standard_M128ms": 29.356800, + "Standard_M128ms_v2": 28.896000, + "Standard_M128s": 14.671800, + "Standard_M128s_v2": 14.211000, + "Standard_M16-4ms": 3.380300, + "Standard_M16-8ms": 3.380300, + "Standard_M16ms": 3.380300, + "Standard_M16s": 2.625700, + "Standard_M192idms_v2": 35.270000, + "Standard_M192ids_v2": 17.635000, + "Standard_M192ims_v2": 34.807000, + "Standard_M192is_v2": 17.172000, + "Standard_M208-104ms_v2": 53.544000, + "Standard_M208-104s_v2": 26.771000, + "Standard_M208-52ms_v2": 53.544000, + "Standard_M208-52s_v2": 26.771000, + "Standard_M208ms_v2": 53.544000, + "Standard_M208s_v2": 26.771000, + "Standard_M32-16ms": 6.760600, + "Standard_M32-8ms": 6.760600, + "Standard_M32dms_v2": 6.761000, + "Standard_M32ls": 3.160300, + "Standard_M32ms": 6.760600, + "Standard_M32ms_v2": 6.645000, + "Standard_M32s": 3.668500, + "Standard_M32ts": 2.977700, + "Standard_M416-104ms_v2": 118.980000, + "Standard_M416-104s_v2": 59.496000, + "Standard_M416-208ms_v2": 118.980000, + "Standard_M416-208s_v2": 59.496000, + "Standard_M416is_v2": 59.496000, + "Standard_M416ms_v2": 118.980000, + "Standard_M416s_8_v2": 79.330000, + "Standard_M416s_v2": 59.496000, + "Standard_M64": 7.335900, + "Standard_M64-16ms": 11.370700, + "Standard_M64-32ms": 11.370700, + "Standard_M64dms_v2": 11.374000, + "Standard_M64ds_v2": 7.336000, + "Standard_M64ls": 5.956500, + "Standard_M64m": 11.370700, + "Standard_M64ms": 11.370700, + "Standard_M64ms_v2": 11.143000, + "Standard_M64s": 7.335900, + "Standard_M64s_v2": 7.104000, + "Standard_M8-2ms": 1.690150, + "Standard_M8-4ms": 1.690150, + "Standard_M8ms": 1.690150, + "Standard_NC12s_v3": 6.732000, + "Standard_NC16as_T4_v3": 1.336000, + "Standard_NC24rs_v3": 14.810000, + "Standard_NC24s_v3": 13.464000, + "Standard_NC4as_T4_v3": 0.584000, + "Standard_NC64as_T4_v3": 4.820000, + "Standard_NC6s_v3": 3.366000, + "Standard_NC8as_T4_v3": 0.834000, + "Standard_ND96ams_A100_v4": 39.324000, + "Standard_ND96amsr_A100_v4": 39.324000, + "Standard_NP10s": 1.980000, + "Standard_NP20s": 3.960000, + "Standard_NP40s": 7.920000, + "Standard_NV12ads_A10_v5": 1.090000, + "Standard_NV16as_v4": 1.118000, + "Standard_NV18ads_A10_v5": 1.920000, + "Standard_NV32as_v4": 2.237000, + "Standard_NV36adms_A10_v5": 5.424000, + "Standard_NV36ads_A10_v5": 3.840000, + "Standard_NV4as_v4": 0.280000, + "Standard_NV6ads_A10_v5": 0.545000, + "Standard_NV72ads_A10_v5": 7.824000, + "Standard_NV8as_v4": 0.559000, + } + // canadaeast + initialOnDemandPrices["canadaeast"] = map[string]float64{ + "Basic_A0": 0.020000, + "Basic_A1": 0.026000, + "Basic_A2": 0.087000, + "Basic_A3": 0.194000, + "Basic_A4": 0.387000, + "DCsv2 Type 1": 0.942000, + "Dadsv5_Type1": 7.084000, + "Dasv4_Type1": 5.885000, + "Dasv4_Type2": 5.885000, + "Dasv5_Type1": 5.914000, + "Ddsv4_Type 1": 4.435000, + "Ddsv4_Type2": 5.267000, + "Ddsv5_Type1": 6.653000, + "Dsv3_Type3": 4.709000, + "Dsv3_Type4": 5.886000, + "Dsv4_Type1": 4.708000, + "Dsv4_Type2": 5.650000, + "Dsv5_Type1": 5.885000, + "Eadsv5_Type1": 7.603000, + "Easv4_Type1": 7.286000, + "Easv4_Type2": 7.286000, + "Easv5_Type1": 6.547000, + "Ebdsv5-Type1": 6.494000, + "Ebsv5-Type1": 5.790000, + "Edsv4_Type 1": 5.562000, + "Edsv4_Type2": 7.231000, + "Edsv5_Type1": 8.342000, + "Esv3_Type3": 4.705000, + "Esv3_Type4": 6.375000, + "Esv4_Type1": 4.857000, + "Esv4_Type2": 6.376000, + "Esv5_Type1": 7.286000, + "FXmds Type1": 5.069000, + "Fsv2 Type3": 4.297000, + "Fsv2_Type2": 3.683000, + "Fsv2_Type4": 4.897000, + "Lasv3_Type1": 7.568000, + "Lsv3_Type1": 8.448000, + "Mdmsv2MedMem _Type1": 32.295000, + "Mdsv2MedMem_Type1": 16.141000, + "Msmv2MedMem Type1": 31.786000, + "Msmv2_Type1": 130.880000, + "Msv2MedMem Type1": 15.632000, + "Msv2_Type1": 65.446000, + "Standard_A0": 0.022000, + "Standard_A1": 0.056000, + "Standard_A1_v2": 0.043000, + "Standard_A2": 0.111000, + "Standard_A2_v2": 0.091000, + "Standard_A2m_v2": 0.129000, + "Standard_A3": 0.222000, + "Standard_A4": 0.446000, + "Standard_A4_v2": 0.191000, + "Standard_A4m_v2": 0.270000, + "Standard_A5": 0.242000, + "Standard_A6": 0.484000, + "Standard_A7": 0.968000, + "Standard_A8_v2": 0.400000, + "Standard_A8m_v2": 0.568000, + "Standard_B12ms": 0.557000, + "Standard_B16als_v2": 0.666000, + "Standard_B16as_v2": 0.669000, + "Standard_B16ls_v2": 0.731000, + "Standard_B16ms": 0.742000, + "Standard_B16s_v2": 0.816000, + "Standard_B1ls": 0.005800, + "Standard_B1ms": 0.023200, + "Standard_B1s": 0.011600, + "Standard_B20ms": 0.928000, + "Standard_B2als_v2": 0.041800, + "Standard_B2as_v2": 0.083600, + "Standard_B2ats_v2": 0.019700, + "Standard_B2ls_v2": 0.046400, + "Standard_B2ms": 0.092800, + "Standard_B2s": 0.046400, + "Standard_B2s_v2": 0.092800, + "Standard_B2ts_v2": 0.020800, + "Standard_B32als_v2": 1.184000, + "Standard_B32as_v2": 1.338000, + "Standard_B32ls_v2": 1.315000, + "Standard_B32s_v2": 1.485000, + "Standard_B4als_v2": 0.148000, + "Standard_B4as_v2": 0.167000, + "Standard_B4ls_v2": 0.183000, + "Standard_B4ms": 0.186000, + "Standard_B4s_v2": 0.186000, + "Standard_B8als_v2": 0.333000, + "Standard_B8as_v2": 0.371000, + "Standard_B8ls_v2": 0.366000, + "Standard_B8ms": 0.371000, + "Standard_B8s_v2": 0.408000, + "Standard_D11_v2": 0.183000, + "Standard_D11_v2_Promo": 0.183000, + "Standard_D12_v2": 0.366000, + "Standard_D12_v2_Promo": 0.366000, + "Standard_D13_v2": 0.732000, + "Standard_D13_v2_Promo": 0.732000, + "Standard_D14_v2": 1.463000, + "Standard_D14_v2_Promo": 1.463000, + "Standard_D15_v2": 1.829000, + "Standard_D15i_v2": 1.829000, + "Standard_D16_v3": 0.888000, + "Standard_D16_v4": 0.856000, + "Standard_D16_v5": 0.856000, + "Standard_D16a_v4": 0.856000, + "Standard_D16ads_v5": 0.920000, + "Standard_D16as_v4": 0.856000, + "Standard_D16as_v5": 1.504000, + "Standard_D16as_v5_Promo": 0.685000, + "Standard_D16d_v4": 1.008000, + "Standard_D16d_v5": 1.008000, + "Standard_D16ds_v4": 1.008000, + "Standard_D16ds_v5": 1.008000, + "Standard_D16lds_v5": 0.848000, + "Standard_D16ls_v5": 0.744000, + "Standard_D16s_v3": 0.888000, + "Standard_D16s_v4": 0.856000, + "Standard_D16s_v5": 0.856000, + "Standard_D1_v2": 0.070000, + "Standard_D2_v2": 0.140000, + "Standard_D2_v2_Promo": 0.140000, + "Standard_D2_v3": 0.111000, + "Standard_D2_v4": 0.107000, + "Standard_D2_v5": 0.107000, + "Standard_D2a_v4": 0.107000, + "Standard_D2ads_v5": 0.115000, + "Standard_D2as_v4": 0.107000, + "Standard_D2as_v5": 0.188000, + "Standard_D2as_v5_Promo": 0.085600, + "Standard_D2d_v4": 0.126000, + "Standard_D2d_v5": 0.126000, + "Standard_D2ds_v4": 0.126000, + "Standard_D2ds_v5": 0.126000, + "Standard_D2lds_v5": 0.106000, + "Standard_D2ls_v5": 0.093000, + "Standard_D2s_v3": 0.111000, + "Standard_D2s_v4": 0.107000, + "Standard_D2s_v5": 0.107000, + "Standard_D32-16s_v3": 1.776000, + "Standard_D32-8s_v3": 1.776000, + "Standard_D32_v3": 1.776000, + "Standard_D32_v4": 1.712000, + "Standard_D32_v5": 1.712000, + "Standard_D32a_v4": 1.712000, + "Standard_D32ads_v5": 1.840000, + "Standard_D32as_v4": 1.712000, + "Standard_D32as_v5": 3.008000, + "Standard_D32as_v5_Promo": 1.370000, + "Standard_D32d_v4": 2.016000, + "Standard_D32d_v5": 2.016000, + "Standard_D32ds_v4": 2.016000, + "Standard_D32ds_v5": 2.016000, + "Standard_D32lds_v5": 1.696000, + "Standard_D32ls_v5": 1.488000, + "Standard_D32s_v3": 1.776000, + "Standard_D32s_v4": 1.712000, + "Standard_D32s_v5": 1.712000, + "Standard_D3_v2": 0.280000, + "Standard_D3_v2_Promo": 0.280000, + "Standard_D48_v3": 2.664000, + "Standard_D48_v4": 2.568000, + "Standard_D48_v5": 2.568000, + "Standard_D48a_v4": 2.568000, + "Standard_D48ads_v5": 4.968000, + "Standard_D48as_v4": 2.568000, + "Standard_D48as_v5": 2.304000, + "Standard_D48as_v5_Promo": 2.054000, + "Standard_D48d_v4": 3.024000, + "Standard_D48d_v5": 3.024000, + "Standard_D48ds_v4": 3.024000, + "Standard_D48ds_v5": 3.024000, + "Standard_D48lds_v5": 2.544000, + "Standard_D48ls_v5": 2.232000, + "Standard_D48s_v3": 2.664000, + "Standard_D48s_v4": 2.568000, + "Standard_D48s_v5": 2.568000, + "Standard_D4_v2": 0.560000, + "Standard_D4_v2_Promo": 0.560000, + "Standard_D4_v3": 0.222000, + "Standard_D4_v4": 0.214000, + "Standard_D4_v5": 0.214000, + "Standard_D4a_v4": 0.214000, + "Standard_D4ads_v5": 0.414000, + "Standard_D4as_v4": 0.214000, + "Standard_D4as_v5": 0.376000, + "Standard_D4as_v5_Promo": 0.171000, + "Standard_D4d_v4": 0.252000, + "Standard_D4d_v5": 0.252000, + "Standard_D4ds_v4": 0.252000, + "Standard_D4ds_v5": 0.252000, + "Standard_D4lds_v5": 0.212000, + "Standard_D4ls_v5": 0.186000, + "Standard_D4s_v3": 0.222000, + "Standard_D4s_v4": 0.214000, + "Standard_D4s_v5": 0.214000, + "Standard_D5_v2": 1.120000, + "Standard_D5_v2_Promo": 1.120000, + "Standard_D64-16s_v3": 3.552000, + "Standard_D64-32s_v3": 3.552000, + "Standard_D64_v3": 3.552000, + "Standard_D64_v4": 3.424000, + "Standard_D64_v5": 3.424000, + "Standard_D64a_v4": 3.424000, + "Standard_D64ads_v5": 3.680000, + "Standard_D64as_v4": 3.424000, + "Standard_D64as_v5": 3.072000, + "Standard_D64as_v5_Promo": 2.739000, + "Standard_D64d_v4": 4.032000, + "Standard_D64d_v5": 4.032000, + "Standard_D64ds_v4": 4.032000, + "Standard_D64ds_v5": 4.032000, + "Standard_D64lds_v5": 3.392000, + "Standard_D64ls_v5": 2.976000, + "Standard_D64s_v3": 3.552000, + "Standard_D64s_v4": 3.424000, + "Standard_D64s_v5": 3.424000, + "Standard_D8_v3": 0.444000, + "Standard_D8_v4": 0.428000, + "Standard_D8_v5": 0.428000, + "Standard_D8a_v4": 0.428000, + "Standard_D8ads_v5": 0.828000, + "Standard_D8as_v4": 0.428000, + "Standard_D8as_v5": 0.752000, + "Standard_D8as_v5_Promo": 0.342000, + "Standard_D8d_v4": 0.504000, + "Standard_D8d_v5": 0.504000, + "Standard_D8ds_v4": 0.504000, + "Standard_D8ds_v5": 0.504000, + "Standard_D8lds_v5": 0.424000, + "Standard_D8ls_v5": 0.372000, + "Standard_D8s_v3": 0.444000, + "Standard_D8s_v4": 0.428000, + "Standard_D8s_v5": 0.428000, + "Standard_D96_v5": 5.136000, + "Standard_D96a_v4": 5.136000, + "Standard_D96ads_v5": 9.936000, + "Standard_D96as_v4": 5.136000, + "Standard_D96as_v5": 9.024000, + "Standard_D96as_v5_Promo": 4.109000, + "Standard_D96d_v5": 6.048000, + "Standard_D96ds_v5": 6.048000, + "Standard_D96lds_v5": 5.088000, + "Standard_D96ls_v5": 4.464000, + "Standard_D96s_v5": 5.136000, + "Standard_DC1s_v2": 0.107000, + "Standard_DC2s_v2": 0.214000, + "Standard_DC4s_v2": 0.428000, + "Standard_DC8_v2": 0.856000, + "Standard_DS11-1_v2": 0.183000, + "Standard_DS11_v2": 0.183000, + "Standard_DS11_v2_Promo": 0.183000, + "Standard_DS12-1_v2": 0.366000, + "Standard_DS12-2_v2": 0.366000, + "Standard_DS12_v2": 0.366000, + "Standard_DS12_v2_Promo": 0.366000, + "Standard_DS13-2_v2": 0.732000, + "Standard_DS13-4_v2": 0.732000, + "Standard_DS13_v2": 0.732000, + "Standard_DS13_v2_Promo": 0.732000, + "Standard_DS14-4_v2": 1.463000, + "Standard_DS14-8_v2": 1.463000, + "Standard_DS14_v2": 1.463000, + "Standard_DS14_v2_Promo": 1.463000, + "Standard_DS15_v2": 1.829000, + "Standard_DS15i_v2": 1.829000, + "Standard_DS1_v2": 0.070000, + "Standard_DS2_v2": 0.140000, + "Standard_DS2_v2_Promo": 0.140000, + "Standard_DS3_v2": 0.280000, + "Standard_DS3_v2_Promo": 0.280000, + "Standard_DS4_v2": 0.560000, + "Standard_DS4_v2_Promo": 0.560000, + "Standard_DS5_v2": 1.120000, + "Standard_DS5_v2_Promo": 1.120000, + "Standard_E104i_v5": 7.894000, + "Standard_E104id_v5": 9.038000, + "Standard_E104ids_v5": 9.038000, + "Standard_E104is_v5": 7.894000, + "Standard_E112iads_v5": 8.870000, + "Standard_E112ias_v5": 7.638000, + "Standard_E112ibds_v5": 11.365000, + "Standard_E112ibs_v5": 10.133000, + "Standard_E16-4ads_v5": 1.888000, + "Standard_E16-4as_v4": 1.104000, + "Standard_E16-4as_v5": 1.728000, + "Standard_E16-4ds_v4": 1.264000, + "Standard_E16-4ds_v5": 1.264000, + "Standard_E16-4s_v3": 1.168000, + "Standard_E16-4s_v4": 1.104000, + "Standard_E16-4s_v5": 1.104000, + "Standard_E16-8ads_v5": 1.888000, + "Standard_E16-8as_v4": 1.104000, + "Standard_E16-8as_v5": 0.992000, + "Standard_E16-8ds_v4": 1.264000, + "Standard_E16-8ds_v5": 1.264000, + "Standard_E16-8s_v3": 1.168000, + "Standard_E16-8s_v4": 1.104000, + "Standard_E16-8s_v5": 1.104000, + "Standard_E16_v3": 1.168000, + "Standard_E16_v4": 1.104000, + "Standard_E16_v5": 1.104000, + "Standard_E16a_v4": 1.104000, + "Standard_E16ads_v5": 1.152000, + "Standard_E16as_v4": 1.104000, + "Standard_E16as_v5": 0.992000, + "Standard_E16bds_v5": 1.476000, + "Standard_E16bs_v5": 1.316000, + "Standard_E16d_v4": 1.264000, + "Standard_E16d_v5": 1.264000, + "Standard_E16ds_v4": 1.264000, + "Standard_E16ds_v5": 1.264000, + "Standard_E16s_v3": 1.168000, + "Standard_E16s_v4": 1.104000, + "Standard_E16s_v5": 1.104000, + "Standard_E20_v3": 1.460000, + "Standard_E20_v4": 1.380000, + "Standard_E20_v5": 1.380000, + "Standard_E20a_v4": 1.380000, + "Standard_E20ads_v5": 1.440000, + "Standard_E20as_v4": 1.380000, + "Standard_E20as_v5": 1.240000, + "Standard_E20d_v4": 1.580000, + "Standard_E20d_v5": 1.580000, + "Standard_E20ds_v4": 1.580000, + "Standard_E20ds_v5": 1.580000, + "Standard_E20s_v3": 1.460000, + "Standard_E20s_v4": 1.380000, + "Standard_E20s_v5": 1.380000, + "Standard_E2_v3": 0.146000, + "Standard_E2_v4": 0.138000, + "Standard_E2_v5": 0.138000, + "Standard_E2a_v4": 0.138000, + "Standard_E2ads_v5": 0.190000, + "Standard_E2as_v4": 0.138000, + "Standard_E2as_v5": 0.124000, + "Standard_E2bds_v5": 0.185000, + "Standard_E2bs_v5": 0.165000, + "Standard_E2d_v4": 0.158000, + "Standard_E2d_v5": 0.158000, + "Standard_E2ds_v4": 0.158000, + "Standard_E2ds_v5": 0.158000, + "Standard_E2s_v3": 0.146000, + "Standard_E2s_v4": 0.138000, + "Standard_E2s_v5": 0.138000, + "Standard_E32-16ads_v5": 2.304000, + "Standard_E32-16as_v4": 2.208000, + "Standard_E32-16as_v5": 1.984000, + "Standard_E32-16ds_v4": 2.528000, + "Standard_E32-16ds_v5": 2.528000, + "Standard_E32-16s_v3": 2.336000, + "Standard_E32-16s_v4": 2.208000, + "Standard_E32-16s_v5": 2.208000, + "Standard_E32-8ads_v5": 2.304000, + "Standard_E32-8as_v4": 2.208000, + "Standard_E32-8as_v5": 1.984000, + "Standard_E32-8ds_v4": 2.528000, + "Standard_E32-8ds_v5": 2.528000, + "Standard_E32-8s_v3": 2.336000, + "Standard_E32-8s_v4": 2.208000, + "Standard_E32-8s_v5": 2.208000, + "Standard_E32_v3": 2.336000, + "Standard_E32_v4": 2.208000, + "Standard_E32_v5": 2.208000, + "Standard_E32a_v4": 2.208000, + "Standard_E32ads_v5": 2.350000, + "Standard_E32as_v4": 2.208000, + "Standard_E32as_v5": 2.030000, + "Standard_E32bds_v5": 2.952000, + "Standard_E32bs_v5": 2.632000, + "Standard_E32d_v4": 2.528000, + "Standard_E32d_v5": 2.528000, + "Standard_E32ds_v4": 2.528000, + "Standard_E32ds_v5": 2.528000, + "Standard_E32s_v3": 2.336000, + "Standard_E32s_v4": 2.208000, + "Standard_E32s_v5": 2.208000, + "Standard_E4-2ads_v5": 0.288000, + "Standard_E4-2as_v4": 0.276000, + "Standard_E4-2as_v5": 0.432000, + "Standard_E4-2ds_v4": 0.316000, + "Standard_E4-2ds_v5": 0.316000, + "Standard_E4-2s_v3": 0.292000, + "Standard_E4-2s_v4": 0.276000, + "Standard_E4-2s_v5": 0.276000, + "Standard_E48_v3": 3.311000, + "Standard_E48_v4": 3.312000, + "Standard_E48_v5": 3.312000, + "Standard_E48a_v4": 3.312000, + "Standard_E48ads_v5": 3.502000, + "Standard_E48as_v4": 3.312000, + "Standard_E48as_v5": 3.022000, + "Standard_E48bds_v5": 4.428000, + "Standard_E48bs_v5": 3.948000, + "Standard_E48d_v4": 3.792000, + "Standard_E48d_v5": 3.792000, + "Standard_E48ds_v4": 3.792000, + "Standard_E48ds_v5": 3.792000, + "Standard_E48s_v3": 3.311000, + "Standard_E48s_v4": 3.312000, + "Standard_E48s_v5": 3.312000, + "Standard_E4_v3": 0.292000, + "Standard_E4_v4": 0.276000, + "Standard_E4_v5": 0.276000, + "Standard_E4a_v4": 0.276000, + "Standard_E4ads_v5": 0.288000, + "Standard_E4as_v4": 0.276000, + "Standard_E4as_v5": 0.294000, + "Standard_E4bds_v5": 0.369000, + "Standard_E4bs_v5": 0.329000, + "Standard_E4d_v4": 0.316000, + "Standard_E4d_v5": 0.316000, + "Standard_E4ds_v4": 0.316000, + "Standard_E4ds_v5": 0.316000, + "Standard_E4s_v3": 0.292000, + "Standard_E4s_v4": 0.276000, + "Standard_E4s_v5": 0.276000, + "Standard_E64-16ads_v5": 4.608000, + "Standard_E64-16as_v4": 4.416000, + "Standard_E64-16as_v5": 6.912000, + "Standard_E64-16ds_v4": 5.056000, + "Standard_E64-16ds_v5": 5.056000, + "Standard_E64-16s_v3": 3.974000, + "Standard_E64-16s_v4": 4.416000, + "Standard_E64-16s_v5": 4.416000, + "Standard_E64-32ads_v5": 7.552000, + "Standard_E64-32as_v4": 4.416000, + "Standard_E64-32as_v5": 6.912000, + "Standard_E64-32ds_v4": 5.056000, + "Standard_E64-32ds_v5": 5.056000, + "Standard_E64-32s_v3": 3.974000, + "Standard_E64-32s_v4": 4.416000, + "Standard_E64-32s_v5": 4.416000, + "Standard_E64_v3": 3.974000, + "Standard_E64_v4": 4.416000, + "Standard_E64_v5": 4.416000, + "Standard_E64a_v4": 4.416000, + "Standard_E64ads_v5": 4.654000, + "Standard_E64as_v4": 4.416000, + "Standard_E64as_v5": 3.968000, + "Standard_E64bds_v5": 5.904000, + "Standard_E64bs_v5": 5.264000, + "Standard_E64d_v4": 5.056000, + "Standard_E64d_v5": 5.056000, + "Standard_E64ds_v4": 5.056000, + "Standard_E64ds_v5": 5.056000, + "Standard_E64i_v3": 3.974000, + "Standard_E64is_v3": 3.974000, + "Standard_E64s_v3": 3.974000, + "Standard_E64s_v4": 4.416000, + "Standard_E64s_v5": 4.416000, + "Standard_E8-2ads_v5": 0.576000, + "Standard_E8-2as_v4": 0.552000, + "Standard_E8-2as_v5": 0.496000, + "Standard_E8-2ds_v4": 0.632000, + "Standard_E8-2ds_v5": 0.632000, + "Standard_E8-2s_v3": 0.584000, + "Standard_E8-2s_v4": 0.552000, + "Standard_E8-2s_v5": 0.552000, + "Standard_E8-4ads_v5": 0.944000, + "Standard_E8-4as_v4": 0.552000, + "Standard_E8-4as_v5": 0.864000, + "Standard_E8-4ds_v4": 0.632000, + "Standard_E8-4ds_v5": 0.632000, + "Standard_E8-4s_v3": 0.584000, + "Standard_E8-4s_v4": 0.552000, + "Standard_E8-4s_v5": 0.552000, + "Standard_E80ids_v4": 6.320000, + "Standard_E80is_v4": 5.520000, + "Standard_E8_v3": 0.584000, + "Standard_E8_v4": 0.552000, + "Standard_E8_v5": 0.552000, + "Standard_E8a_v4": 0.552000, + "Standard_E8ads_v5": 0.622000, + "Standard_E8as_v4": 0.552000, + "Standard_E8as_v5": 0.542000, + "Standard_E8bds_v5": 0.738000, + "Standard_E8bs_v5": 0.658000, + "Standard_E8d_v4": 0.632000, + "Standard_E8d_v5": 0.632000, + "Standard_E8ds_v4": 0.632000, + "Standard_E8ds_v5": 0.632000, + "Standard_E8s_v3": 0.584000, + "Standard_E8s_v4": 0.552000, + "Standard_E8s_v5": 0.552000, + "Standard_E96-24ads_v5": 6.912000, + "Standard_E96-24as_v4": 6.624000, + "Standard_E96-24as_v5": 5.952000, + "Standard_E96-24ds_v5": 7.584000, + "Standard_E96-24s_v5": 6.624000, + "Standard_E96-48ads_v5": 11.328000, + "Standard_E96-48as_v4": 6.624000, + "Standard_E96-48as_v5": 10.368000, + "Standard_E96-48ds_v5": 7.584000, + "Standard_E96-48s_v5": 6.624000, + "Standard_E96_v5": 6.624000, + "Standard_E96a_v4": 6.624000, + "Standard_E96ads_v5": 6.958000, + "Standard_E96as_v4": 6.624000, + "Standard_E96as_v5": 5.952000, + "Standard_E96bds_v5": 8.856000, + "Standard_E96bs_v5": 7.896000, + "Standard_E96d_v5": 7.584000, + "Standard_E96ds_v5": 7.584000, + "Standard_E96iads_v5": 7.603000, + "Standard_E96ias_v4": 6.624000, + "Standard_E96ias_v5": 6.547000, + "Standard_E96s_v5": 6.624000, + "Standard_F1": 0.054600, + "Standard_F16": 0.874000, + "Standard_F16s": 0.874000, + "Standard_F16s_v2": 0.741000, + "Standard_F1s": 0.054600, + "Standard_F2": 0.109000, + "Standard_F2s": 0.109000, + "Standard_F2s_v2": 0.092700, + "Standard_F32s_v2": 1.482000, + "Standard_F4": 0.218000, + "Standard_F48s_v2": 2.224000, + "Standard_F4s": 0.218000, + "Standard_F4s_v2": 0.185000, + "Standard_F64s_v2": 2.965000, + "Standard_F72s_v2": 3.335000, + "Standard_F8": 0.437000, + "Standard_F8s": 0.437000, + "Standard_F8s_v2": 0.371000, + "Standard_FX12mds": 1.152000, + "Standard_FX24mds": 2.304000, + "Standard_FX36mds": 3.456000, + "Standard_FX48mds": 4.608000, + "Standard_FX4mds": 0.384000, + "Standard_G1": 0.484000, + "Standard_G2": 0.968000, + "Standard_G3": 1.936000, + "Standard_G4": 3.872000, + "Standard_G5": 7.744000, + "Standard_GS1": 0.484000, + "Standard_GS2": 0.968000, + "Standard_GS3": 1.936000, + "Standard_GS4": 3.872000, + "Standard_GS4-4": 3.872000, + "Standard_GS4-8": 3.872000, + "Standard_GS5": 7.744000, + "Standard_GS5-16": 7.744000, + "Standard_GS5-8": 7.744000, + "Standard_L16as_v3": 1.376000, + "Standard_L16s": 1.376000, + "Standard_L16s_v3": 1.536000, + "Standard_L32as_v3": 2.752000, + "Standard_L32s": 2.752000, + "Standard_L32s_v3": 3.072000, + "Standard_L48as_v3": 4.128000, + "Standard_L48s_v3": 4.608000, + "Standard_L4s": 0.344000, + "Standard_L64as_v3": 5.504000, + "Standard_L64s_v3": 6.144000, + "Standard_L80as_v3": 6.880000, + "Standard_L80s_v3": 7.680000, + "Standard_L8as_v3": 0.688000, + "Standard_L8s": 0.688000, + "Standard_L8s_v3": 0.768000, + "Standard_M128": 14.671800, + "Standard_M128-32ms": 29.356800, + "Standard_M128-64ms": 29.356800, + "Standard_M128dms_v2": 29.359000, + "Standard_M128ds_v2": 14.674000, + "Standard_M128m": 29.356800, + "Standard_M128ms": 29.356800, + "Standard_M128ms_v2": 28.896000, + "Standard_M128s": 14.671800, + "Standard_M128s_v2": 14.211000, + "Standard_M16-4ms": 3.380300, + "Standard_M16-8ms": 3.380300, + "Standard_M16ms": 3.380300, + "Standard_M16s": 2.625700, + "Standard_M192idms_v2": 35.270000, + "Standard_M192ids_v2": 17.635000, + "Standard_M192ims_v2": 34.807000, + "Standard_M192is_v2": 17.172000, + "Standard_M208-104ms_v2": 53.544000, + "Standard_M208-104s_v2": 26.771000, + "Standard_M208-52ms_v2": 53.544000, + "Standard_M208-52s_v2": 26.771000, + "Standard_M208ms_v2": 53.544000, + "Standard_M208s_v2": 26.771000, + "Standard_M32-16ms": 6.760600, + "Standard_M32-8ms": 6.760600, + "Standard_M32dms_v2": 6.761000, + "Standard_M32ls": 3.160300, + "Standard_M32ms": 6.760600, + "Standard_M32ms_v2": 6.645000, + "Standard_M32s": 3.668500, + "Standard_M32ts": 2.977700, + "Standard_M416-104ms_v2": 118.980000, + "Standard_M416-104s_v2": 59.496000, + "Standard_M416-208ms_v2": 118.980000, + "Standard_M416-208s_v2": 59.496000, + "Standard_M416is_v2": 59.496000, + "Standard_M416ms_v2": 118.980000, + "Standard_M416s_8_v2": 79.330000, + "Standard_M416s_v2": 59.496000, + "Standard_M64": 7.335900, + "Standard_M64-16ms": 11.370700, + "Standard_M64-32ms": 11.370700, + "Standard_M64dms_v2": 11.374000, + "Standard_M64ds_v2": 7.336000, + "Standard_M64ls": 5.956500, + "Standard_M64m": 11.370700, + "Standard_M64ms": 11.370700, + "Standard_M64ms_v2": 11.143000, + "Standard_M64s": 7.335900, + "Standard_M64s_v2": 7.104000, + "Standard_M8-2ms": 1.690150, + "Standard_M8-4ms": 1.690150, + "Standard_M8ms": 1.690150, + "Standard_ND96ams_A100_v4": 39.324000, + "Standard_ND96amsr_A100_v4": 39.324000, + } + // centralindia + initialOnDemandPrices["centralindia"] = map[string]float64{ + "Basic_A0": 0.018000, + "Basic_A1": 0.030000, + "Basic_A2": 0.096000, + "Basic_A3": 0.232000, + "Basic_A4": 0.464000, + "DCdsv3 Type1": 6.336000, + "DCsv3 Type1": 5.333000, + "Dadsv5_Type1": 4.133000, + "Dasv4_Type1": 5.333000, + "Dasv4_Type2": 5.333000, + "Dasv5_Type1": 3.422000, + "Ddsv4_Type 1": 4.294000, + "Ddsv4_Type2": 5.100000, + "Ddsv5_Type1": 6.442000, + "Dsv3_Type3": 4.445000, + "Dsv3_Type4": 5.556000, + "Dsv4_Type1": 4.444000, + "Dsv4_Type2": 5.333000, + "Dsv5_Type1": 5.555000, + "Eadsv5_Type1": 4.385000, + "Easv4_Type1": 6.864000, + "Easv4_Type2": 6.864000, + "Easv5_Type1": 3.775000, + "Ebdsv5-Type1": 6.195000, + "Ebsv5-Type1": 5.456000, + "Edsv4_Type 1": 5.315000, + "Edsv4_Type2": 6.312000, + "Edsv5_Type1": 7.973000, + "Esv3_Type3": 4.506000, + "Esv3_Type4": 6.105000, + "Esv4_Type1": 4.576000, + "Esv4_Type2": 6.006000, + "Esv5_Type1": 6.864000, + "FXmds Type1": 5.174000, + "Fsv2 Type3": 3.927000, + "Fsv2_Type2": 3.366000, + "Lasv3_Type1": 7.788000, + "Lsv2_Type1": 7.788000, + "Lsv3_Type1": 8.712000, + "Mdmsv2MedMem _Type1": 30.290000, + "Mdsv2MedMem_Type1": 15.139000, + "Ms_Type1": 15.134000, + "Msm_Type1": 30.294000, + "Msmv2MedMem Type1": 29.812000, + "Msmv2_Type1": 112.340000, + "Msv2MedMem Type1": 14.662000, + "Msv2_Type1": 56.174000, + "NVasv4_Type1": 10.047000, + "NVsv3_Type1": 7.022000, + "Standard_A0": 0.020000, + "Standard_A1": 0.066000, + "Standard_A1_v2": 0.047000, + "Standard_A2": 0.131000, + "Standard_A2_v2": 0.098000, + "Standard_A2m_v2": 0.158000, + "Standard_A3": 0.262000, + "Standard_A4": 0.524000, + "Standard_A4_v2": 0.206000, + "Standard_A4m_v2": 0.333000, + "Standard_A5": 0.260000, + "Standard_A6": 0.520000, + "Standard_A7": 1.040000, + "Standard_A8_v2": 0.433000, + "Standard_A8m_v2": 0.699000, + "Standard_B12ms": 0.538000, + "Standard_B16als_v2": 0.349000, + "Standard_B16as_v2": 0.467000, + "Standard_B16ls_v2": 0.708000, + "Standard_B16ms": 0.717000, + "Standard_B16pls_v2": 0.391000, + "Standard_B16ps_v2": 0.432000, + "Standard_B16s_v2": 0.790000, + "Standard_B1ls": 0.005600, + "Standard_B1ms": 0.022400, + "Standard_B1s": 0.011200, + "Standard_B20ms": 0.896000, + "Standard_B2als_v2": 0.024600, + "Standard_B2as_v2": 0.058400, + "Standard_B2ats_v2": 0.006150, + "Standard_B2ls_v2": 0.044800, + "Standard_B2ms": 0.089600, + "Standard_B2pls_v2": 0.022400, + "Standard_B2ps_v2": 0.044800, + "Standard_B2pts_v2": 0.005600, + "Standard_B2s": 0.044800, + "Standard_B2s_v2": 0.089600, + "Standard_B2ts_v2": 0.011200, + "Standard_B32als_v2": 0.844000, + "Standard_B32as_v2": 0.934000, + "Standard_B32ls_v2": 1.417000, + "Standard_B32s_v2": 1.434000, + "Standard_B4als_v2": 0.106000, + "Standard_B4as_v2": 0.098400, + "Standard_B4ls_v2": 0.177000, + "Standard_B4ms": 0.179000, + "Standard_B4pls_v2": 0.097700, + "Standard_B4ps_v2": 0.108000, + "Standard_B4s_v2": 0.198000, + "Standard_B8als_v2": 0.211000, + "Standard_B8as_v2": 0.234000, + "Standard_B8ls_v2": 0.317000, + "Standard_B8ms": 0.358000, + "Standard_B8pls_v2": 0.159000, + "Standard_B8ps_v2": 0.216000, + "Standard_B8s_v2": 0.358000, + "Standard_D1": 0.070500, + "Standard_D11": 0.182000, + "Standard_D11_v2": 0.189000, + "Standard_D11_v2_Promo": 0.189000, + "Standard_D12": 0.365000, + "Standard_D12_v2": 0.379000, + "Standard_D12_v2_Promo": 0.379000, + "Standard_D13": 0.730000, + "Standard_D13_v2": 0.758000, + "Standard_D13_v2_Promo": 0.758000, + "Standard_D14": 1.460000, + "Standard_D14_v2": 1.516000, + "Standard_D14_v2_Promo": 1.516000, + "Standard_D15_v2": 1.895000, + "Standard_D15i_v2": 1.895000, + "Standard_D16_v3": 0.840000, + "Standard_D16_v4": 0.808000, + "Standard_D16_v5": 0.808000, + "Standard_D16a_v4": 0.493000, + "Standard_D16ads_v5": 1.273000, + "Standard_D16as_v4": 0.493000, + "Standard_D16as_v5": 0.444000, + "Standard_D16as_v5_Promo": 0.405000, + "Standard_D16d_v4": 0.976000, + "Standard_D16d_v5": 0.976000, + "Standard_D16ds_v4": 0.976000, + "Standard_D16ds_v5": 0.976000, + "Standard_D16lds_v5": 0.792000, + "Standard_D16ls_v5": 0.680000, + "Standard_D16pds_v5": 0.483000, + "Standard_D16plds_v5": 0.392000, + "Standard_D16pls_v5": 0.341000, + "Standard_D16ps_v5": 0.405000, + "Standard_D16s_v3": 0.840000, + "Standard_D16s_v4": 0.808000, + "Standard_D16s_v5": 0.808000, + "Standard_D1_v2": 0.084000, + "Standard_D2": 0.141000, + "Standard_D2_v2": 0.169000, + "Standard_D2_v2_Promo": 0.169000, + "Standard_D2_v3": 0.105000, + "Standard_D2_v4": 0.101000, + "Standard_D2_v5": 0.101000, + "Standard_D2a_v4": 0.062200, + "Standard_D2ads_v5": 0.067100, + "Standard_D2as_v4": 0.062200, + "Standard_D2as_v5": 0.055600, + "Standard_D2as_v5_Promo": 0.050600, + "Standard_D2d_v4": 0.122000, + "Standard_D2d_v5": 0.122000, + "Standard_D2ds_v4": 0.122000, + "Standard_D2ds_v5": 0.122000, + "Standard_D2lds_v5": 0.099000, + "Standard_D2ls_v5": 0.085000, + "Standard_D2pds_v5": 0.060400, + "Standard_D2plds_v5": 0.049000, + "Standard_D2pls_v5": 0.042600, + "Standard_D2ps_v5": 0.050600, + "Standard_D2s_v3": 0.105000, + "Standard_D2s_v4": 0.101000, + "Standard_D2s_v5": 0.101000, + "Standard_D3": 0.282000, + "Standard_D32-16s_v3": 1.680000, + "Standard_D32-8s_v3": 1.680000, + "Standard_D32_v3": 1.680000, + "Standard_D32_v4": 1.616000, + "Standard_D32_v5": 1.616000, + "Standard_D32a_v4": 0.987000, + "Standard_D32ads_v5": 1.074000, + "Standard_D32as_v4": 0.987000, + "Standard_D32as_v5": 2.361000, + "Standard_D32as_v5_Promo": 0.810000, + "Standard_D32d_v4": 1.952000, + "Standard_D32d_v5": 1.952000, + "Standard_D32ds_v4": 1.952000, + "Standard_D32ds_v5": 1.952000, + "Standard_D32lds_v5": 1.584000, + "Standard_D32ls_v5": 1.360000, + "Standard_D32pds_v5": 0.966000, + "Standard_D32plds_v5": 0.784000, + "Standard_D32pls_v5": 0.682000, + "Standard_D32ps_v5": 0.810000, + "Standard_D32s_v3": 1.680000, + "Standard_D32s_v4": 1.616000, + "Standard_D32s_v5": 1.616000, + "Standard_D3_v2": 0.337000, + "Standard_D3_v2_Promo": 0.337000, + "Standard_D4": 0.564000, + "Standard_D48_v3": 2.520000, + "Standard_D48_v4": 2.424000, + "Standard_D48_v5": 2.424000, + "Standard_D48a_v4": 1.481000, + "Standard_D48ads_v5": 1.610000, + "Standard_D48as_v4": 1.481000, + "Standard_D48as_v5": 1.333000, + "Standard_D48as_v5_Promo": 1.214000, + "Standard_D48d_v4": 2.928000, + "Standard_D48d_v5": 2.928000, + "Standard_D48ds_v4": 2.928000, + "Standard_D48ds_v5": 2.928000, + "Standard_D48lds_v5": 2.376000, + "Standard_D48ls_v5": 2.040000, + "Standard_D48pds_v5": 1.450000, + "Standard_D48plds_v5": 1.176000, + "Standard_D48pls_v5": 1.022000, + "Standard_D48ps_v5": 1.214000, + "Standard_D48s_v3": 2.520000, + "Standard_D48s_v4": 2.424000, + "Standard_D48s_v5": 2.424000, + "Standard_D4_v2": 0.675000, + "Standard_D4_v2_Promo": 0.675000, + "Standard_D4_v3": 0.210000, + "Standard_D4_v4": 0.202000, + "Standard_D4_v5": 0.202000, + "Standard_D4a_v4": 0.123000, + "Standard_D4ads_v5": 0.318000, + "Standard_D4as_v4": 0.123000, + "Standard_D4as_v5": 0.295000, + "Standard_D4as_v5_Promo": 0.101000, + "Standard_D4d_v4": 0.244000, + "Standard_D4d_v5": 0.244000, + "Standard_D4ds_v4": 0.244000, + "Standard_D4ds_v5": 0.244000, + "Standard_D4lds_v5": 0.198000, + "Standard_D4ls_v5": 0.170000, + "Standard_D4pds_v5": 0.121000, + "Standard_D4plds_v5": 0.098000, + "Standard_D4pls_v5": 0.085200, + "Standard_D4ps_v5": 0.101000, + "Standard_D4s_v3": 0.210000, + "Standard_D4s_v4": 0.202000, + "Standard_D4s_v5": 0.202000, + "Standard_D5_v2": 1.350000, + "Standard_D5_v2_Promo": 1.350000, + "Standard_D64-16s_v3": 3.360000, + "Standard_D64-32s_v3": 3.360000, + "Standard_D64_v3": 3.360000, + "Standard_D64_v4": 3.232000, + "Standard_D64_v5": 3.232000, + "Standard_D64a_v4": 1.975000, + "Standard_D64ads_v5": 2.147000, + "Standard_D64as_v4": 1.975000, + "Standard_D64as_v5": 4.722000, + "Standard_D64as_v5_Promo": 1.619000, + "Standard_D64d_v4": 3.904000, + "Standard_D64d_v5": 3.904000, + "Standard_D64ds_v4": 3.904000, + "Standard_D64ds_v5": 3.904000, + "Standard_D64lds_v5": 3.168000, + "Standard_D64ls_v5": 2.720000, + "Standard_D64pds_v5": 1.933000, + "Standard_D64plds_v5": 1.568000, + "Standard_D64pls_v5": 1.363000, + "Standard_D64ps_v5": 1.619000, + "Standard_D64s_v3": 3.360000, + "Standard_D64s_v4": 3.232000, + "Standard_D64s_v5": 3.232000, + "Standard_D8_v3": 0.420000, + "Standard_D8_v4": 0.404000, + "Standard_D8_v5": 0.404000, + "Standard_D8a_v4": 0.246000, + "Standard_D8ads_v5": 0.268000, + "Standard_D8as_v4": 0.246000, + "Standard_D8as_v5": 0.222000, + "Standard_D8as_v5_Promo": 0.202000, + "Standard_D8d_v4": 0.488000, + "Standard_D8d_v5": 0.488000, + "Standard_D8ds_v4": 0.488000, + "Standard_D8ds_v5": 0.488000, + "Standard_D8lds_v5": 0.396000, + "Standard_D8ls_v5": 0.340000, + "Standard_D8pds_v5": 0.242000, + "Standard_D8plds_v5": 0.196000, + "Standard_D8pls_v5": 0.170000, + "Standard_D8ps_v5": 0.202000, + "Standard_D8s_v3": 0.420000, + "Standard_D8s_v4": 0.404000, + "Standard_D8s_v5": 0.404000, + "Standard_D96_v5": 4.848000, + "Standard_D96a_v4": 2.962000, + "Standard_D96ads_v5": 7.637000, + "Standard_D96as_v4": 2.962000, + "Standard_D96as_v5": 7.082000, + "Standard_D96as_v5_Promo": 2.429000, + "Standard_D96d_v5": 5.856000, + "Standard_D96ds_v5": 5.856000, + "Standard_D96lds_v5": 4.752000, + "Standard_D96ls_v5": 4.080000, + "Standard_D96s_v5": 4.848000, + "Standard_DC16ads_v5": 0.537000, + "Standard_DC16as_v5": 0.444000, + "Standard_DC16ds_v3": 1.920000, + "Standard_DC16s_v3": 1.616000, + "Standard_DC1ds_v3": 0.120000, + "Standard_DC1s_v3": 0.101000, + "Standard_DC24ds_v3": 2.880000, + "Standard_DC24s_v3": 2.424000, + "Standard_DC2ads_v5": 0.067100, + "Standard_DC2as_v5": 0.055600, + "Standard_DC2ds_v3": 0.240000, + "Standard_DC2s_v3": 0.202000, + "Standard_DC32ads_v5": 1.074000, + "Standard_DC32as_v5": 0.889000, + "Standard_DC32ds_v3": 3.840000, + "Standard_DC32s_v3": 3.232000, + "Standard_DC48ads_v5": 1.610000, + "Standard_DC48as_v5": 1.333000, + "Standard_DC48ds_v3": 5.760000, + "Standard_DC48s_v3": 4.848000, + "Standard_DC4ads_v5": 0.134000, + "Standard_DC4as_v5": 0.111000, + "Standard_DC4ds_v3": 0.480000, + "Standard_DC4s_v3": 0.404000, + "Standard_DC64ads_v5": 2.147000, + "Standard_DC64as_v5": 1.778000, + "Standard_DC8ads_v5": 0.268000, + "Standard_DC8as_v5": 0.222000, + "Standard_DC8ds_v3": 0.960000, + "Standard_DC8s_v3": 0.808000, + "Standard_DC96ads_v5": 3.221000, + "Standard_DC96as_v5": 2.666000, + "Standard_DS1": 0.070500, + "Standard_DS11": 0.182000, + "Standard_DS11-1_v2": 0.189000, + "Standard_DS11_v2": 0.189000, + "Standard_DS11_v2_Promo": 0.189000, + "Standard_DS12": 0.365000, + "Standard_DS12-1_v2": 0.379000, + "Standard_DS12-2_v2": 0.379000, + "Standard_DS12_v2": 0.379000, + "Standard_DS12_v2_Promo": 0.379000, + "Standard_DS13": 0.730000, + "Standard_DS13-2_v2": 0.758000, + "Standard_DS13-4_v2": 0.758000, + "Standard_DS13_v2": 0.758000, + "Standard_DS13_v2_Promo": 0.758000, + "Standard_DS14": 1.460000, + "Standard_DS14-4_v2": 1.516000, + "Standard_DS14-8_v2": 1.516000, + "Standard_DS14_v2": 1.516000, + "Standard_DS14_v2_Promo": 1.516000, + "Standard_DS15_v2": 1.895000, + "Standard_DS15i_v2": 1.895000, + "Standard_DS1_v2": 0.084000, + "Standard_DS2": 0.141000, + "Standard_DS2_v2": 0.169000, + "Standard_DS2_v2_Promo": 0.169000, + "Standard_DS3": 0.282000, + "Standard_DS3_v2": 0.337000, + "Standard_DS3_v2_Promo": 0.337000, + "Standard_DS4": 0.564000, + "Standard_DS4_v2": 0.675000, + "Standard_DS4_v2_Promo": 0.675000, + "Standard_DS5_v2": 1.350000, + "Standard_DS5_v2_Promo": 1.350000, + "Standard_E104i_v5": 7.436000, + "Standard_E104id_v5": 8.637000, + "Standard_E104ids_v5": 8.637000, + "Standard_E104is_v5": 7.436000, + "Standard_E112iads_v5": 5.116000, + "Standard_E112ias_v5": 4.404000, + "Standard_E112ibds_v5": 10.842000, + "Standard_E112ibs_v5": 9.548000, + "Standard_E16-4ads_v5": 1.400000, + "Standard_E16-4as_v4": 1.040000, + "Standard_E16-4as_v5": 0.572000, + "Standard_E16-4ds_v4": 1.208000, + "Standard_E16-4ds_v5": 1.208000, + "Standard_E16-4s_v3": 1.096000, + "Standard_E16-4s_v4": 1.040000, + "Standard_E16-4s_v5": 1.040000, + "Standard_E16-8ads_v5": 0.664000, + "Standard_E16-8as_v4": 1.040000, + "Standard_E16-8as_v5": 0.572000, + "Standard_E16-8ds_v4": 1.208000, + "Standard_E16-8ds_v5": 1.208000, + "Standard_E16-8s_v3": 1.096000, + "Standard_E16-8s_v4": 1.040000, + "Standard_E16-8s_v5": 1.040000, + "Standard_E16_v3": 1.096000, + "Standard_E16_v4": 1.040000, + "Standard_E16_v5": 1.040000, + "Standard_E16a_v4": 0.635000, + "Standard_E16ads_v5": 0.710000, + "Standard_E16as_v4": 0.635000, + "Standard_E16as_v5": 0.572000, + "Standard_E16bds_v5": 1.408000, + "Standard_E16bs_v5": 1.240000, + "Standard_E16d_v4": 1.208000, + "Standard_E16d_v5": 1.208000, + "Standard_E16ds_v4": 1.208000, + "Standard_E16ds_v5": 1.208000, + "Standard_E16pds_v5": 0.656000, + "Standard_E16ps_v5": 0.520000, + "Standard_E16s_v3": 1.096000, + "Standard_E16s_v4": 1.040000, + "Standard_E16s_v5": 1.040000, + "Standard_E20_v3": 1.370000, + "Standard_E20_v4": 1.300000, + "Standard_E20_v5": 1.300000, + "Standard_E20a_v4": 0.794000, + "Standard_E20ads_v5": 0.830000, + "Standard_E20as_v4": 0.794000, + "Standard_E20as_v5": 0.761000, + "Standard_E20d_v4": 1.510000, + "Standard_E20d_v5": 1.510000, + "Standard_E20ds_v4": 1.510000, + "Standard_E20ds_v5": 1.510000, + "Standard_E20pds_v5": 0.820000, + "Standard_E20ps_v5": 0.650000, + "Standard_E20s_v3": 1.370000, + "Standard_E20s_v4": 1.300000, + "Standard_E20s_v5": 1.300000, + "Standard_E2_v3": 0.137000, + "Standard_E2_v4": 0.130000, + "Standard_E2_v5": 0.130000, + "Standard_E2a_v4": 0.080000, + "Standard_E2ads_v5": 0.129000, + "Standard_E2as_v4": 0.080000, + "Standard_E2as_v5": 0.071500, + "Standard_E2bds_v5": 0.176000, + "Standard_E2bs_v5": 0.155000, + "Standard_E2d_v4": 0.151000, + "Standard_E2d_v5": 0.151000, + "Standard_E2ds_v4": 0.151000, + "Standard_E2ds_v5": 0.151000, + "Standard_E2pds_v5": 0.082000, + "Standard_E2ps_v5": 0.065000, + "Standard_E2s_v3": 0.137000, + "Standard_E2s_v4": 0.130000, + "Standard_E2s_v5": 0.130000, + "Standard_E32-16ads_v5": 1.329000, + "Standard_E32-16as_v4": 2.080000, + "Standard_E32-16as_v5": 2.616000, + "Standard_E32-16ds_v4": 2.416000, + "Standard_E32-16ds_v5": 2.416000, + "Standard_E32-16s_v3": 2.192000, + "Standard_E32-16s_v4": 2.080000, + "Standard_E32-16s_v5": 2.080000, + "Standard_E32-8ads_v5": 1.329000, + "Standard_E32-8as_v4": 2.080000, + "Standard_E32-8as_v5": 1.144000, + "Standard_E32-8ds_v4": 2.416000, + "Standard_E32-8ds_v5": 2.416000, + "Standard_E32-8s_v3": 2.192000, + "Standard_E32-8s_v4": 2.080000, + "Standard_E32-8s_v5": 2.080000, + "Standard_E32_v3": 2.192000, + "Standard_E32_v4": 2.080000, + "Standard_E32_v5": 2.080000, + "Standard_E32a_v4": 1.271000, + "Standard_E32ads_v5": 1.329000, + "Standard_E32as_v4": 1.271000, + "Standard_E32as_v5": 1.190000, + "Standard_E32bds_v5": 2.816000, + "Standard_E32bs_v5": 2.480000, + "Standard_E32d_v4": 2.416000, + "Standard_E32d_v5": 2.416000, + "Standard_E32ds_v4": 2.416000, + "Standard_E32ds_v5": 2.416000, + "Standard_E32pds_v5": 1.312000, + "Standard_E32ps_v5": 1.040000, + "Standard_E32s_v3": 2.192000, + "Standard_E32s_v4": 2.080000, + "Standard_E32s_v5": 2.080000, + "Standard_E4-2ads_v5": 0.166000, + "Standard_E4-2as_v4": 0.260000, + "Standard_E4-2as_v5": 0.143000, + "Standard_E4-2ds_v4": 0.302000, + "Standard_E4-2ds_v5": 0.302000, + "Standard_E4-2s_v3": 0.274000, + "Standard_E4-2s_v4": 0.260000, + "Standard_E4-2s_v5": 0.260000, + "Standard_E48_v3": 3.281000, + "Standard_E48_v4": 3.120000, + "Standard_E48_v5": 3.120000, + "Standard_E48a_v4": 1.906000, + "Standard_E48ads_v5": 2.039000, + "Standard_E48as_v4": 1.906000, + "Standard_E48as_v5": 1.762000, + "Standard_E48bds_v5": 4.224000, + "Standard_E48bs_v5": 3.720000, + "Standard_E48d_v4": 3.624000, + "Standard_E48d_v5": 3.624000, + "Standard_E48ds_v4": 3.624000, + "Standard_E48ds_v5": 3.624000, + "Standard_E48s_v3": 3.281000, + "Standard_E48s_v4": 3.120000, + "Standard_E48s_v5": 3.120000, + "Standard_E4_v3": 0.274000, + "Standard_E4_v4": 0.260000, + "Standard_E4_v5": 0.260000, + "Standard_E4a_v4": 0.158000, + "Standard_E4ads_v5": 0.166000, + "Standard_E4as_v4": 0.158000, + "Standard_E4as_v5": 0.143000, + "Standard_E4bds_v5": 0.352000, + "Standard_E4bs_v5": 0.310000, + "Standard_E4d_v4": 0.302000, + "Standard_E4d_v5": 0.302000, + "Standard_E4ds_v4": 0.302000, + "Standard_E4ds_v5": 0.302000, + "Standard_E4pds_v5": 0.164000, + "Standard_E4ps_v5": 0.130000, + "Standard_E4s_v3": 0.274000, + "Standard_E4s_v4": 0.260000, + "Standard_E4s_v5": 0.260000, + "Standard_E64-16ads_v5": 2.658000, + "Standard_E64-16as_v4": 4.160000, + "Standard_E64-16as_v5": 5.232000, + "Standard_E64-16ds_v4": 4.832000, + "Standard_E64-16ds_v5": 4.832000, + "Standard_E64-16s_v3": 3.937000, + "Standard_E64-16s_v4": 4.160000, + "Standard_E64-16s_v5": 4.160000, + "Standard_E64-32ads_v5": 5.602000, + "Standard_E64-32as_v4": 4.160000, + "Standard_E64-32as_v5": 5.232000, + "Standard_E64-32ds_v4": 4.832000, + "Standard_E64-32ds_v5": 4.832000, + "Standard_E64-32s_v3": 3.937000, + "Standard_E64-32s_v4": 4.160000, + "Standard_E64-32s_v5": 4.160000, + "Standard_E64_v3": 3.937000, + "Standard_E64_v4": 4.160000, + "Standard_E64_v5": 4.160000, + "Standard_E64a_v4": 2.542000, + "Standard_E64ads_v5": 2.704000, + "Standard_E64as_v4": 2.542000, + "Standard_E64as_v5": 2.334000, + "Standard_E64bds_v5": 5.632000, + "Standard_E64bs_v5": 4.960000, + "Standard_E64d_v4": 4.832000, + "Standard_E64d_v5": 4.832000, + "Standard_E64ds_v4": 4.832000, + "Standard_E64ds_v5": 4.832000, + "Standard_E64s_v3": 3.937000, + "Standard_E64s_v4": 4.160000, + "Standard_E64s_v5": 4.160000, + "Standard_E8-2ads_v5": 0.332000, + "Standard_E8-2as_v4": 0.520000, + "Standard_E8-2as_v5": 0.654000, + "Standard_E8-2ds_v4": 0.604000, + "Standard_E8-2ds_v5": 0.604000, + "Standard_E8-2s_v3": 0.548000, + "Standard_E8-2s_v4": 0.520000, + "Standard_E8-2s_v5": 0.520000, + "Standard_E8-4ads_v5": 0.332000, + "Standard_E8-4as_v4": 0.520000, + "Standard_E8-4as_v5": 0.654000, + "Standard_E8-4ds_v4": 0.604000, + "Standard_E8-4ds_v5": 0.604000, + "Standard_E8-4s_v3": 0.548000, + "Standard_E8-4s_v4": 0.520000, + "Standard_E8-4s_v5": 0.520000, + "Standard_E80ids_v4": 6.040000, + "Standard_E80is_v4": 5.200000, + "Standard_E8_v3": 0.548000, + "Standard_E8_v4": 0.520000, + "Standard_E8_v5": 0.520000, + "Standard_E8a_v4": 0.317000, + "Standard_E8ads_v5": 0.378000, + "Standard_E8as_v4": 0.317000, + "Standard_E8as_v5": 0.286000, + "Standard_E8bds_v5": 0.704000, + "Standard_E8bs_v5": 0.620000, + "Standard_E8d_v4": 0.604000, + "Standard_E8d_v5": 0.604000, + "Standard_E8ds_v4": 0.604000, + "Standard_E8ds_v5": 0.604000, + "Standard_E8pds_v5": 0.328000, + "Standard_E8ps_v5": 0.260000, + "Standard_E8s_v3": 0.548000, + "Standard_E8s_v4": 0.520000, + "Standard_E8s_v5": 0.520000, + "Standard_E96-24ads_v5": 3.986000, + "Standard_E96-24as_v4": 6.240000, + "Standard_E96-24as_v5": 3.432000, + "Standard_E96-24ds_v5": 7.248000, + "Standard_E96-24s_v5": 6.240000, + "Standard_E96-48ads_v5": 3.986000, + "Standard_E96-48as_v4": 6.240000, + "Standard_E96-48as_v5": 3.432000, + "Standard_E96-48ds_v5": 7.248000, + "Standard_E96-48s_v5": 6.240000, + "Standard_E96_v5": 6.240000, + "Standard_E96a_v4": 3.813000, + "Standard_E96ads_v5": 3.986000, + "Standard_E96as_v4": 3.813000, + "Standard_E96as_v5": 3.478000, + "Standard_E96bds_v5": 8.448000, + "Standard_E96bs_v5": 7.440000, + "Standard_E96d_v5": 7.248000, + "Standard_E96ds_v5": 7.248000, + "Standard_E96iads_v5": 4.385000, + "Standard_E96ias_v5": 3.775000, + "Standard_E96s_v5": 6.240000, + "Standard_EC16ads_v5": 0.664000, + "Standard_EC16as_v5": 0.572000, + "Standard_EC20ads_v5": 0.831000, + "Standard_EC20as_v5": 0.715000, + "Standard_EC2ads_v5": 0.083100, + "Standard_EC2as_v5": 0.071500, + "Standard_EC32ads_v5": 1.329000, + "Standard_EC32as_v5": 1.144000, + "Standard_EC48ads_v5": 1.993000, + "Standard_EC48as_v5": 1.716000, + "Standard_EC4ads_v5": 0.166000, + "Standard_EC4as_v5": 0.143000, + "Standard_EC64ads_v5": 2.658000, + "Standard_EC64as_v5": 2.288000, + "Standard_EC8ads_v5": 0.332000, + "Standard_EC8as_v5": 0.286000, + "Standard_EC96ads_v5": 3.986000, + "Standard_EC96as_v5": 3.432000, + "Standard_EC96iads_v5": 4.385000, + "Standard_EC96ias_v5": 3.775000, + "Standard_F1": 0.049000, + "Standard_F16": 0.790000, + "Standard_F16s": 0.790000, + "Standard_F16s_v2": 0.680000, + "Standard_F1s": 0.049000, + "Standard_F2": 0.098800, + "Standard_F2s": 0.098800, + "Standard_F2s_v2": 0.085000, + "Standard_F32s_v2": 1.360000, + "Standard_F4": 0.198000, + "Standard_F48s_v2": 2.040000, + "Standard_F4s": 0.198000, + "Standard_F4s_v2": 0.170000, + "Standard_F64s_v2": 2.720000, + "Standard_F72s_v2": 3.060000, + "Standard_F8": 0.395000, + "Standard_F8s": 0.395000, + "Standard_F8s_v2": 0.340000, + "Standard_FX12mds": 1.176000, + "Standard_FX24mds": 2.352000, + "Standard_FX36mds": 3.528000, + "Standard_FX48mds": 4.704000, + "Standard_FX4mds": 0.392000, + "Standard_H16": 1.995000, + "Standard_H16m": 2.674000, + "Standard_H16mr": 2.941000, + "Standard_H16r": 2.195000, + "Standard_H8": 0.998000, + "Standard_H8m": 1.337000, + "Standard_HB120-16rs_v3": 5.040000, + "Standard_HB120-32rs_v3": 5.040000, + "Standard_HB120-64rs_v3": 5.040000, + "Standard_HB120-96rs_v3": 5.040000, + "Standard_HB120rs_v3": 5.040000, + "Standard_HC44-16rs": 4.435000, + "Standard_HC44-32rs": 4.435000, + "Standard_HC44rs": 4.435000, + "Standard_L16as_v3": 1.416000, + "Standard_L16s_v2": 1.416000, + "Standard_L16s_v3": 1.584000, + "Standard_L32as_v3": 2.832000, + "Standard_L32s_v2": 2.832000, + "Standard_L32s_v3": 3.168000, + "Standard_L48as_v3": 4.248000, + "Standard_L48s_v2": 4.248000, + "Standard_L48s_v3": 4.752000, + "Standard_L64as_v3": 5.664000, + "Standard_L64s_v2": 5.664000, + "Standard_L64s_v3": 6.336000, + "Standard_L80as_v3": 7.080000, + "Standard_L80s_v2": 7.080000, + "Standard_L80s_v3": 7.920000, + "Standard_L88is_v2": 7.788000, + "Standard_L8as_v3": 0.708000, + "Standard_L8s_v2": 0.708000, + "Standard_L8s_v3": 0.792000, + "Standard_L96s_v2": 8.496000, + "Standard_M128": 13.762000, + "Standard_M128-32ms": 27.524000, + "Standard_M128-64ms": 27.524000, + "Standard_M128dms_v2": 27.536000, + "Standard_M128ds_v2": 13.763000, + "Standard_M128m": 27.536000, + "Standard_M128ms": 27.524000, + "Standard_M128ms_v2": 27.102000, + "Standard_M128s": 13.762000, + "Standard_M128s_v2": 13.329000, + "Standard_M16-4ms": 3.170000, + "Standard_M16-8ms": 3.170000, + "Standard_M16ms": 3.170000, + "Standard_M16s": 2.462000, + "Standard_M192idms_v2": 33.080000, + "Standard_M192ids_v2": 16.540000, + "Standard_M192ims_v2": 32.646000, + "Standard_M192is_v2": 16.106000, + "Standard_M208ms_v2": 45.959000, + "Standard_M208s_v2": 22.979000, + "Standard_M32-16ms": 6.341000, + "Standard_M32-8ms": 6.341000, + "Standard_M32dms_v2": 6.341000, + "Standard_M32ls": 2.964074, + "Standard_M32ms": 6.341000, + "Standard_M32ms_v2": 6.232000, + "Standard_M32s": 3.440000, + "Standard_M32ts": 2.792812, + "Standard_M416is_v2": 51.067000, + "Standard_M416ms_v2": 102.120000, + "Standard_M416s_8_v2": 92.550000, + "Standard_M416s_v2": 51.067000, + "Standard_M64": 6.881000, + "Standard_M64-16ms": 10.666000, + "Standard_M64-32ms": 10.666000, + "Standard_M64dms_v2": 10.668000, + "Standard_M64ds_v2": 6.880000, + "Standard_M64ls": 5.586656, + "Standard_M64m": 10.666000, + "Standard_M64ms": 10.666000, + "Standard_M64ms_v2": 10.451000, + "Standard_M64s": 6.881000, + "Standard_M64s_v2": 6.663000, + "Standard_M8-2ms": 1.585000, + "Standard_M8-4ms": 1.585000, + "Standard_M8ms": 1.585000, + "Standard_NC12s_v3": 8.468000, + "Standard_NC16as_T4_v3": 1.324000, + "Standard_NC24ads_A100_v4": 5.142000, + "Standard_NC24rs_v3": 18.630000, + "Standard_NC24s_v3": 16.936000, + "Standard_NC48ads_A100_v4": 10.284000, + "Standard_NC4as_T4_v3": 0.579000, + "Standard_NC64as_T4_v3": 4.787000, + "Standard_NC6s_v3": 4.234000, + "Standard_NC8as_T4_v3": 0.827000, + "Standard_NC96ads_A100_v4": 20.569000, + "Standard_NV12": 3.436000, + "Standard_NV12ads_A10_v5": 1.271000, + "Standard_NV12s_v3": 1.596000, + "Standard_NV16as_v4": 1.305000, + "Standard_NV18ads_A10_v5": 2.240000, + "Standard_NV24": 6.872000, + "Standard_NV24s_v3": 3.192000, + "Standard_NV32as_v4": 2.610000, + "Standard_NV36adms_A10_v5": 6.328000, + "Standard_NV36ads_A10_v5": 4.480000, + "Standard_NV48s_v3": 6.384000, + "Standard_NV4as_v4": 0.326000, + "Standard_NV6": 1.718000, + "Standard_NV6ads_A10_v5": 0.636000, + "Standard_NV72ads_A10_v5": 9.128000, + "Standard_NV8as_v4": 0.652000, + } + // centralus + initialOnDemandPrices["centralus"] = map[string]float64{ + "Basic_A0": 0.018000, + "Basic_A1": 0.025000, + "Basic_A2": 0.085000, + "Basic_A3": 0.188000, + "Basic_A4": 0.376000, + "DCdsv3 Type1": 6.742000, + "DCsv3 Type1": 5.728000, + "Dadsv5_Type1": 7.170000, + "Dasv4_Type1": 5.728000, + "Dasv4_Type2": 5.728000, + "Dasv5_Type1": 5.986000, + "Ddsv4_Type 1": 4.495000, + "Ddsv4_Type2": 5.337000, + "Ddsv5_Type1": 6.742000, + "Dsv3_Type3": 4.774000, + "Dsv3_Type4": 5.968000, + "Dsv4_Type1": 4.773000, + "Dsv4_Type2": 5.966000, + "Dsv5_Type1": 5.966000, + "Eadsv5_Type1": 7.816000, + "Easv4_Type1": 7.518000, + "Easv4_Type2": 7.518000, + "Easv5_Type1": 6.742000, + "Ebdsv5-Type1": 6.643000, + "Ebsv5-Type1": 5.927000, + "Edsv4_Type 1": 5.728000, + "Edsv4_Type2": 6.802000, + "Edsv5_Type1": 8.592000, + "Esv3_Type3": 4.856000, + "Esv3_Type4": 6.579000, + "Esv4_Type1": 5.012000, + "Esv4_Type2": 6.578000, + "Esv5_Type1": 7.518000, + "Fsv2 Type3": 5.161000, + "Fsv2_Type2": 3.804000, + "Fsv2_Type4": 5.283000, + "Lasv3_Type1": 7.756000, + "Lsv2_Type1": 8.237000, + "Lsv3_Type1": 8.651000, + "Mdmsv2MedMem _Type1": 33.176000, + "Mdsv2MedMem_Type1": 16.582000, + "Ms_Type1": 16.576000, + "Msm_Type1": 33.180000, + "Msmv2MedMem Type1": 32.653000, + "Msmv2_Type1": 123.240000, + "Msv2MedMem Type1": 16.059000, + "Msv2_Type1": 61.628000, + "Standard_A0": 0.020000, + "Standard_A1": 0.060000, + "Standard_A1_v2": 0.043000, + "Standard_A2": 0.120000, + "Standard_A2_v2": 0.091000, + "Standard_A2m_v2": 0.129000, + "Standard_A3": 0.240000, + "Standard_A4": 0.480000, + "Standard_A4_v2": 0.191000, + "Standard_A4m_v2": 0.270000, + "Standard_A5": 0.250000, + "Standard_A6": 0.500000, + "Standard_A7": 1.000000, + "Standard_A8_v2": 0.400000, + "Standard_A8m_v2": 0.568000, + "Standard_B12ms": 0.599000, + "Standard_B16als_v2": 0.602000, + "Standard_B16as_v2": 0.680000, + "Standard_B16ls_v2": 0.666000, + "Standard_B16ms": 0.799000, + "Standard_B16pls_v2": 0.538000, + "Standard_B16ps_v2": 0.681000, + "Standard_B16s_v2": 0.826000, + "Standard_B1ls": 0.006240, + "Standard_B1ms": 0.025000, + "Standard_B1s": 0.012500, + "Standard_B20ms": 0.998000, + "Standard_B2als_v2": 0.042500, + "Standard_B2as_v2": 0.094200, + "Standard_B2ats_v2": 0.019800, + "Standard_B2ls_v2": 0.056200, + "Standard_B2ms": 0.099800, + "Standard_B2pls_v2": 0.047200, + "Standard_B2ps_v2": 0.085100, + "Standard_B2pts_v2": 0.018700, + "Standard_B2s": 0.049900, + "Standard_B2s_v2": 0.103000, + "Standard_B2ts_v2": 0.011800, + "Standard_B32als_v2": 1.204000, + "Standard_B32as_v2": 1.360000, + "Standard_B32ls_v2": 1.332000, + "Standard_B32s_v2": 1.504000, + "Standard_B4als_v2": 0.151000, + "Standard_B4as_v2": 0.170000, + "Standard_B4ls_v2": 0.185000, + "Standard_B4ms": 0.200000, + "Standard_B4pls_v2": 0.153000, + "Standard_B4ps_v2": 0.152000, + "Standard_B4s_v2": 0.188000, + "Standard_B8als_v2": 0.301000, + "Standard_B8as_v2": 0.377000, + "Standard_B8ls_v2": 0.370000, + "Standard_B8ms": 0.399000, + "Standard_B8pls_v2": 0.269000, + "Standard_B8ps_v2": 0.304000, + "Standard_B8s_v2": 0.413000, + "Standard_D1": 0.077000, + "Standard_D11": 0.193000, + "Standard_D11_v2": 0.185000, + "Standard_D11_v2_Promo": 0.185000, + "Standard_D12": 0.386000, + "Standard_D12_v2": 0.371000, + "Standard_D12_v2_Promo": 0.371000, + "Standard_D13": 0.771000, + "Standard_D13_v2": 0.741000, + "Standard_D13_v2_Promo": 0.741000, + "Standard_D14": 1.542000, + "Standard_D14_v2": 1.482000, + "Standard_D14_v2_Promo": 1.482000, + "Standard_D15_v2": 1.853000, + "Standard_D15i_v2": 1.853000, + "Standard_D16_v3": 0.880000, + "Standard_D16_v4": 0.868000, + "Standard_D16_v5": 0.868000, + "Standard_D16a_v4": 0.868000, + "Standard_D16ads_v5": 0.931000, + "Standard_D16as_v4": 0.868000, + "Standard_D16as_v5": 0.777000, + "Standard_D16d_v4": 1.022000, + "Standard_D16d_v5": 1.022000, + "Standard_D16ds_v4": 1.022000, + "Standard_D16ds_v5": 1.022000, + "Standard_D16lds_v5": 0.868000, + "Standard_D16ls_v5": 0.768000, + "Standard_D16pds_v5": 0.818000, + "Standard_D16plds_v5": 0.694000, + "Standard_D16pls_v5": 0.615000, + "Standard_D16ps_v5": 0.696000, + "Standard_D16s_v3": 0.880000, + "Standard_D16s_v4": 0.868000, + "Standard_D16s_v5": 0.868000, + "Standard_D1_v2": 0.073000, + "Standard_D2": 0.154000, + "Standard_D2_v2": 0.146000, + "Standard_D2_v2_Promo": 0.146000, + "Standard_D2_v3": 0.110000, + "Standard_D2_v4": 0.108000, + "Standard_D2_v5": 0.108000, + "Standard_D2a_v4": 0.108000, + "Standard_D2ads_v5": 0.116000, + "Standard_D2as_v4": 0.108000, + "Standard_D2as_v5": 0.189000, + "Standard_D2d_v4": 0.128000, + "Standard_D2d_v5": 0.128000, + "Standard_D2ds_v4": 0.128000, + "Standard_D2ds_v5": 0.128000, + "Standard_D2lds_v5": 0.108000, + "Standard_D2ls_v5": 0.096100, + "Standard_D2pds_v5": 0.102000, + "Standard_D2plds_v5": 0.086800, + "Standard_D2pls_v5": 0.076800, + "Standard_D2ps_v5": 0.087000, + "Standard_D2s_v3": 0.110000, + "Standard_D2s_v4": 0.108000, + "Standard_D2s_v5": 0.108000, + "Standard_D3": 0.308000, + "Standard_D32-16s_v3": 1.760000, + "Standard_D32-8s_v3": 1.760000, + "Standard_D32_v3": 1.760000, + "Standard_D32_v4": 1.736000, + "Standard_D32_v5": 1.736000, + "Standard_D32a_v4": 1.736000, + "Standard_D32ads_v5": 3.334000, + "Standard_D32as_v4": 1.736000, + "Standard_D32as_v5": 1.555000, + "Standard_D32d_v4": 2.043000, + "Standard_D32d_v5": 2.043000, + "Standard_D32ds_v4": 2.043000, + "Standard_D32ds_v5": 2.043000, + "Standard_D32lds_v5": 1.736000, + "Standard_D32ls_v5": 1.537000, + "Standard_D32pds_v5": 1.635000, + "Standard_D32plds_v5": 1.389000, + "Standard_D32pls_v5": 1.229000, + "Standard_D32ps_v5": 1.392000, + "Standard_D32s_v3": 1.760000, + "Standard_D32s_v4": 1.736000, + "Standard_D32s_v5": 1.736000, + "Standard_D3_v2": 0.293000, + "Standard_D3_v2_Promo": 0.293000, + "Standard_D4": 0.616000, + "Standard_D48_v3": 2.640000, + "Standard_D48_v4": 2.604000, + "Standard_D48_v5": 2.604000, + "Standard_D48a_v4": 2.604000, + "Standard_D48ads_v5": 2.793000, + "Standard_D48as_v4": 2.604000, + "Standard_D48as_v5": 2.332000, + "Standard_D48d_v4": 3.065000, + "Standard_D48d_v5": 3.065000, + "Standard_D48ds_v4": 3.065000, + "Standard_D48ds_v5": 3.065000, + "Standard_D48lds_v5": 2.604000, + "Standard_D48ls_v5": 2.305000, + "Standard_D48pds_v5": 2.453000, + "Standard_D48plds_v5": 2.083000, + "Standard_D48pls_v5": 1.844000, + "Standard_D48ps_v5": 2.088000, + "Standard_D48s_v3": 2.640000, + "Standard_D48s_v4": 2.604000, + "Standard_D48s_v5": 2.604000, + "Standard_D4_v2": 0.585000, + "Standard_D4_v2_Promo": 0.585000, + "Standard_D4_v3": 0.220000, + "Standard_D4_v4": 0.217000, + "Standard_D4_v5": 0.217000, + "Standard_D4a_v4": 0.217000, + "Standard_D4ads_v5": 0.417000, + "Standard_D4as_v4": 0.217000, + "Standard_D4as_v5": 0.378000, + "Standard_D4d_v4": 0.255000, + "Standard_D4d_v5": 0.255000, + "Standard_D4ds_v4": 0.255000, + "Standard_D4ds_v5": 0.255000, + "Standard_D4lds_v5": 0.217000, + "Standard_D4ls_v5": 0.192000, + "Standard_D4pds_v5": 0.204000, + "Standard_D4plds_v5": 0.174000, + "Standard_D4pls_v5": 0.154000, + "Standard_D4ps_v5": 0.174000, + "Standard_D4s_v3": 0.220000, + "Standard_D4s_v4": 0.217000, + "Standard_D4s_v5": 0.217000, + "Standard_D5_v2": 1.170000, + "Standard_D5_v2_Promo": 1.170000, + "Standard_D64-16s_v3": 3.520000, + "Standard_D64-32s_v3": 3.520000, + "Standard_D64_v3": 3.520000, + "Standard_D64_v4": 3.471000, + "Standard_D64_v5": 3.471000, + "Standard_D64a_v4": 3.471000, + "Standard_D64ads_v5": 3.724000, + "Standard_D64as_v4": 3.471000, + "Standard_D64as_v5": 3.110000, + "Standard_D64d_v4": 4.086000, + "Standard_D64d_v5": 4.086000, + "Standard_D64ds_v4": 4.086000, + "Standard_D64ds_v5": 4.086000, + "Standard_D64lds_v5": 3.471000, + "Standard_D64ls_v5": 3.074000, + "Standard_D64pds_v5": 3.270000, + "Standard_D64plds_v5": 2.777000, + "Standard_D64pls_v5": 2.459000, + "Standard_D64ps_v5": 2.784000, + "Standard_D64s_v3": 3.520000, + "Standard_D64s_v4": 3.471000, + "Standard_D64s_v5": 3.471000, + "Standard_D8_v3": 0.440000, + "Standard_D8_v4": 0.434000, + "Standard_D8_v5": 0.434000, + "Standard_D8a_v4": 0.434000, + "Standard_D8ads_v5": 0.466000, + "Standard_D8as_v4": 0.434000, + "Standard_D8as_v5": 0.757000, + "Standard_D8d_v4": 0.511000, + "Standard_D8d_v5": 0.511000, + "Standard_D8ds_v4": 0.511000, + "Standard_D8ds_v5": 0.511000, + "Standard_D8lds_v5": 0.434000, + "Standard_D8ls_v5": 0.384000, + "Standard_D8pds_v5": 0.409000, + "Standard_D8plds_v5": 0.347000, + "Standard_D8pls_v5": 0.307000, + "Standard_D8ps_v5": 0.348000, + "Standard_D8s_v3": 0.440000, + "Standard_D8s_v4": 0.434000, + "Standard_D8s_v5": 0.434000, + "Standard_D96_v5": 5.207000, + "Standard_D96a_v4": 5.207000, + "Standard_D96ads_v5": 10.003000, + "Standard_D96as_v4": 5.207000, + "Standard_D96as_v5": 9.081000, + "Standard_D96d_v5": 6.129000, + "Standard_D96ds_v5": 6.129000, + "Standard_D96lds_v5": 5.207000, + "Standard_D96ls_v5": 4.610000, + "Standard_D96s_v5": 5.207000, + "Standard_DC16ads_v5": 0.931000, + "Standard_DC16as_v5": 0.777000, + "Standard_DC16ds_v3": 2.043000, + "Standard_DC16eds_v5": 1.022000, + "Standard_DC16es_v5": 0.868000, + "Standard_DC16s_v3": 1.736000, + "Standard_DC1ds_v3": 0.128000, + "Standard_DC1s_v3": 0.108000, + "Standard_DC24ds_v3": 3.065000, + "Standard_DC24s_v3": 2.604000, + "Standard_DC2ads_v5": 0.116000, + "Standard_DC2as_v5": 0.097200, + "Standard_DC2ds_v3": 0.255000, + "Standard_DC2eds_v5": 0.128000, + "Standard_DC2es_v5": 0.108000, + "Standard_DC2s": 0.474000, + "Standard_DC2s_v3": 0.217000, + "Standard_DC32ads_v5": 1.862000, + "Standard_DC32as_v5": 1.555000, + "Standard_DC32ds_v3": 4.086000, + "Standard_DC32eds_v5": 2.043000, + "Standard_DC32es_v5": 1.736000, + "Standard_DC32s_v3": 3.471000, + "Standard_DC48ads_v5": 2.793000, + "Standard_DC48as_v5": 2.332000, + "Standard_DC48ds_v3": 6.129000, + "Standard_DC48eds_v5": 3.065000, + "Standard_DC48es_v5": 2.604000, + "Standard_DC48s_v3": 5.207000, + "Standard_DC4ads_v5": 0.233000, + "Standard_DC4as_v5": 0.194000, + "Standard_DC4ds_v3": 0.511000, + "Standard_DC4eds_v5": 0.255000, + "Standard_DC4es_v5": 0.217000, + "Standard_DC4s": 0.949000, + "Standard_DC4s_v3": 0.434000, + "Standard_DC64ads_v5": 3.724000, + "Standard_DC64as_v5": 3.110000, + "Standard_DC64eds_v5": 4.086000, + "Standard_DC64es_v5": 3.471000, + "Standard_DC8ads_v5": 0.466000, + "Standard_DC8as_v5": 0.389000, + "Standard_DC8ds_v3": 1.022000, + "Standard_DC8eds_v5": 0.511000, + "Standard_DC8es_v5": 0.434000, + "Standard_DC8s_v3": 0.868000, + "Standard_DC96ads_v5": 5.587000, + "Standard_DC96as_v5": 4.665000, + "Standard_DC96eds_v5": 6.129000, + "Standard_DC96es_v5": 5.207000, + "Standard_DS1": 0.077000, + "Standard_DS11": 0.193000, + "Standard_DS11-1_v2": 0.185000, + "Standard_DS11_v2": 0.185000, + "Standard_DS11_v2_Promo": 0.185000, + "Standard_DS12": 0.386000, + "Standard_DS12-1_v2": 0.371000, + "Standard_DS12-2_v2": 0.371000, + "Standard_DS12_v2": 0.371000, + "Standard_DS12_v2_Promo": 0.371000, + "Standard_DS13": 0.771000, + "Standard_DS13-2_v2": 0.741000, + "Standard_DS13-4_v2": 0.741000, + "Standard_DS13_v2": 0.741000, + "Standard_DS13_v2_Promo": 0.741000, + "Standard_DS14": 1.542000, + "Standard_DS14-4_v2": 1.482000, + "Standard_DS14-8_v2": 1.482000, + "Standard_DS14_v2": 1.482000, + "Standard_DS14_v2_Promo": 1.482000, + "Standard_DS15_v2": 1.853000, + "Standard_DS15i_v2": 1.853000, + "Standard_DS1_v2": 0.073000, + "Standard_DS2": 0.154000, + "Standard_DS2_v2": 0.146000, + "Standard_DS2_v2_Promo": 0.146000, + "Standard_DS3": 0.308000, + "Standard_DS3_v2": 0.293000, + "Standard_DS3_v2_Promo": 0.293000, + "Standard_DS4": 0.616000, + "Standard_DS4_v2": 0.585000, + "Standard_DS4_v2_Promo": 0.585000, + "Standard_DS5_v2": 1.170000, + "Standard_DS5_v2_Promo": 1.170000, + "Standard_E104i_v5": 8.144000, + "Standard_E104id_v5": 9.308000, + "Standard_E104ids_v5": 9.308000, + "Standard_E104is_v5": 8.144000, + "Standard_E112iads_v5": 9.119000, + "Standard_E112ias_v5": 7.866000, + "Standard_E112ibds_v5": 11.625000, + "Standard_E112ibs_v5": 10.372000, + "Standard_E16-4ads_v5": 1.920000, + "Standard_E16-4as_v4": 1.139000, + "Standard_E16-4as_v5": 1.758000, + "Standard_E16-4ds_v4": 1.302000, + "Standard_E16-4ds_v5": 1.302000, + "Standard_E16-4s_v3": 1.167000, + "Standard_E16-4s_v4": 1.139000, + "Standard_E16-4s_v5": 1.139000, + "Standard_E16-8ads_v5": 1.920000, + "Standard_E16-8as_v4": 1.139000, + "Standard_E16-8as_v5": 1.758000, + "Standard_E16-8ds_v4": 1.302000, + "Standard_E16-8ds_v5": 1.302000, + "Standard_E16-8s_v3": 1.167000, + "Standard_E16-8s_v4": 1.139000, + "Standard_E16-8s_v5": 1.139000, + "Standard_E16_v3": 1.167000, + "Standard_E16_v4": 1.139000, + "Standard_E16_v5": 1.139000, + "Standard_E16a_v4": 1.139000, + "Standard_E16ads_v5": 1.184000, + "Standard_E16as_v4": 1.139000, + "Standard_E16as_v5": 1.022000, + "Standard_E16bds_v5": 1.510000, + "Standard_E16bs_v5": 1.347000, + "Standard_E16d_v4": 1.302000, + "Standard_E16d_v5": 1.302000, + "Standard_E16ds_v4": 1.302000, + "Standard_E16ds_v5": 1.302000, + "Standard_E16pds_v5": 1.041000, + "Standard_E16ps_v5": 0.911000, + "Standard_E16s_v3": 1.167000, + "Standard_E16s_v4": 1.139000, + "Standard_E16s_v5": 1.139000, + "Standard_E20_v3": 1.460000, + "Standard_E20_v4": 1.424000, + "Standard_E20_v5": 1.424000, + "Standard_E20a_v4": 1.424000, + "Standard_E20ads_v5": 1.526000, + "Standard_E20as_v4": 1.424000, + "Standard_E20as_v5": 1.277000, + "Standard_E20d_v4": 1.627000, + "Standard_E20d_v5": 1.627000, + "Standard_E20ds_v4": 1.627000, + "Standard_E20ds_v5": 1.627000, + "Standard_E20pds_v5": 1.302000, + "Standard_E20ps_v5": 1.139000, + "Standard_E20s_v3": 1.460000, + "Standard_E20s_v4": 1.424000, + "Standard_E20s_v5": 1.424000, + "Standard_E2_v3": 0.146000, + "Standard_E2_v4": 0.142000, + "Standard_E2_v5": 0.142000, + "Standard_E2a_v4": 0.142000, + "Standard_E2ads_v5": 0.194000, + "Standard_E2as_v4": 0.142000, + "Standard_E2as_v5": 0.128000, + "Standard_E2bds_v5": 0.189000, + "Standard_E2bs_v5": 0.168000, + "Standard_E2d_v4": 0.163000, + "Standard_E2d_v5": 0.163000, + "Standard_E2ds_v4": 0.163000, + "Standard_E2ds_v5": 0.163000, + "Standard_E2pds_v5": 0.130000, + "Standard_E2ps_v5": 0.114000, + "Standard_E2s_v3": 0.146000, + "Standard_E2s_v4": 0.142000, + "Standard_E2s_v5": 0.142000, + "Standard_E32-16ads_v5": 2.368000, + "Standard_E32-16as_v4": 2.278000, + "Standard_E32-16as_v5": 2.043000, + "Standard_E32-16ds_v4": 2.604000, + "Standard_E32-16ds_v5": 2.604000, + "Standard_E32-16s_v3": 2.334000, + "Standard_E32-16s_v4": 2.278000, + "Standard_E32-16s_v5": 2.278000, + "Standard_E32-8ads_v5": 3.840000, + "Standard_E32-8as_v4": 2.278000, + "Standard_E32-8as_v5": 2.043000, + "Standard_E32-8ds_v4": 2.604000, + "Standard_E32-8ds_v5": 2.604000, + "Standard_E32-8s_v3": 2.334000, + "Standard_E32-8s_v4": 2.278000, + "Standard_E32-8s_v5": 2.278000, + "Standard_E32_v3": 2.334000, + "Standard_E32_v4": 2.278000, + "Standard_E32_v5": 2.278000, + "Standard_E32a_v4": 2.278000, + "Standard_E32ads_v5": 2.368000, + "Standard_E32as_v4": 2.278000, + "Standard_E32as_v5": 2.089000, + "Standard_E32bds_v5": 3.019000, + "Standard_E32bs_v5": 2.694000, + "Standard_E32d_v4": 2.604000, + "Standard_E32d_v5": 2.604000, + "Standard_E32ds_v4": 2.604000, + "Standard_E32ds_v5": 2.604000, + "Standard_E32pds_v5": 2.083000, + "Standard_E32ps_v5": 1.822000, + "Standard_E32s_v3": 2.334000, + "Standard_E32s_v4": 2.278000, + "Standard_E32s_v5": 2.278000, + "Standard_E4-2ads_v5": 0.296000, + "Standard_E4-2as_v4": 0.285000, + "Standard_E4-2as_v5": 0.255000, + "Standard_E4-2ds_v4": 0.325000, + "Standard_E4-2ds_v5": 0.325000, + "Standard_E4-2s_v3": 0.292000, + "Standard_E4-2s_v4": 0.285000, + "Standard_E4-2s_v5": 0.285000, + "Standard_E48_v3": 3.501000, + "Standard_E48_v4": 3.417000, + "Standard_E48_v5": 3.417000, + "Standard_E48a_v4": 3.417000, + "Standard_E48ads_v5": 3.553000, + "Standard_E48as_v4": 3.417000, + "Standard_E48as_v5": 3.065000, + "Standard_E48bds_v5": 4.529000, + "Standard_E48bs_v5": 4.041000, + "Standard_E48d_v4": 3.905000, + "Standard_E48d_v5": 3.905000, + "Standard_E48ds_v4": 3.905000, + "Standard_E48ds_v5": 3.905000, + "Standard_E48s_v3": 3.501000, + "Standard_E48s_v4": 3.417000, + "Standard_E48s_v5": 3.417000, + "Standard_E4_v3": 0.292000, + "Standard_E4_v4": 0.285000, + "Standard_E4_v5": 0.285000, + "Standard_E4a_v4": 0.285000, + "Standard_E4ads_v5": 0.342000, + "Standard_E4as_v4": 0.285000, + "Standard_E4as_v5": 0.301000, + "Standard_E4bds_v5": 0.377000, + "Standard_E4bs_v5": 0.337000, + "Standard_E4d_v4": 0.325000, + "Standard_E4d_v5": 0.325000, + "Standard_E4ds_v4": 0.325000, + "Standard_E4ds_v5": 0.325000, + "Standard_E4pds_v5": 0.260000, + "Standard_E4ps_v5": 0.228000, + "Standard_E4s_v3": 0.292000, + "Standard_E4s_v4": 0.285000, + "Standard_E4s_v5": 0.285000, + "Standard_E64-16ads_v5": 4.737000, + "Standard_E64-16as_v4": 4.556000, + "Standard_E64-16as_v5": 7.030000, + "Standard_E64-16ds_v4": 5.207000, + "Standard_E64-16ds_v5": 5.207000, + "Standard_E64-16s_v3": 4.200000, + "Standard_E64-16s_v4": 4.556000, + "Standard_E64-16s_v5": 4.556000, + "Standard_E64-32ads_v5": 7.681000, + "Standard_E64-32as_v4": 4.556000, + "Standard_E64-32as_v5": 4.086000, + "Standard_E64-32ds_v4": 5.207000, + "Standard_E64-32ds_v5": 5.207000, + "Standard_E64-32s_v3": 4.200000, + "Standard_E64-32s_v4": 4.556000, + "Standard_E64-32s_v5": 4.556000, + "Standard_E64_v3": 4.200000, + "Standard_E64_v4": 4.556000, + "Standard_E64_v5": 4.556000, + "Standard_E64a_v4": 4.556000, + "Standard_E64ads_v5": 4.783000, + "Standard_E64as_v4": 4.556000, + "Standard_E64as_v5": 4.132000, + "Standard_E64bds_v5": 6.039000, + "Standard_E64bs_v5": 5.388000, + "Standard_E64d_v4": 5.207000, + "Standard_E64d_v5": 5.207000, + "Standard_E64ds_v4": 5.207000, + "Standard_E64ds_v5": 5.207000, + "Standard_E64i_v3": 4.199000, + "Standard_E64is_v3": 4.199000, + "Standard_E64s_v3": 4.200000, + "Standard_E64s_v4": 4.556000, + "Standard_E64s_v5": 4.556000, + "Standard_E8-2ads_v5": 0.960000, + "Standard_E8-2as_v4": 0.570000, + "Standard_E8-2as_v5": 0.879000, + "Standard_E8-2ds_v4": 0.651000, + "Standard_E8-2ds_v5": 0.651000, + "Standard_E8-2s_v3": 0.583000, + "Standard_E8-2s_v4": 0.570000, + "Standard_E8-2s_v5": 0.570000, + "Standard_E8-4ads_v5": 0.960000, + "Standard_E8-4as_v4": 0.570000, + "Standard_E8-4as_v5": 0.879000, + "Standard_E8-4ds_v4": 0.651000, + "Standard_E8-4ds_v5": 0.651000, + "Standard_E8-4s_v3": 0.583000, + "Standard_E8-4s_v4": 0.570000, + "Standard_E8-4s_v5": 0.570000, + "Standard_E80ids_v4": 6.509000, + "Standard_E80is_v4": 5.695000, + "Standard_E8_v3": 0.583000, + "Standard_E8_v4": 0.570000, + "Standard_E8_v5": 0.570000, + "Standard_E8a_v4": 0.570000, + "Standard_E8ads_v5": 0.638000, + "Standard_E8as_v4": 0.570000, + "Standard_E8as_v5": 0.557000, + "Standard_E8bds_v5": 0.755000, + "Standard_E8bs_v5": 0.673000, + "Standard_E8d_v4": 0.651000, + "Standard_E8d_v5": 0.651000, + "Standard_E8ds_v4": 0.651000, + "Standard_E8ds_v5": 0.651000, + "Standard_E8pds_v5": 0.521000, + "Standard_E8ps_v5": 0.456000, + "Standard_E8s_v3": 0.583000, + "Standard_E8s_v4": 0.570000, + "Standard_E8s_v5": 0.570000, + "Standard_E96-24ads_v5": 11.521000, + "Standard_E96-24as_v4": 6.834000, + "Standard_E96-24as_v5": 6.129000, + "Standard_E96-24ds_v5": 7.811000, + "Standard_E96-24s_v5": 6.834000, + "Standard_E96-48ads_v5": 11.521000, + "Standard_E96-48as_v4": 6.834000, + "Standard_E96-48as_v5": 10.545000, + "Standard_E96-48ds_v5": 7.811000, + "Standard_E96-48s_v5": 6.834000, + "Standard_E96_v5": 6.834000, + "Standard_E96a_v4": 6.834000, + "Standard_E96ads_v5": 7.105000, + "Standard_E96as_v4": 6.834000, + "Standard_E96as_v5": 6.129000, + "Standard_E96bds_v5": 9.058000, + "Standard_E96bs_v5": 8.082000, + "Standard_E96d_v5": 7.811000, + "Standard_E96ds_v5": 7.811000, + "Standard_E96iads_v5": 7.816000, + "Standard_E96ias_v5": 6.742000, + "Standard_E96s_v5": 6.834000, + "Standard_EC16ads_v5": 1.184000, + "Standard_EC16as_v5": 1.022000, + "Standard_EC16eds_v5": 1.302000, + "Standard_EC16es_v5": 1.139000, + "Standard_EC20ads_v5": 1.480000, + "Standard_EC20as_v5": 1.277000, + "Standard_EC2ads_v5": 0.148000, + "Standard_EC2as_v5": 0.128000, + "Standard_EC2eds_v5": 0.163000, + "Standard_EC2es_v5": 0.142000, + "Standard_EC32ads_v5": 2.368000, + "Standard_EC32as_v5": 2.043000, + "Standard_EC32eds_v5": 2.604000, + "Standard_EC32es_v5": 2.278000, + "Standard_EC48ads_v5": 3.553000, + "Standard_EC48as_v5": 3.065000, + "Standard_EC48eds_v5": 3.905000, + "Standard_EC48es_v5": 3.417000, + "Standard_EC4ads_v5": 0.296000, + "Standard_EC4as_v5": 0.255000, + "Standard_EC4eds_v5": 0.325000, + "Standard_EC4es_v5": 0.285000, + "Standard_EC64ads_v5": 4.737000, + "Standard_EC64as_v5": 4.086000, + "Standard_EC64eds_v5": 5.207000, + "Standard_EC64es_v5": 4.556000, + "Standard_EC8ads_v5": 0.592000, + "Standard_EC8as_v5": 0.511000, + "Standard_EC8eds_v5": 0.651000, + "Standard_EC8es_v5": 0.570000, + "Standard_EC96ads_v5": 7.105000, + "Standard_EC96as_v5": 6.129000, + "Standard_EC96eds_v5": 7.811000, + "Standard_EC96es_v5": 6.834000, + "Standard_EC96iads_v5": 7.816000, + "Standard_EC96ias_v5": 6.742000, + "Standard_F1": 0.054700, + "Standard_F16": 0.875000, + "Standard_F16s": 0.875000, + "Standard_F16s_v2": 0.816000, + "Standard_F1s": 0.054700, + "Standard_F2": 0.109000, + "Standard_F2s": 0.109000, + "Standard_F2s_v2": 0.102000, + "Standard_F32s_v2": 1.632000, + "Standard_F4": 0.219000, + "Standard_F48s_v2": 2.448000, + "Standard_F4s": 0.219000, + "Standard_F4s_v2": 0.204000, + "Standard_F64s_v2": 3.264000, + "Standard_F72s_v2": 3.672000, + "Standard_F8": 0.438000, + "Standard_F8s": 0.438000, + "Standard_F8s_v2": 0.408000, + "Standard_HB120-16rs_v2": 4.428000, + "Standard_HB120-32rs_v2": 4.428000, + "Standard_HB120-64rs_v2": 4.428000, + "Standard_HB120-96rs_v2": 4.428000, + "Standard_HB120rs_v2": 4.428000, + "Standard_L16as_v3": 1.410000, + "Standard_L16s_v2": 1.555000, + "Standard_L16s_v3": 1.573000, + "Standard_L32as_v3": 2.820000, + "Standard_L32s_v2": 3.110000, + "Standard_L32s_v3": 3.146000, + "Standard_L48as_v3": 4.231000, + "Standard_L48s_v2": 4.665000, + "Standard_L48s_v3": 4.719000, + "Standard_L64as_v3": 5.641000, + "Standard_L64s_v2": 6.220000, + "Standard_L64s_v3": 6.292000, + "Standard_L80as_v3": 7.051000, + "Standard_L80s_v2": 7.774000, + "Standard_L80s_v3": 7.865000, + "Standard_L88is_v2": 8.552000, + "Standard_L8as_v3": 0.705000, + "Standard_L8s_v2": 0.777000, + "Standard_L8s_v3": 0.786000, + "Standard_M128": 15.072000, + "Standard_M128-32ms": 30.157000, + "Standard_M128-64ms": 30.157000, + "Standard_M128dms_v2": 30.160000, + "Standard_M128ds_v2": 15.074000, + "Standard_M128m": 30.157000, + "Standard_M128ms": 30.157000, + "Standard_M128ms_v2": 29.684000, + "Standard_M128s": 15.072000, + "Standard_M128s_v2": 14.599000, + "Standard_M16-4ms": 3.472000, + "Standard_M16-8ms": 3.472000, + "Standard_M16ms": 3.472000, + "Standard_M16s": 2.697000, + "Standard_M192idms_v2": 36.232000, + "Standard_M192ids_v2": 18.116000, + "Standard_M192ims_v2": 35.757000, + "Standard_M192is_v2": 17.641000, + "Standard_M208-104ms_v2": 50.421000, + "Standard_M208-104s_v2": 25.210000, + "Standard_M208-52ms_v2": 50.421000, + "Standard_M208-52s_v2": 25.210000, + "Standard_M208ms_v2": 50.421000, + "Standard_M208s_v2": 25.210000, + "Standard_M32-16ms": 6.945000, + "Standard_M32-8ms": 6.945000, + "Standard_M32dms_v2": 6.945000, + "Standard_M32ls": 3.246000, + "Standard_M32ms": 6.945000, + "Standard_M32ms_v2": 6.826000, + "Standard_M32s": 5.395000, + "Standard_M32ts": 3.059000, + "Standard_M416-104ms_v2": 112.040000, + "Standard_M416-104s_v2": 56.025000, + "Standard_M416-208ms_v2": 112.040000, + "Standard_M416-208s_v2": 56.025000, + "Standard_M416is_v2": 56.025000, + "Standard_M416ms_v2": 112.040000, + "Standard_M416s_8_v2": 74.700000, + "Standard_M416s_v2": 56.025000, + "Standard_M64": 7.536000, + "Standard_M64-16ms": 15.079000, + "Standard_M64-32ms": 15.079000, + "Standard_M64dms_v2": 11.684000, + "Standard_M64ds_v2": 7.536000, + "Standard_M64ls": 6.493000, + "Standard_M64m": 15.079000, + "Standard_M64ms": 15.079000, + "Standard_M64ms_v2": 11.446000, + "Standard_M64s": 7.536000, + "Standard_M64s_v2": 7.298000, + "Standard_M8-2ms": 1.736000, + "Standard_M8-4ms": 1.736000, + "Standard_M8ms": 1.736000, + "Standard_NC12s_v3": 6.916000, + "Standard_NC16as_T4_v3": 1.360000, + "Standard_NC24rs_v3": 15.214000, + "Standard_NC24s_v3": 13.831000, + "Standard_NC4as_T4_v3": 0.594000, + "Standard_NC64as_T4_v3": 4.907000, + "Standard_NC6s_v3": 3.458000, + "Standard_NC8as_T4_v3": 0.849000, + "Standard_ND96amsr_A100_v4": 37.030000, + "Standard_NV12ads_A10_v5": 1.117000, + "Standard_NV18ads_A10_v5": 1.968000, + "Standard_NV36adms_A10_v5": 5.560000, + "Standard_NV36ads_A10_v5": 3.936000, + "Standard_NV6ads_A10_v5": 0.558000, + "Standard_NV72ads_A10_v5": 8.020000, + } + // eastasia + initialOnDemandPrices["eastasia"] = map[string]float64{ + "Basic_A0": 0.018000, + "Basic_A1": 0.038000, + "Basic_A2": 0.104000, + "Basic_A3": 0.232000, + "Basic_A4": 0.464000, + "Dadsv5_Type1": 8.747000, + "Dasv4_Type1": 6.970000, + "Dasv4_Type2": 7.260000, + "Dasv5_Type1": 7.330000, + "Ddsv4_Type 1": 5.456000, + "Ddsv4_Type2": 6.479000, + "Ddsv5_Type1": 8.184000, + "Dsv3_Type3": 5.796000, + "Dsv3_Type4": 7.245000, + "Dsv4_Type1": 5.808000, + "Dsv4_Type2": 6.970000, + "Dsv5_Type1": 7.260000, + "Eadsv5_Type1": 9.161000, + "Easv4_Type1": 8.818000, + "Easv4_Type2": 8.818000, + "Easv5_Type1": 7.946000, + "Ebdsv5-Type1": 7.814000, + "Ebsv5-Type1": 7.005000, + "Edsv4_Type 1": 6.160000, + "Edsv4_Type2": 9.321000, + "Edsv5_Type1": 9.240000, + "Esv3_Type3": 5.222000, + "Esv3_Type4": 7.075000, + "Esv4_Type1": 5.878000, + "Esv4_Type2": 7.715000, + "Esv5_Type1": 8.818000, + "Fsv2 Type3": 4.990000, + "Fsv2_Type2": 4.277000, + "Fsv2_Type4": 5.702000, + "Lasv3_Type1": 9.064000, + "Lsv3_Type1": 10.120000, + "Mdmsv2MedMem _Type1": 46.974000, + "Mdsv2MedMem_Type1": 23.478000, + "Msmv2MedMem Type1": 46.234000, + "Msmv2_Type1": 169.050000, + "Msv2MedMem Type1": 22.738000, + "Msv2_Type1": 84.534000, + "Standard_A0": 0.020000, + "Standard_A1": 0.060000, + "Standard_A1_v2": 0.050000, + "Standard_A2": 0.120000, + "Standard_A2_v2": 0.106000, + "Standard_A2m_v2": 0.178000, + "Standard_A3": 0.240000, + "Standard_A4": 0.480000, + "Standard_A4_v2": 0.222000, + "Standard_A4m_v2": 0.374000, + "Standard_A5": 0.294000, + "Standard_A6": 0.588000, + "Standard_A7": 1.176000, + "Standard_A8_v2": 0.467000, + "Standard_A8m_v2": 0.786000, + "Standard_B12ms": 0.701000, + "Standard_B16als_v2": 0.745000, + "Standard_B16as_v2": 0.841000, + "Standard_B16ls_v2": 0.901000, + "Standard_B16ms": 0.934000, + "Standard_B16pls_v2": 0.731000, + "Standard_B16ps_v2": 0.816000, + "Standard_B16s_v2": 1.008000, + "Standard_B1ls": 0.007300, + "Standard_B1ms": 0.029200, + "Standard_B1s": 0.014600, + "Standard_B20ms": 1.168000, + "Standard_B2als_v2": 0.061800, + "Standard_B2as_v2": 0.105000, + "Standard_B2ats_v2": 0.022300, + "Standard_B2ls_v2": 0.058400, + "Standard_B2ms": 0.117000, + "Standard_B2pls_v2": 0.046400, + "Standard_B2ps_v2": 0.092800, + "Standard_B2pts_v2": 0.020800, + "Standard_B2s": 0.058400, + "Standard_B2s_v2": 0.117000, + "Standard_B2ts_v2": 0.014600, + "Standard_B32als_v2": 1.489000, + "Standard_B32as_v2": 1.829000, + "Standard_B32ls_v2": 1.655000, + "Standard_B32s_v2": 2.016000, + "Standard_B4als_v2": 0.205000, + "Standard_B4as_v2": 0.229000, + "Standard_B4ls_v2": 0.207000, + "Standard_B4ms": 0.234000, + "Standard_B4pls_v2": 0.164000, + "Standard_B4ps_v2": 0.204000, + "Standard_B4s_v2": 0.252000, + "Standard_B8als_v2": 0.372000, + "Standard_B8as_v2": 0.457000, + "Standard_B8ls_v2": 0.451000, + "Standard_B8ms": 0.467000, + "Standard_B8pls_v2": 0.329000, + "Standard_B8ps_v2": 0.371000, + "Standard_B8s_v2": 0.504000, + "Standard_D1": 0.113000, + "Standard_D11": 0.242000, + "Standard_D11_v2": 0.229000, + "Standard_D11_v2_Promo": 0.229000, + "Standard_D12": 0.483000, + "Standard_D12_v2": 0.459000, + "Standard_D12_v2_Promo": 0.459000, + "Standard_D13": 0.966000, + "Standard_D13_v2": 0.918000, + "Standard_D13_v2_Promo": 0.918000, + "Standard_D14": 1.932000, + "Standard_D14_v2": 1.835000, + "Standard_D14_v2_Promo": 1.835000, + "Standard_D15_v2": 2.294000, + "Standard_D15i_v2": 2.294000, + "Standard_D16_v3": 1.056000, + "Standard_D16_v4": 1.056000, + "Standard_D16_v5": 1.056000, + "Standard_D16a_v4": 1.056000, + "Standard_D16ads_v5": 1.136000, + "Standard_D16as_v4": 1.056000, + "Standard_D16as_v5": 1.688000, + "Standard_D16as_v5_Promo": 0.848000, + "Standard_D16d_v4": 1.240000, + "Standard_D16d_v5": 1.240000, + "Standard_D16ds_v4": 1.240000, + "Standard_D16ds_v5": 1.240000, + "Standard_D16lds_v5": 0.984000, + "Standard_D16ls_v5": 0.864000, + "Standard_D16pds_v5": 0.996000, + "Standard_D16plds_v5": 0.784000, + "Standard_D16pls_v5": 0.688000, + "Standard_D16ps_v5": 0.848000, + "Standard_D16s_v3": 1.056000, + "Standard_D16s_v4": 1.056000, + "Standard_D16s_v5": 1.056000, + "Standard_D1_v2": 0.107000, + "Standard_D2": 0.225000, + "Standard_D2_v2": 0.214000, + "Standard_D2_v2_Promo": 0.214000, + "Standard_D2_v3": 0.132000, + "Standard_D2_v4": 0.132000, + "Standard_D2_v5": 0.132000, + "Standard_D2a_v4": 0.132000, + "Standard_D2ads_v5": 0.234000, + "Standard_D2as_v4": 0.132000, + "Standard_D2as_v5": 0.211000, + "Standard_D2as_v5_Promo": 0.106000, + "Standard_D2d_v4": 0.155000, + "Standard_D2d_v5": 0.155000, + "Standard_D2ds_v4": 0.155000, + "Standard_D2ds_v5": 0.155000, + "Standard_D2lds_v5": 0.123000, + "Standard_D2ls_v5": 0.108000, + "Standard_D2pds_v5": 0.125000, + "Standard_D2plds_v5": 0.098000, + "Standard_D2pls_v5": 0.086000, + "Standard_D2ps_v5": 0.106000, + "Standard_D2s_v3": 0.132000, + "Standard_D2s_v4": 0.132000, + "Standard_D2s_v5": 0.132000, + "Standard_D3": 0.451000, + "Standard_D32-16s_v3": 2.112000, + "Standard_D32-8s_v3": 2.112000, + "Standard_D32_v3": 2.112000, + "Standard_D32_v4": 2.112000, + "Standard_D32_v5": 2.112000, + "Standard_D32a_v4": 2.112000, + "Standard_D32ads_v5": 3.744000, + "Standard_D32as_v4": 2.112000, + "Standard_D32as_v5": 3.376000, + "Standard_D32as_v5_Promo": 1.696000, + "Standard_D32d_v4": 2.480000, + "Standard_D32d_v5": 2.480000, + "Standard_D32ds_v4": 2.480000, + "Standard_D32ds_v5": 2.480000, + "Standard_D32lds_v5": 1.968000, + "Standard_D32ls_v5": 1.728000, + "Standard_D32pds_v5": 1.992000, + "Standard_D32plds_v5": 1.568000, + "Standard_D32pls_v5": 1.376000, + "Standard_D32ps_v5": 1.696000, + "Standard_D32s_v3": 2.112000, + "Standard_D32s_v4": 2.112000, + "Standard_D32s_v5": 2.112000, + "Standard_D3_v2": 0.428000, + "Standard_D3_v2_Promo": 0.428000, + "Standard_D4": 0.902000, + "Standard_D48_v3": 3.168000, + "Standard_D48_v4": 3.168000, + "Standard_D48_v5": 3.168000, + "Standard_D48a_v4": 3.168000, + "Standard_D48ads_v5": 3.408000, + "Standard_D48as_v4": 3.168000, + "Standard_D48as_v5": 2.856000, + "Standard_D48as_v5_Promo": 2.544000, + "Standard_D48d_v4": 3.720000, + "Standard_D48d_v5": 3.720000, + "Standard_D48ds_v4": 3.720000, + "Standard_D48ds_v5": 3.720000, + "Standard_D48lds_v5": 2.952000, + "Standard_D48ls_v5": 2.592000, + "Standard_D48pds_v5": 2.988000, + "Standard_D48plds_v5": 2.352000, + "Standard_D48pls_v5": 2.064000, + "Standard_D48ps_v5": 2.544000, + "Standard_D48s_v3": 3.168000, + "Standard_D48s_v4": 3.168000, + "Standard_D48s_v5": 3.168000, + "Standard_D4_v2": 0.857000, + "Standard_D4_v2_Promo": 0.857000, + "Standard_D4_v3": 0.264000, + "Standard_D4_v4": 0.264000, + "Standard_D4_v5": 0.264000, + "Standard_D4a_v4": 0.264000, + "Standard_D4ads_v5": 0.468000, + "Standard_D4as_v4": 0.264000, + "Standard_D4as_v5": 0.238000, + "Standard_D4as_v5_Promo": 0.212000, + "Standard_D4d_v4": 0.310000, + "Standard_D4d_v5": 0.310000, + "Standard_D4ds_v4": 0.310000, + "Standard_D4ds_v5": 0.310000, + "Standard_D4lds_v5": 0.246000, + "Standard_D4ls_v5": 0.216000, + "Standard_D4pds_v5": 0.249000, + "Standard_D4plds_v5": 0.196000, + "Standard_D4pls_v5": 0.172000, + "Standard_D4ps_v5": 0.212000, + "Standard_D4s_v3": 0.264000, + "Standard_D4s_v4": 0.264000, + "Standard_D4s_v5": 0.264000, + "Standard_D5_v2": 1.714000, + "Standard_D5_v2_Promo": 1.714000, + "Standard_D64-16s_v3": 4.224000, + "Standard_D64-32s_v3": 4.224000, + "Standard_D64_v3": 4.224000, + "Standard_D64_v4": 4.224000, + "Standard_D64_v5": 4.224000, + "Standard_D64a_v4": 4.224000, + "Standard_D64ads_v5": 4.544000, + "Standard_D64as_v4": 4.224000, + "Standard_D64as_v5": 3.808000, + "Standard_D64as_v5_Promo": 3.392000, + "Standard_D64d_v4": 4.960000, + "Standard_D64d_v5": 4.960000, + "Standard_D64ds_v4": 4.960000, + "Standard_D64ds_v5": 4.960000, + "Standard_D64lds_v5": 3.936000, + "Standard_D64ls_v5": 3.456000, + "Standard_D64pds_v5": 3.984000, + "Standard_D64plds_v5": 3.136000, + "Standard_D64pls_v5": 2.752000, + "Standard_D64ps_v5": 3.392000, + "Standard_D64s_v3": 4.224000, + "Standard_D64s_v4": 4.224000, + "Standard_D64s_v5": 4.224000, + "Standard_D8_v3": 0.528000, + "Standard_D8_v4": 0.528000, + "Standard_D8_v5": 0.528000, + "Standard_D8a_v4": 0.528000, + "Standard_D8ads_v5": 0.936000, + "Standard_D8as_v4": 0.528000, + "Standard_D8as_v5": 0.844000, + "Standard_D8as_v5_Promo": 0.424000, + "Standard_D8d_v4": 0.620000, + "Standard_D8d_v5": 0.620000, + "Standard_D8ds_v4": 0.620000, + "Standard_D8ds_v5": 0.620000, + "Standard_D8lds_v5": 0.492000, + "Standard_D8ls_v5": 0.432000, + "Standard_D8pds_v5": 0.498000, + "Standard_D8plds_v5": 0.392000, + "Standard_D8pls_v5": 0.344000, + "Standard_D8ps_v5": 0.424000, + "Standard_D8s_v3": 0.528000, + "Standard_D8s_v4": 0.528000, + "Standard_D8s_v5": 0.528000, + "Standard_D96_v5": 6.336000, + "Standard_D96a_v4": 6.336000, + "Standard_D96ads_v5": 6.816000, + "Standard_D96as_v4": 6.336000, + "Standard_D96as_v5": 5.712000, + "Standard_D96as_v5_Promo": 5.088000, + "Standard_D96d_v5": 7.440000, + "Standard_D96ds_v5": 7.440000, + "Standard_D96lds_v5": 5.904000, + "Standard_D96ls_v5": 5.184000, + "Standard_D96s_v5": 6.336000, + "Standard_DC16ads_v5": 1.136000, + "Standard_DC16as_v5": 0.952000, + "Standard_DC2ads_v5": 0.142000, + "Standard_DC2as_v5": 0.119000, + "Standard_DC32ads_v5": 2.272000, + "Standard_DC32as_v5": 1.904000, + "Standard_DC48ads_v5": 3.408000, + "Standard_DC48as_v5": 2.856000, + "Standard_DC4ads_v5": 0.284000, + "Standard_DC4as_v5": 0.238000, + "Standard_DC64ads_v5": 4.544000, + "Standard_DC64as_v5": 3.808000, + "Standard_DC8ads_v5": 0.568000, + "Standard_DC8as_v5": 0.476000, + "Standard_DC96ads_v5": 6.816000, + "Standard_DC96as_v5": 5.712000, + "Standard_DS1": 0.113000, + "Standard_DS11": 0.242000, + "Standard_DS11-1_v2": 0.229000, + "Standard_DS11_v2": 0.229000, + "Standard_DS11_v2_Promo": 0.229000, + "Standard_DS12": 0.483000, + "Standard_DS12-1_v2": 0.459000, + "Standard_DS12-2_v2": 0.459000, + "Standard_DS12_v2": 0.459000, + "Standard_DS12_v2_Promo": 0.459000, + "Standard_DS13": 0.966000, + "Standard_DS13-2_v2": 0.918000, + "Standard_DS13-4_v2": 0.918000, + "Standard_DS13_v2": 0.918000, + "Standard_DS13_v2_Promo": 0.918000, + "Standard_DS14": 1.932000, + "Standard_DS14-4_v2": 1.835000, + "Standard_DS14-8_v2": 1.835000, + "Standard_DS14_v2": 1.835000, + "Standard_DS14_v2_Promo": 1.835000, + "Standard_DS15_v2": 2.294000, + "Standard_DS15i_v2": 2.294000, + "Standard_DS1_v2": 0.107000, + "Standard_DS2": 0.225000, + "Standard_DS2_v2": 0.214000, + "Standard_DS2_v2_Promo": 0.214000, + "Standard_DS3": 0.451000, + "Standard_DS3_v2": 0.428000, + "Standard_DS3_v2_Promo": 0.428000, + "Standard_DS4": 0.902000, + "Standard_DS4_v2": 0.857000, + "Standard_DS4_v2_Promo": 0.857000, + "Standard_DS5_v2": 1.714000, + "Standard_DS5_v2_Promo": 1.714000, + "Standard_E104i_v5": 9.552000, + "Standard_E104id_v5": 10.010000, + "Standard_E104ids_v5": 10.010000, + "Standard_E104is_v5": 9.552000, + "Standard_E112iads_v5": 10.688000, + "Standard_E112ias_v5": 9.271000, + "Standard_E112ibds_v5": 13.675000, + "Standard_E112ibs_v5": 12.258000, + "Standard_E16-4ads_v5": 1.388000, + "Standard_E16-4as_v4": 1.336000, + "Standard_E16-4as_v5": 1.940000, + "Standard_E16-4ds_v4": 1.400000, + "Standard_E16-4ds_v5": 1.400000, + "Standard_E16-4s_v3": 1.336000, + "Standard_E16-4s_v4": 1.336000, + "Standard_E16-4s_v5": 1.336000, + "Standard_E16-8ads_v5": 1.388000, + "Standard_E16-8as_v4": 1.336000, + "Standard_E16-8as_v5": 1.940000, + "Standard_E16-8ds_v4": 1.400000, + "Standard_E16-8ds_v5": 1.400000, + "Standard_E16-8s_v3": 1.336000, + "Standard_E16-8s_v4": 1.336000, + "Standard_E16-8s_v5": 1.336000, + "Standard_E16_v3": 1.336000, + "Standard_E16_v4": 1.336000, + "Standard_E16_v5": 1.336000, + "Standard_E16a_v4": 1.336000, + "Standard_E16ads_v5": 1.388000, + "Standard_E16as_v4": 1.336000, + "Standard_E16as_v5": 1.204000, + "Standard_E16bds_v5": 1.776000, + "Standard_E16bs_v5": 1.592000, + "Standard_E16d_v4": 1.400000, + "Standard_E16d_v5": 1.400000, + "Standard_E16ds_v4": 1.400000, + "Standard_E16ds_v5": 1.400000, + "Standard_E16pds_v5": 1.124000, + "Standard_E16ps_v5": 1.072000, + "Standard_E16s_v3": 1.336000, + "Standard_E16s_v4": 1.336000, + "Standard_E16s_v5": 1.336000, + "Standard_E20_v3": 1.670000, + "Standard_E20_v4": 1.670000, + "Standard_E20_v5": 1.670000, + "Standard_E20a_v4": 1.670000, + "Standard_E20ads_v5": 1.781000, + "Standard_E20as_v4": 1.670000, + "Standard_E20as_v5": 1.505000, + "Standard_E20d_v4": 1.750000, + "Standard_E20d_v5": 1.750000, + "Standard_E20ds_v4": 1.750000, + "Standard_E20ds_v5": 1.750000, + "Standard_E20pds_v5": 1.405000, + "Standard_E20ps_v5": 1.340000, + "Standard_E20s_v3": 1.670000, + "Standard_E20s_v4": 1.670000, + "Standard_E20s_v5": 1.670000, + "Standard_E2_v3": 0.167000, + "Standard_E2_v4": 0.167000, + "Standard_E2_v5": 0.167000, + "Standard_E2a_v4": 0.167000, + "Standard_E2ads_v5": 0.220000, + "Standard_E2as_v4": 0.167000, + "Standard_E2as_v5": 0.150000, + "Standard_E2bds_v5": 0.222000, + "Standard_E2bs_v5": 0.199000, + "Standard_E2d_v4": 0.175000, + "Standard_E2d_v5": 0.175000, + "Standard_E2ds_v4": 0.175000, + "Standard_E2ds_v5": 0.175000, + "Standard_E2pds_v5": 0.141000, + "Standard_E2ps_v5": 0.134000, + "Standard_E2s_v3": 0.167000, + "Standard_E2s_v4": 0.167000, + "Standard_E2s_v5": 0.167000, + "Standard_E32-16ads_v5": 4.248000, + "Standard_E32-16as_v4": 2.672000, + "Standard_E32-16as_v5": 2.408000, + "Standard_E32-16ds_v4": 2.800000, + "Standard_E32-16ds_v5": 2.800000, + "Standard_E32-16s_v3": 2.672000, + "Standard_E32-16s_v4": 2.672000, + "Standard_E32-16s_v5": 2.672000, + "Standard_E32-8ads_v5": 4.248000, + "Standard_E32-8as_v4": 2.672000, + "Standard_E32-8as_v5": 3.880000, + "Standard_E32-8ds_v4": 2.800000, + "Standard_E32-8ds_v5": 2.800000, + "Standard_E32-8s_v3": 2.672000, + "Standard_E32-8s_v4": 2.672000, + "Standard_E32-8s_v5": 2.672000, + "Standard_E32_v3": 2.672000, + "Standard_E32_v4": 2.672000, + "Standard_E32_v5": 2.672000, + "Standard_E32a_v4": 2.672000, + "Standard_E32ads_v5": 2.822000, + "Standard_E32as_v4": 2.672000, + "Standard_E32as_v5": 2.454000, + "Standard_E32bds_v5": 3.552000, + "Standard_E32bs_v5": 3.184000, + "Standard_E32d_v4": 2.800000, + "Standard_E32d_v5": 2.800000, + "Standard_E32ds_v4": 2.800000, + "Standard_E32ds_v5": 2.800000, + "Standard_E32pds_v5": 2.248000, + "Standard_E32ps_v5": 2.144000, + "Standard_E32s_v3": 2.672000, + "Standard_E32s_v4": 2.672000, + "Standard_E32s_v5": 2.672000, + "Standard_E4-2ads_v5": 0.347000, + "Standard_E4-2as_v4": 0.334000, + "Standard_E4-2as_v5": 0.301000, + "Standard_E4-2ds_v4": 0.350000, + "Standard_E4-2ds_v5": 0.350000, + "Standard_E4-2s_v3": 0.334000, + "Standard_E4-2s_v4": 0.334000, + "Standard_E4-2s_v5": 0.334000, + "Standard_E48_v3": 4.008000, + "Standard_E48_v4": 4.008000, + "Standard_E48_v5": 4.008000, + "Standard_E48a_v4": 4.008000, + "Standard_E48ads_v5": 4.164000, + "Standard_E48as_v4": 4.008000, + "Standard_E48as_v5": 3.658000, + "Standard_E48bds_v5": 5.328000, + "Standard_E48bs_v5": 4.776000, + "Standard_E48d_v4": 4.200000, + "Standard_E48d_v5": 4.200000, + "Standard_E48ds_v4": 4.200000, + "Standard_E48ds_v5": 4.200000, + "Standard_E48s_v3": 4.008000, + "Standard_E48s_v4": 4.008000, + "Standard_E48s_v5": 4.008000, + "Standard_E4_v3": 0.334000, + "Standard_E4_v4": 0.334000, + "Standard_E4_v5": 0.334000, + "Standard_E4a_v4": 0.334000, + "Standard_E4ads_v5": 0.347000, + "Standard_E4as_v4": 0.334000, + "Standard_E4as_v5": 0.301000, + "Standard_E4bds_v5": 0.444000, + "Standard_E4bs_v5": 0.398000, + "Standard_E4d_v4": 0.350000, + "Standard_E4d_v5": 0.350000, + "Standard_E4ds_v4": 0.350000, + "Standard_E4ds_v5": 0.350000, + "Standard_E4pds_v5": 0.281000, + "Standard_E4ps_v5": 0.268000, + "Standard_E4s_v3": 0.334000, + "Standard_E4s_v4": 0.334000, + "Standard_E4s_v5": 0.334000, + "Standard_E64-16ads_v5": 8.496000, + "Standard_E64-16as_v4": 5.344000, + "Standard_E64-16as_v5": 4.816000, + "Standard_E64-16ds_v4": 5.600000, + "Standard_E64-16ds_v5": 5.600000, + "Standard_E64-16s_v3": 4.810000, + "Standard_E64-16s_v4": 5.344000, + "Standard_E64-16s_v5": 5.344000, + "Standard_E64-32ads_v5": 8.496000, + "Standard_E64-32as_v4": 5.344000, + "Standard_E64-32as_v5": 7.760000, + "Standard_E64-32ds_v4": 5.600000, + "Standard_E64-32ds_v5": 5.600000, + "Standard_E64-32s_v3": 4.810000, + "Standard_E64-32s_v4": 5.344000, + "Standard_E64-32s_v5": 5.344000, + "Standard_E64_v3": 4.810000, + "Standard_E64_v4": 5.344000, + "Standard_E64_v5": 5.344000, + "Standard_E64a_v4": 5.344000, + "Standard_E64ads_v5": 5.552000, + "Standard_E64as_v4": 5.344000, + "Standard_E64as_v5": 4.816000, + "Standard_E64bds_v5": 7.104000, + "Standard_E64bs_v5": 6.368000, + "Standard_E64d_v4": 5.600000, + "Standard_E64d_v5": 5.600000, + "Standard_E64ds_v4": 5.600000, + "Standard_E64ds_v5": 5.600000, + "Standard_E64i_v3": 4.810000, + "Standard_E64is_v3": 4.810000, + "Standard_E64s_v3": 4.810000, + "Standard_E64s_v4": 5.344000, + "Standard_E64s_v5": 5.344000, + "Standard_E8-2ads_v5": 0.694000, + "Standard_E8-2as_v4": 0.668000, + "Standard_E8-2as_v5": 0.970000, + "Standard_E8-2ds_v4": 0.700000, + "Standard_E8-2ds_v5": 0.700000, + "Standard_E8-2s_v3": 0.668000, + "Standard_E8-2s_v4": 0.668000, + "Standard_E8-2s_v5": 0.668000, + "Standard_E8-4ads_v5": 0.694000, + "Standard_E8-4as_v4": 0.668000, + "Standard_E8-4as_v5": 0.970000, + "Standard_E8-4ds_v4": 0.700000, + "Standard_E8-4ds_v5": 0.700000, + "Standard_E8-4s_v3": 0.668000, + "Standard_E8-4s_v4": 0.668000, + "Standard_E8-4s_v5": 0.668000, + "Standard_E80ids_v4": 7.000000, + "Standard_E80is_v4": 6.680000, + "Standard_E8_v3": 0.668000, + "Standard_E8_v4": 0.668000, + "Standard_E8_v5": 0.668000, + "Standard_E8a_v4": 0.668000, + "Standard_E8ads_v5": 0.694000, + "Standard_E8as_v4": 0.668000, + "Standard_E8as_v5": 0.648000, + "Standard_E8bds_v5": 0.888000, + "Standard_E8bs_v5": 0.796000, + "Standard_E8d_v4": 0.700000, + "Standard_E8d_v5": 0.700000, + "Standard_E8ds_v4": 0.700000, + "Standard_E8ds_v5": 0.700000, + "Standard_E8pds_v5": 0.562000, + "Standard_E8ps_v5": 0.536000, + "Standard_E8s_v3": 0.668000, + "Standard_E8s_v4": 0.668000, + "Standard_E8s_v5": 0.668000, + "Standard_E96-24ads_v5": 12.744000, + "Standard_E96-24as_v4": 8.016000, + "Standard_E96-24as_v5": 7.224000, + "Standard_E96-24ds_v5": 8.400000, + "Standard_E96-24s_v5": 8.016000, + "Standard_E96-48ads_v5": 8.328000, + "Standard_E96-48as_v4": 8.016000, + "Standard_E96-48as_v5": 11.640000, + "Standard_E96-48ds_v5": 8.400000, + "Standard_E96-48s_v5": 8.016000, + "Standard_E96_v5": 8.016000, + "Standard_E96a_v4": 8.016000, + "Standard_E96ads_v5": 8.374000, + "Standard_E96as_v4": 8.016000, + "Standard_E96as_v5": 7.270000, + "Standard_E96bds_v5": 10.656000, + "Standard_E96bs_v5": 9.552000, + "Standard_E96d_v5": 8.400000, + "Standard_E96ds_v5": 8.400000, + "Standard_E96iads_v5": 9.161000, + "Standard_E96ias_v5": 7.946000, + "Standard_E96s_v5": 8.016000, + "Standard_EC16ads_v5": 1.388000, + "Standard_EC16as_v5": 1.204000, + "Standard_EC20ads_v5": 1.735000, + "Standard_EC20as_v5": 1.505000, + "Standard_EC2ads_v5": 0.174000, + "Standard_EC2as_v5": 0.151000, + "Standard_EC32ads_v5": 2.776000, + "Standard_EC32as_v5": 2.408000, + "Standard_EC48ads_v5": 4.164000, + "Standard_EC48as_v5": 3.612000, + "Standard_EC4ads_v5": 0.347000, + "Standard_EC4as_v5": 0.301000, + "Standard_EC64ads_v5": 5.552000, + "Standard_EC64as_v5": 4.816000, + "Standard_EC8ads_v5": 0.694000, + "Standard_EC8as_v5": 0.602000, + "Standard_EC96ads_v5": 8.328000, + "Standard_EC96as_v5": 7.224000, + "Standard_EC96iads_v5": 9.161000, + "Standard_EC96ias_v5": 7.946000, + "Standard_F1": 0.072000, + "Standard_F16": 1.155000, + "Standard_F16s": 1.155000, + "Standard_F16s_v2": 0.864000, + "Standard_F1s": 0.072000, + "Standard_F2": 0.144000, + "Standard_F2s": 0.144000, + "Standard_F2s_v2": 0.108000, + "Standard_F32s_v2": 1.728000, + "Standard_F4": 0.289000, + "Standard_F48s_v2": 2.592000, + "Standard_F4s": 0.289000, + "Standard_F4s_v2": 0.216000, + "Standard_F64s_v2": 3.456000, + "Standard_F72s_v2": 3.888000, + "Standard_F8": 0.578000, + "Standard_F8s": 0.578000, + "Standard_F8s_v2": 0.432000, + "Standard_L16as_v3": 1.648000, + "Standard_L16s_v3": 1.840000, + "Standard_L32as_v3": 3.296000, + "Standard_L32s_v3": 3.680000, + "Standard_L48as_v3": 4.944000, + "Standard_L48s_v3": 5.520000, + "Standard_L64as_v3": 6.592000, + "Standard_L64s_v3": 7.360000, + "Standard_L80as_v3": 8.240000, + "Standard_L80s_v3": 9.200000, + "Standard_L8as_v3": 0.824000, + "Standard_L8s_v3": 0.920000, + "Standard_M128": 24.177500, + "Standard_M128-32ms": 48.376250, + "Standard_M128-64ms": 48.376250, + "Standard_M128dms_v2": 42.704000, + "Standard_M128ds_v2": 21.344000, + "Standard_M128m": 48.376250, + "Standard_M128ms": 48.376250, + "Standard_M128ms_v2": 42.031000, + "Standard_M128s": 24.177500, + "Standard_M128s_v2": 20.671000, + "Standard_M16-4ms": 5.569875, + "Standard_M16-8ms": 5.569875, + "Standard_M16ms": 5.569875, + "Standard_M16s": 4.326500, + "Standard_M192idms_v2": 51.302000, + "Standard_M192ids_v2": 25.651000, + "Standard_M192ims_v2": 50.629000, + "Standard_M192is_v2": 24.978000, + "Standard_M208-104ms_v2": 69.161000, + "Standard_M208-104s_v2": 34.580000, + "Standard_M208-52ms_v2": 69.161000, + "Standard_M208-52s_v2": 34.580000, + "Standard_M208ms_v2": 69.161000, + "Standard_M208s_v2": 34.580000, + "Standard_M32-16ms": 11.139625, + "Standard_M32-8ms": 11.139625, + "Standard_M32dms_v2": 9.834000, + "Standard_M32ls": 3.806725, + "Standard_M32ms": 11.139625, + "Standard_M32ms_v2": 9.665000, + "Standard_M32s": 6.044750, + "Standard_M32ts": 3.586775, + "Standard_M416-104ms_v2": 153.680000, + "Standard_M416-104s_v2": 76.849000, + "Standard_M416-208ms_v2": 153.680000, + "Standard_M416-208s_v2": 76.849000, + "Standard_M416is_v2": 76.849000, + "Standard_M416ms_v2": 153.680000, + "Standard_M416s_8_v2": 102.470000, + "Standard_M416s_v2": 76.849000, + "Standard_M64": 12.088750, + "Standard_M64-16ms": 18.737500, + "Standard_M64-32ms": 18.737500, + "Standard_M64dms_v2": 16.544000, + "Standard_M64ds_v2": 10.670000, + "Standard_M64ls": 7.174875, + "Standard_M64m": 18.737500, + "Standard_M64ms": 18.737500, + "Standard_M64ms_v2": 16.207000, + "Standard_M64s": 12.088750, + "Standard_M64s_v2": 10.334000, + "Standard_M8-2ms": 2.784875, + "Standard_M8-4ms": 2.784875, + "Standard_M8ms": 2.784875, + "Standard_NC12s_v3": 9.486000, + "Standard_NC24rs_v3": 20.869000, + "Standard_NC24s_v3": 18.972000, + "Standard_NC6s_v3": 4.743000, + "Standard_ND40s_v2": 29.205000, + "Standard_NV12ads_A10_v5": 1.407000, + "Standard_NV18ads_A10_v5": 2.480000, + "Standard_NV36adms_A10_v5": 7.006000, + "Standard_NV36ads_A10_v5": 4.960000, + "Standard_NV6ads_A10_v5": 0.704000, + "Standard_NV72ads_A10_v5": 10.106000, + } + // eastus + initialOnDemandPrices["eastus"] = map[string]float64{ + "Basic_A0": 0.018000, + "Basic_A1": 0.023000, + "Basic_A2": 0.079000, + "Basic_A3": 0.176000, + "Basic_A4": 0.352000, + "DCadsv5 Type 1": 6.345000, + "DCasv5 Type 1": 5.298000, + "DCdsv3 Type1": 5.966000, + "DCsv2 Type 1": 0.845000, + "DCsv3 Type1": 5.069000, + "Dadsv5_Type1": 6.345000, + "Dasv4_Type1": 5.069000, + "Dasv4_Type2": 5.280000, + "Dasv5_Type1": 5.298000, + "Ddsv4_Type 1": 3.978000, + "Ddsv4_Type2": 4.723000, + "Ddsv5_Type1": 5.966000, + "Dsv3_Type3": 4.225000, + "Dsv3_Type4": 5.281000, + "Dsv4_Type1": 4.224000, + "Dsv4_Type2": 5.069000, + "Dsv5_Type1": 5.280000, + "ECadsv5 Type 1": 7.608000, + "ECasv5 Type 1": 6.563000, + "Eadsv5_Type1": 6.917000, + "Easv4_Type1": 6.653000, + "Easv4_Type2": 6.653000, + "Easv5_Type1": 5.966000, + "Ebdsv5-Type1": 5.878000, + "Ebsv5-Type1": 5.245000, + "Edsv4_Type 1": 5.069000, + "Edsv4_Type2": 6.019000, + "Edsv5_Type1": 7.603000, + "Esv3_Type3": 4.297000, + "Esv3_Type4": 5.822000, + "Esv4_Type1": 4.435000, + "Esv4_Type2": 5.821000, + "Esv5_Type1": 6.653000, + "FXmds Type1": 4.910000, + "Fsv2 Type3": 3.927000, + "Fsv2_Type2": 3.366000, + "Fsv2_Type4": 4.468000, + "Lasv3_Type1": 6.864000, + "Lsv2_Type1": 6.864000, + "Lsv3_Type1": 7.656000, + "Mdmsv2MedMem _Type1": 29.359000, + "Mdsv2MedMem_Type1": 14.674000, + "Ms_Type1": 14.669000, + "Msm_Type1": 29.363000, + "Msmv2MedMem Type1": 28.896000, + "Msmv2_Type1": 109.060000, + "Msv2MedMem Type1": 14.211000, + "Msv2_Type1": 54.538000, + "NDamsrA100v4_Type1": 36.047000, + "NDasrA100v4_Type1": 29.917000, + "NVasv4_Type1": 7.176000, + "NVsv3_Type1": 5.016000, + "Standard_A0": 0.020000, + "Standard_A1": 0.060000, + "Standard_A1_v2": 0.043000, + "Standard_A2": 0.120000, + "Standard_A2_v2": 0.091000, + "Standard_A2m_v2": 0.119000, + "Standard_A3": 0.240000, + "Standard_A4": 0.480000, + "Standard_A4_v2": 0.191000, + "Standard_A4m_v2": 0.238000, + "Standard_A5": 0.250000, + "Standard_A6": 0.500000, + "Standard_A7": 1.000000, + "Standard_A8_v2": 0.400000, + "Standard_A8m_v2": 0.475000, + "Standard_B12ms": 0.499000, + "Standard_B16als_v2": 0.533000, + "Standard_B16as_v2": 0.675000, + "Standard_B16ls_v2": 0.663000, + "Standard_B16ms": 0.666000, + "Standard_B16pls_v2": 0.550000, + "Standard_B16ps_v2": 0.611000, + "Standard_B16s_v2": 0.739000, + "Standard_B1ls": 0.005200, + "Standard_B1ms": 0.020700, + "Standard_B1s": 0.010400, + "Standard_B20ms": 0.832000, + "Standard_B2als_v2": 0.046800, + "Standard_B2as_v2": 0.084400, + "Standard_B2ats_v2": 0.009400, + "Standard_B2ls_v2": 0.041600, + "Standard_B2ms": 0.083200, + "Standard_B2pls_v2": 0.042800, + "Standard_B2ps_v2": 0.067200, + "Standard_B2pts_v2": 0.008400, + "Standard_B2s": 0.041600, + "Standard_B2s_v2": 0.092400, + "Standard_B2ts_v2": 0.019600, + "Standard_B32als_v2": 1.213000, + "Standard_B32as_v2": 1.203000, + "Standard_B32ls_v2": 1.326000, + "Standard_B32s_v2": 1.478000, + "Standard_B4als_v2": 0.133000, + "Standard_B4as_v2": 0.169000, + "Standard_B4ls_v2": 0.166000, + "Standard_B4ms": 0.166000, + "Standard_B4pls_v2": 0.119000, + "Standard_B4ps_v2": 0.153000, + "Standard_B4s_v2": 0.166000, + "Standard_B8als_v2": 0.303000, + "Standard_B8as_v2": 0.338000, + "Standard_B8ls_v2": 0.332000, + "Standard_B8ms": 0.333000, + "Standard_B8pls_v2": 0.238000, + "Standard_B8ps_v2": 0.269000, + "Standard_B8s_v2": 0.333000, + "Standard_D1": 0.077000, + "Standard_D11": 0.193000, + "Standard_D11_v2": 0.185000, + "Standard_D11_v2_Promo": 0.185000, + "Standard_D12": 0.386000, + "Standard_D12_v2": 0.371000, + "Standard_D12_v2_Promo": 0.371000, + "Standard_D13": 0.771000, + "Standard_D13_v2": 0.741000, + "Standard_D13_v2_Promo": 0.741000, + "Standard_D14": 1.542000, + "Standard_D14_v2": 1.482000, + "Standard_D14_v2_Promo": 1.482000, + "Standard_D15_v2": 1.853000, + "Standard_D15i_v2": 1.853000, + "Standard_D16_v3": 0.768000, + "Standard_D16_v4": 0.768000, + "Standard_D16_v5": 0.768000, + "Standard_D16a_v4": 0.768000, + "Standard_D16ads_v5": 1.560000, + "Standard_D16as_v4": 0.768000, + "Standard_D16as_v5": 1.424000, + "Standard_D16d_v4": 0.904000, + "Standard_D16d_v5": 0.904000, + "Standard_D16ds_v4": 0.904000, + "Standard_D16ds_v5": 0.904000, + "Standard_D16lds_v5": 0.768000, + "Standard_D16ls_v5": 0.680000, + "Standard_D16pds_v5": 0.723000, + "Standard_D16plds_v5": 0.614000, + "Standard_D16pls_v5": 0.544000, + "Standard_D16ps_v5": 0.616000, + "Standard_D16s_v3": 0.768000, + "Standard_D16s_v4": 0.768000, + "Standard_D16s_v5": 0.768000, + "Standard_D1_v2": 0.073000, + "Standard_D2": 0.154000, + "Standard_D2_v2": 0.146000, + "Standard_D2_v2_Promo": 0.146000, + "Standard_D2_v3": 0.096000, + "Standard_D2_v4": 0.096000, + "Standard_D2_v5": 0.096000, + "Standard_D2a_v4": 0.096000, + "Standard_D2ads_v5": 0.103000, + "Standard_D2as_v4": 0.096000, + "Standard_D2as_v5": 0.178000, + "Standard_D2d_v4": 0.113000, + "Standard_D2d_v5": 0.113000, + "Standard_D2ds_v4": 0.113000, + "Standard_D2ds_v5": 0.113000, + "Standard_D2lds_v5": 0.096000, + "Standard_D2ls_v5": 0.085000, + "Standard_D2pds_v5": 0.090400, + "Standard_D2plds_v5": 0.076800, + "Standard_D2pls_v5": 0.068000, + "Standard_D2ps_v5": 0.077000, + "Standard_D2s_v3": 0.096000, + "Standard_D2s_v4": 0.096000, + "Standard_D2s_v5": 0.096000, + "Standard_D3": 0.308000, + "Standard_D32-16s_v3": 1.536000, + "Standard_D32-8s_v3": 1.536000, + "Standard_D32_v3": 1.536000, + "Standard_D32_v4": 1.536000, + "Standard_D32_v5": 1.536000, + "Standard_D32a_v4": 1.536000, + "Standard_D32ads_v5": 1.648000, + "Standard_D32as_v4": 1.536000, + "Standard_D32as_v5": 1.376000, + "Standard_D32d_v4": 1.808000, + "Standard_D32d_v5": 1.808000, + "Standard_D32ds_v4": 1.808000, + "Standard_D32ds_v5": 1.808000, + "Standard_D32lds_v5": 1.536000, + "Standard_D32ls_v5": 1.360000, + "Standard_D32pds_v5": 1.446000, + "Standard_D32plds_v5": 1.229000, + "Standard_D32pls_v5": 1.088000, + "Standard_D32ps_v5": 1.232000, + "Standard_D32s_v3": 1.536000, + "Standard_D32s_v4": 1.536000, + "Standard_D32s_v5": 1.536000, + "Standard_D3_v2": 0.293000, + "Standard_D3_v2_Promo": 0.293000, + "Standard_D4": 0.616000, + "Standard_D48_v3": 2.304000, + "Standard_D48_v4": 2.304000, + "Standard_D48_v5": 2.304000, + "Standard_D48a_v4": 2.304000, + "Standard_D48ads_v5": 2.472000, + "Standard_D48as_v4": 2.304000, + "Standard_D48as_v5": 4.272000, + "Standard_D48d_v4": 2.712000, + "Standard_D48d_v5": 2.712000, + "Standard_D48ds_v4": 2.712000, + "Standard_D48ds_v5": 2.712000, + "Standard_D48lds_v5": 2.304000, + "Standard_D48ls_v5": 2.040000, + "Standard_D48pds_v5": 2.170000, + "Standard_D48plds_v5": 1.843000, + "Standard_D48pls_v5": 1.632000, + "Standard_D48ps_v5": 1.848000, + "Standard_D48s_v3": 2.304000, + "Standard_D48s_v4": 2.304000, + "Standard_D48s_v5": 2.304000, + "Standard_D4_v2": 0.585000, + "Standard_D4_v2_Promo": 0.585000, + "Standard_D4_v3": 0.192000, + "Standard_D4_v4": 0.192000, + "Standard_D4_v5": 0.192000, + "Standard_D4a_v4": 0.192000, + "Standard_D4ads_v5": 0.390000, + "Standard_D4as_v4": 0.192000, + "Standard_D4as_v5": 0.172000, + "Standard_D4d_v4": 0.226000, + "Standard_D4d_v5": 0.226000, + "Standard_D4ds_v4": 0.226000, + "Standard_D4ds_v5": 0.226000, + "Standard_D4lds_v5": 0.192000, + "Standard_D4ls_v5": 0.170000, + "Standard_D4pds_v5": 0.181000, + "Standard_D4plds_v5": 0.154000, + "Standard_D4pls_v5": 0.136000, + "Standard_D4ps_v5": 0.154000, + "Standard_D4s_v3": 0.192000, + "Standard_D4s_v4": 0.192000, + "Standard_D4s_v5": 0.192000, + "Standard_D5_v2": 1.170000, + "Standard_D5_v2_Promo": 1.170000, + "Standard_D64-16s_v3": 3.072000, + "Standard_D64-32s_v3": 3.072000, + "Standard_D64_v3": 3.072000, + "Standard_D64_v4": 3.072000, + "Standard_D64_v5": 3.072000, + "Standard_D64a_v4": 3.072000, + "Standard_D64ads_v5": 3.296000, + "Standard_D64as_v4": 3.072000, + "Standard_D64as_v5": 2.752000, + "Standard_D64d_v4": 3.616000, + "Standard_D64d_v5": 3.616000, + "Standard_D64ds_v4": 3.616000, + "Standard_D64ds_v5": 3.616000, + "Standard_D64lds_v5": 3.072000, + "Standard_D64ls_v5": 2.720000, + "Standard_D64pds_v5": 2.893000, + "Standard_D64plds_v5": 2.458000, + "Standard_D64pls_v5": 2.176000, + "Standard_D64ps_v5": 2.464000, + "Standard_D64s_v3": 3.072000, + "Standard_D64s_v4": 3.072000, + "Standard_D64s_v5": 3.072000, + "Standard_D8_v3": 0.384000, + "Standard_D8_v4": 0.384000, + "Standard_D8_v5": 0.384000, + "Standard_D8a_v4": 0.384000, + "Standard_D8ads_v5": 0.780000, + "Standard_D8as_v4": 0.384000, + "Standard_D8as_v5": 0.712000, + "Standard_D8d_v4": 0.452000, + "Standard_D8d_v5": 0.452000, + "Standard_D8ds_v4": 0.452000, + "Standard_D8ds_v5": 0.452000, + "Standard_D8lds_v5": 0.384000, + "Standard_D8ls_v5": 0.340000, + "Standard_D8pds_v5": 0.362000, + "Standard_D8plds_v5": 0.307000, + "Standard_D8pls_v5": 0.272000, + "Standard_D8ps_v5": 0.308000, + "Standard_D8s_v3": 0.384000, + "Standard_D8s_v4": 0.384000, + "Standard_D8s_v5": 0.384000, + "Standard_D96_v5": 4.608000, + "Standard_D96a_v4": 4.608000, + "Standard_D96ads_v5": 4.944000, + "Standard_D96as_v4": 4.608000, + "Standard_D96as_v5": 4.128000, + "Standard_D96d_v5": 5.424000, + "Standard_D96ds_v5": 5.424000, + "Standard_D96lds_v5": 4.608000, + "Standard_D96ls_v5": 4.080000, + "Standard_D96s_v5": 4.608000, + "Standard_DC16ads_cc_v5": 0.824000, + "Standard_DC16ads_v5": 0.824000, + "Standard_DC16as_cc_v5": 0.688000, + "Standard_DC16as_v5": 0.688000, + "Standard_DC16ds_v3": 1.808000, + "Standard_DC16s_v3": 1.536000, + "Standard_DC1ds_v3": 0.113000, + "Standard_DC1s_v2": 0.096000, + "Standard_DC1s_v3": 0.096000, + "Standard_DC24ds_v3": 2.712000, + "Standard_DC24s_v3": 2.304000, + "Standard_DC2ads_v5": 0.103000, + "Standard_DC2as_v5": 0.086000, + "Standard_DC2ds_v3": 0.226000, + "Standard_DC2s": 0.198000, + "Standard_DC2s_v2": 0.192000, + "Standard_DC2s_v3": 0.192000, + "Standard_DC32ads_cc_v5": 1.648000, + "Standard_DC32ads_v5": 1.648000, + "Standard_DC32as_cc_v5": 1.376000, + "Standard_DC32as_v5": 1.376000, + "Standard_DC32ds_v3": 3.616000, + "Standard_DC32s_v3": 3.072000, + "Standard_DC48ads_cc_v5": 2.472000, + "Standard_DC48ads_v5": 2.472000, + "Standard_DC48as_cc_v5": 2.064000, + "Standard_DC48as_v5": 2.064000, + "Standard_DC48ds_v3": 5.424000, + "Standard_DC48s_v3": 4.608000, + "Standard_DC4ads_cc_v5": 0.206000, + "Standard_DC4ads_v5": 0.206000, + "Standard_DC4as_cc_v5": 0.172000, + "Standard_DC4as_v5": 0.172000, + "Standard_DC4ds_v3": 0.452000, + "Standard_DC4s": 0.395000, + "Standard_DC4s_v2": 0.384000, + "Standard_DC4s_v3": 0.384000, + "Standard_DC64ads_cc_v5": 3.296000, + "Standard_DC64ads_v5": 3.296000, + "Standard_DC64as_cc_v5": 2.752000, + "Standard_DC64as_v5": 2.752000, + "Standard_DC8_v2": 0.768000, + "Standard_DC8ads_cc_v5": 0.412000, + "Standard_DC8ads_v5": 0.412000, + "Standard_DC8as_cc_v5": 0.344000, + "Standard_DC8as_v5": 0.344000, + "Standard_DC8ds_v3": 0.904000, + "Standard_DC8s_v3": 0.768000, + "Standard_DC96ads_cc_v5": 4.944000, + "Standard_DC96ads_v5": 4.944000, + "Standard_DC96as_cc_v5": 4.128000, + "Standard_DC96as_v5": 4.128000, + "Standard_DS1": 0.077000, + "Standard_DS11": 0.193000, + "Standard_DS11-1_v2": 0.185000, + "Standard_DS11_v2": 0.185000, + "Standard_DS11_v2_Promo": 0.185000, + "Standard_DS12": 0.386000, + "Standard_DS12-1_v2": 0.371000, + "Standard_DS12-2_v2": 0.371000, + "Standard_DS12_v2": 0.371000, + "Standard_DS12_v2_Promo": 0.371000, + "Standard_DS13": 0.771000, + "Standard_DS13-2_v2": 0.741000, + "Standard_DS13-4_v2": 0.741000, + "Standard_DS13_v2": 0.741000, + "Standard_DS13_v2_Promo": 0.741000, + "Standard_DS14": 1.542000, + "Standard_DS14-4_v2": 1.482000, + "Standard_DS14-8_v2": 1.482000, + "Standard_DS14_v2": 1.482000, + "Standard_DS14_v2_Promo": 1.482000, + "Standard_DS15_v2": 1.853000, + "Standard_DS15i_v2": 1.853000, + "Standard_DS1_v2": 0.073000, + "Standard_DS2": 0.154000, + "Standard_DS2_v2": 0.146000, + "Standard_DS2_v2_Promo": 0.146000, + "Standard_DS3": 0.308000, + "Standard_DS3_v2": 0.293000, + "Standard_DS3_v2_Promo": 0.293000, + "Standard_DS4": 0.616000, + "Standard_DS4_v2": 0.585000, + "Standard_DS4_v2_Promo": 0.585000, + "Standard_DS5_v2": 1.170000, + "Standard_DS5_v2_Promo": 1.170000, + "Standard_E104i_v5": 7.207000, + "Standard_E104id_v5": 8.237000, + "Standard_E104ids_v5": 8.237000, + "Standard_E104is_v5": 7.207000, + "Standard_E112iads_v5": 8.070000, + "Standard_E112ias_v5": 6.961000, + "Standard_E112ibds_v5": 10.287000, + "Standard_E112ibs_v5": 9.178000, + "Standard_E16-4ads_v5": 1.048000, + "Standard_E16-4as_v4": 1.008000, + "Standard_E16-4as_v5": 0.904000, + "Standard_E16-4ds_v4": 1.152000, + "Standard_E16-4ds_v5": 1.152000, + "Standard_E16-4s_v3": 1.008000, + "Standard_E16-4s_v4": 1.008000, + "Standard_E16-4s_v5": 1.008000, + "Standard_E16-8ads_v5": 1.784000, + "Standard_E16-8as_v4": 1.008000, + "Standard_E16-8as_v5": 0.904000, + "Standard_E16-8ds_v4": 1.152000, + "Standard_E16-8ds_v5": 1.152000, + "Standard_E16-8s_v3": 1.008000, + "Standard_E16-8s_v4": 1.008000, + "Standard_E16-8s_v5": 1.008000, + "Standard_E16_v3": 1.008000, + "Standard_E16_v4": 1.008000, + "Standard_E16_v5": 1.008000, + "Standard_E16a_v4": 1.008000, + "Standard_E16ads_v5": 1.048000, + "Standard_E16as_v4": 1.008000, + "Standard_E16as_v5": 0.904000, + "Standard_E16bds_v5": 1.336000, + "Standard_E16bs_v5": 1.192000, + "Standard_E16d_v4": 1.152000, + "Standard_E16d_v5": 1.152000, + "Standard_E16ds_v4": 1.152000, + "Standard_E16ds_v5": 1.152000, + "Standard_E16pds_v5": 0.922000, + "Standard_E16ps_v5": 0.806000, + "Standard_E16s_v3": 1.008000, + "Standard_E16s_v4": 1.008000, + "Standard_E16s_v5": 1.008000, + "Standard_E20_v3": 1.260000, + "Standard_E20_v4": 1.260000, + "Standard_E20_v5": 1.260000, + "Standard_E20a_v4": 1.260000, + "Standard_E20ads_v5": 1.310000, + "Standard_E20as_v4": 1.260000, + "Standard_E20as_v5": 1.130000, + "Standard_E20d_v4": 1.440000, + "Standard_E20d_v5": 1.440000, + "Standard_E20ds_v4": 1.440000, + "Standard_E20ds_v5": 1.440000, + "Standard_E20pds_v5": 1.152000, + "Standard_E20ps_v5": 1.008000, + "Standard_E20s_v3": 1.260000, + "Standard_E20s_v4": 1.260000, + "Standard_E20s_v5": 1.260000, + "Standard_E2_v3": 0.126000, + "Standard_E2_v4": 0.126000, + "Standard_E2_v5": 0.126000, + "Standard_E2a_v4": 0.126000, + "Standard_E2ads_v5": 0.177000, + "Standard_E2as_v4": 0.126000, + "Standard_E2as_v5": 0.113000, + "Standard_E2bds_v5": 0.167000, + "Standard_E2bs_v5": 0.149000, + "Standard_E2d_v4": 0.144000, + "Standard_E2d_v5": 0.144000, + "Standard_E2ds_v4": 0.144000, + "Standard_E2ds_v5": 0.144000, + "Standard_E2pds_v5": 0.115000, + "Standard_E2ps_v5": 0.101000, + "Standard_E2s_v3": 0.126000, + "Standard_E2s_v4": 0.126000, + "Standard_E2s_v5": 0.126000, + "Standard_E32-16ads_v5": 2.096000, + "Standard_E32-16as_v4": 2.016000, + "Standard_E32-16as_v5": 3.280000, + "Standard_E32-16ds_v4": 2.304000, + "Standard_E32-16ds_v5": 2.304000, + "Standard_E32-16s_v3": 2.016000, + "Standard_E32-16s_v4": 2.016000, + "Standard_E32-16s_v5": 2.016000, + "Standard_E32-8ads_v5": 2.096000, + "Standard_E32-8as_v4": 2.016000, + "Standard_E32-8as_v5": 1.808000, + "Standard_E32-8ds_v4": 2.304000, + "Standard_E32-8ds_v5": 2.304000, + "Standard_E32-8s_v3": 2.016000, + "Standard_E32-8s_v4": 2.016000, + "Standard_E32-8s_v5": 2.016000, + "Standard_E32_v3": 2.016000, + "Standard_E32_v4": 2.016000, + "Standard_E32_v5": 2.016000, + "Standard_E32a_v4": 2.016000, + "Standard_E32ads_v5": 2.096000, + "Standard_E32as_v4": 2.016000, + "Standard_E32as_v5": 1.854000, + "Standard_E32bds_v5": 2.672000, + "Standard_E32bs_v5": 2.384000, + "Standard_E32d_v4": 2.304000, + "Standard_E32d_v5": 2.304000, + "Standard_E32ds_v4": 2.304000, + "Standard_E32ds_v5": 2.304000, + "Standard_E32pds_v5": 1.843000, + "Standard_E32ps_v5": 1.613000, + "Standard_E32s_v3": 2.016000, + "Standard_E32s_v4": 2.016000, + "Standard_E32s_v5": 2.016000, + "Standard_E4-2ads_v5": 0.262000, + "Standard_E4-2as_v4": 0.252000, + "Standard_E4-2as_v5": 0.226000, + "Standard_E4-2ds_v4": 0.288000, + "Standard_E4-2ds_v5": 0.288000, + "Standard_E4-2s_v3": 0.252000, + "Standard_E4-2s_v4": 0.252000, + "Standard_E4-2s_v5": 0.252000, + "Standard_E48_v3": 3.024000, + "Standard_E48_v4": 3.024000, + "Standard_E48_v5": 3.024000, + "Standard_E48a_v4": 3.024000, + "Standard_E48ads_v5": 3.144000, + "Standard_E48as_v4": 3.024000, + "Standard_E48as_v5": 2.712000, + "Standard_E48bds_v5": 4.008000, + "Standard_E48bs_v5": 3.576000, + "Standard_E48d_v4": 3.456000, + "Standard_E48d_v5": 3.456000, + "Standard_E48ds_v4": 3.456000, + "Standard_E48ds_v5": 3.456000, + "Standard_E48s_v3": 3.024000, + "Standard_E48s_v4": 3.024000, + "Standard_E48s_v5": 3.024000, + "Standard_E4_v3": 0.252000, + "Standard_E4_v4": 0.252000, + "Standard_E4_v5": 0.252000, + "Standard_E4a_v4": 0.252000, + "Standard_E4ads_v5": 0.308000, + "Standard_E4as_v4": 0.252000, + "Standard_E4as_v5": 0.226000, + "Standard_E4bds_v5": 0.334000, + "Standard_E4bs_v5": 0.298000, + "Standard_E4d_v4": 0.288000, + "Standard_E4d_v5": 0.288000, + "Standard_E4ds_v4": 0.288000, + "Standard_E4ds_v5": 0.288000, + "Standard_E4pds_v5": 0.230000, + "Standard_E4ps_v5": 0.202000, + "Standard_E4s_v3": 0.252000, + "Standard_E4s_v4": 0.252000, + "Standard_E4s_v5": 0.252000, + "Standard_E64-16ads_v5": 7.136000, + "Standard_E64-16as_v4": 4.032000, + "Standard_E64-16as_v5": 3.616000, + "Standard_E64-16ds_v4": 4.608000, + "Standard_E64-16ds_v5": 4.608000, + "Standard_E64-16s_v3": 3.629000, + "Standard_E64-16s_v4": 4.032000, + "Standard_E64-16s_v5": 4.032000, + "Standard_E64-32ads_v5": 4.192000, + "Standard_E64-32as_v4": 4.032000, + "Standard_E64-32as_v5": 6.560000, + "Standard_E64-32ds_v4": 4.608000, + "Standard_E64-32ds_v5": 4.608000, + "Standard_E64-32s_v3": 3.629000, + "Standard_E64-32s_v4": 4.032000, + "Standard_E64-32s_v5": 4.032000, + "Standard_E64_v3": 3.629000, + "Standard_E64_v4": 4.032000, + "Standard_E64_v5": 4.032000, + "Standard_E64a_v4": 4.032000, + "Standard_E64ads_v5": 4.238000, + "Standard_E64as_v4": 4.032000, + "Standard_E64as_v5": 3.662000, + "Standard_E64bds_v5": 5.344000, + "Standard_E64bs_v5": 4.768000, + "Standard_E64d_v4": 4.608000, + "Standard_E64d_v5": 4.608000, + "Standard_E64ds_v4": 4.608000, + "Standard_E64ds_v5": 4.608000, + "Standard_E64i_v3": 3.629000, + "Standard_E64is_v3": 3.629000, + "Standard_E64s_v3": 3.629000, + "Standard_E64s_v4": 4.032000, + "Standard_E64s_v5": 4.032000, + "Standard_E8-2ads_v5": 0.524000, + "Standard_E8-2as_v4": 0.504000, + "Standard_E8-2as_v5": 0.820000, + "Standard_E8-2ds_v4": 0.576000, + "Standard_E8-2ds_v5": 0.576000, + "Standard_E8-2s_v3": 0.504000, + "Standard_E8-2s_v4": 0.504000, + "Standard_E8-2s_v5": 0.504000, + "Standard_E8-4ads_v5": 0.892000, + "Standard_E8-4as_v4": 0.504000, + "Standard_E8-4as_v5": 0.820000, + "Standard_E8-4ds_v4": 0.576000, + "Standard_E8-4ds_v5": 0.576000, + "Standard_E8-4s_v3": 0.504000, + "Standard_E8-4s_v4": 0.504000, + "Standard_E8-4s_v5": 0.504000, + "Standard_E80ids_v4": 5.760000, + "Standard_E80is_v4": 5.040000, + "Standard_E8_v3": 0.504000, + "Standard_E8_v4": 0.504000, + "Standard_E8_v5": 0.504000, + "Standard_E8a_v4": 0.504000, + "Standard_E8ads_v5": 0.524000, + "Standard_E8as_v4": 0.504000, + "Standard_E8as_v5": 0.498000, + "Standard_E8bds_v5": 0.668000, + "Standard_E8bs_v5": 0.596000, + "Standard_E8d_v4": 0.576000, + "Standard_E8d_v5": 0.576000, + "Standard_E8ds_v4": 0.576000, + "Standard_E8ds_v5": 0.576000, + "Standard_E8pds_v5": 0.461000, + "Standard_E8ps_v5": 0.403000, + "Standard_E8s_v3": 0.504000, + "Standard_E8s_v4": 0.504000, + "Standard_E8s_v5": 0.504000, + "Standard_E96-24ads_v5": 6.288000, + "Standard_E96-24as_v4": 6.048000, + "Standard_E96-24as_v5": 9.840000, + "Standard_E96-24ds_v5": 6.912000, + "Standard_E96-24s_v5": 6.048000, + "Standard_E96-48ads_v5": 6.288000, + "Standard_E96-48as_v4": 6.048000, + "Standard_E96-48as_v5": 5.424000, + "Standard_E96-48ds_v5": 6.912000, + "Standard_E96-48s_v5": 6.048000, + "Standard_E96_v5": 6.048000, + "Standard_E96a_v4": 6.048000, + "Standard_E96ads_v5": 6.334000, + "Standard_E96as_v4": 6.048000, + "Standard_E96as_v5": 5.470000, + "Standard_E96bds_v5": 8.016000, + "Standard_E96bs_v5": 7.152000, + "Standard_E96d_v5": 6.912000, + "Standard_E96ds_v5": 6.912000, + "Standard_E96iads_v5": 6.917000, + "Standard_E96ias_v5": 5.966000, + "Standard_E96s_v5": 6.048000, + "Standard_EC16ads_cc_v5": 1.048000, + "Standard_EC16ads_v5": 1.048000, + "Standard_EC16as_cc_v5": 0.904000, + "Standard_EC16as_v5": 0.904000, + "Standard_EC20ads_cc_v5": 1.310000, + "Standard_EC20ads_v5": 1.310000, + "Standard_EC20as_cc_v5": 1.130000, + "Standard_EC20as_v5": 1.130000, + "Standard_EC2ads_v5": 0.131000, + "Standard_EC2as_v5": 0.113000, + "Standard_EC32ads_cc_v5": 2.096000, + "Standard_EC32ads_v5": 2.096000, + "Standard_EC32as_cc_v5": 1.808000, + "Standard_EC32as_v5": 1.808000, + "Standard_EC48ads_cc_v5": 3.144000, + "Standard_EC48ads_v5": 3.144000, + "Standard_EC48as_cc_v5": 2.712000, + "Standard_EC48as_v5": 2.712000, + "Standard_EC4ads_cc_v5": 0.262000, + "Standard_EC4ads_v5": 0.262000, + "Standard_EC4as_cc_v5": 0.226000, + "Standard_EC4as_v5": 0.226000, + "Standard_EC64ads_cc_v5": 4.192000, + "Standard_EC64ads_v5": 4.192000, + "Standard_EC64as_cc_v5": 3.616000, + "Standard_EC64as_v5": 3.616000, + "Standard_EC8ads_cc_v5": 0.524000, + "Standard_EC8ads_v5": 0.524000, + "Standard_EC8as_cc_v5": 0.452000, + "Standard_EC8as_v5": 0.452000, + "Standard_EC96ads_cc_v5": 6.288000, + "Standard_EC96ads_v5": 6.288000, + "Standard_EC96as_cc_v5": 5.424000, + "Standard_EC96as_v5": 5.424000, + "Standard_EC96iads_v5": 6.917000, + "Standard_EC96ias_v5": 5.966000, + "Standard_F1": 0.049700, + "Standard_F16": 0.796000, + "Standard_F16s": 0.796000, + "Standard_F16s_v2": 0.677000, + "Standard_F1s": 0.049700, + "Standard_F2": 0.099000, + "Standard_F2s": 0.099000, + "Standard_F2s_v2": 0.084600, + "Standard_F32s_v2": 1.353000, + "Standard_F4": 0.199000, + "Standard_F48s_v2": 2.030000, + "Standard_F4s": 0.199000, + "Standard_F4s_v2": 0.169000, + "Standard_F64s_v2": 2.706000, + "Standard_F72s_v2": 3.045000, + "Standard_F8": 0.398000, + "Standard_F8s": 0.398000, + "Standard_F8s_v2": 0.338000, + "Standard_FX12mds": 1.116000, + "Standard_FX24mds": 2.232000, + "Standard_FX36mds": 3.348000, + "Standard_FX48mds": 4.464000, + "Standard_FX4mds": 0.372000, + "Standard_H16": 1.807000, + "Standard_H16m": 2.422000, + "Standard_H16mr": 2.664000, + "Standard_H16r": 1.988000, + "Standard_H8": 0.904000, + "Standard_H8m": 1.211000, + "Standard_HB120-16rs_v2": 3.600000, + "Standard_HB120-16rs_v3": 3.600000, + "Standard_HB120-32rs_v2": 3.600000, + "Standard_HB120-32rs_v3": 3.600000, + "Standard_HB120-64rs_v2": 3.600000, + "Standard_HB120-64rs_v3": 3.600000, + "Standard_HB120-96rs_v2": 3.600000, + "Standard_HB120-96rs_v3": 3.600000, + "Standard_HB120rs_v2": 3.600000, + "Standard_HB120rs_v3": 3.600000, + "Standard_HB176-144rs_v4": 7.200000, + "Standard_HB176-24rs_v4": 7.200000, + "Standard_HB176-48rs_v4": 7.200000, + "Standard_HB176-96rs_v4": 7.200000, + "Standard_HB176rs_v4": 7.200000, + "Standard_HB60-15rs": 2.280000, + "Standard_HB60-30rs": 2.280000, + "Standard_HB60-45rs": 2.280000, + "Standard_HB60rs": 2.280000, + "Standard_HC44-16rs": 3.168000, + "Standard_HC44-32rs": 3.168000, + "Standard_HC44rs": 3.168000, + "Standard_HX176-144rs": 8.640000, + "Standard_HX176-24rs": 8.640000, + "Standard_HX176-48rs": 8.640000, + "Standard_HX176-96rs": 8.640000, + "Standard_HX176rs": 8.640000, + "Standard_L16as_v3": 1.248000, + "Standard_L16s_v2": 1.248000, + "Standard_L16s_v3": 1.392000, + "Standard_L32as_v3": 2.496000, + "Standard_L32s_v2": 2.496000, + "Standard_L32s_v3": 2.784000, + "Standard_L48as_v3": 3.744000, + "Standard_L48s_v2": 3.744000, + "Standard_L48s_v3": 4.176000, + "Standard_L64as_v3": 4.992000, + "Standard_L64s_v2": 4.992000, + "Standard_L64s_v3": 5.568000, + "Standard_L80as_v3": 6.240000, + "Standard_L80s_v2": 6.240000, + "Standard_L80s_v3": 6.960000, + "Standard_L88is_v2": 6.864000, + "Standard_L8as_v3": 0.624000, + "Standard_L8s_v2": 0.624000, + "Standard_L8s_v3": 0.696000, + "Standard_M128": 13.338000, + "Standard_M128-32ms": 26.688000, + "Standard_M128-64ms": 26.688000, + "Standard_M128dms_v2": 26.690000, + "Standard_M128ds_v2": 13.340000, + "Standard_M128m": 26.688000, + "Standard_M128ms": 26.688000, + "Standard_M128ms_v2": 26.269000, + "Standard_M128s": 13.338000, + "Standard_M128s_v2": 12.919000, + "Standard_M16-4ms": 3.073000, + "Standard_M16-8ms": 3.073000, + "Standard_M16ms": 3.073000, + "Standard_M16s": 2.387000, + "Standard_M192idms_v2": 32.064000, + "Standard_M192ids_v2": 16.032000, + "Standard_M192ims_v2": 31.643000, + "Standard_M192is_v2": 15.611000, + "Standard_M208-104ms_v2": 44.620000, + "Standard_M208-104s_v2": 22.310000, + "Standard_M208-52ms_v2": 44.620000, + "Standard_M208-52s_v2": 22.310000, + "Standard_M208ms_v2": 44.620000, + "Standard_M208s_v2": 22.310000, + "Standard_M32-16ms": 6.146000, + "Standard_M32-8ms": 6.146000, + "Standard_M32dms_v2": 6.146000, + "Standard_M32ls": 2.873000, + "Standard_M32ms": 6.146000, + "Standard_M32ms_v2": 6.041000, + "Standard_M32s": 3.335000, + "Standard_M32ts": 2.707000, + "Standard_M416-104ms_v2": 99.150000, + "Standard_M416-104s_v2": 49.580000, + "Standard_M416-208ms_v2": 99.150000, + "Standard_M416-208s_v2": 49.580000, + "Standard_M416is_v2": 49.580000, + "Standard_M416ms_v2": 99.150000, + "Standard_M416s_8_v2": 66.110000, + "Standard_M416s_v2": 49.580000, + "Standard_M64": 6.669000, + "Standard_M64-16ms": 10.337000, + "Standard_M64-32ms": 10.337000, + "Standard_M64dms_v2": 10.340000, + "Standard_M64ds_v2": 6.669000, + "Standard_M64ls": 5.415000, + "Standard_M64m": 10.337000, + "Standard_M64ms": 10.337000, + "Standard_M64ms_v2": 10.130000, + "Standard_M64s": 6.669000, + "Standard_M64s_v2": 6.459000, + "Standard_M8-2ms": 1.536500, + "Standard_M8-4ms": 1.536500, + "Standard_M8ms": 1.536500, + "Standard_NC12": 1.800000, + "Standard_NC12s_v2": 4.140000, + "Standard_NC12s_v3": 6.120000, + "Standard_NC16as_T4_v3": 1.204000, + "Standard_NC24": 3.600000, + "Standard_NC24ads_A100_v4": 3.673000, + "Standard_NC24r": 3.960000, + "Standard_NC24rs_v2": 9.108000, + "Standard_NC24rs_v3": 13.460000, + "Standard_NC24s_v2": 8.280000, + "Standard_NC24s_v3": 12.240000, + "Standard_NC48ads_A100_v4": 7.346000, + "Standard_NC4as_T4_v3": 0.526000, + "Standard_NC6": 0.900000, + "Standard_NC64as_T4_v3": 4.352000, + "Standard_NC6s_v2": 2.070000, + "Standard_NC6s_v3": 3.060000, + "Standard_NC8as_T4_v3": 0.752000, + "Standard_NC96ads_A100_v4": 14.692000, + "Standard_ND12s": 4.140000, + "Standard_ND24rs": 9.108000, + "Standard_ND24s": 8.280000, + "Standard_ND40rs_v2": 22.032000, + "Standard_ND40s_v2": 12.240000, + "Standard_ND6s": 2.070000, + "Standard_ND96ams_A100_v4": 32.770000, + "Standard_ND96amsr_A100_v4": 32.770000, + "Standard_ND96asr_A100_v4": 27.197000, + "Standard_ND96asr_v4": 27.197000, + "Standard_NP10s": 1.650000, + "Standard_NP20s": 3.300000, + "Standard_NP40s": 6.600000, + "Standard_NV12": 2.280000, + "Standard_NV12ads_A10_v5": 0.908000, + "Standard_NV12s_v3": 1.140000, + "Standard_NV16as_v4": 0.932000, + "Standard_NV18ads_A10_v5": 1.600000, + "Standard_NV24": 4.560000, + "Standard_NV24s_v3": 2.280000, + "Standard_NV32as_v4": 1.864000, + "Standard_NV36adms_A10_v5": 4.520000, + "Standard_NV36ads_A10_v5": 3.200000, + "Standard_NV48s_v3": 4.560000, + "Standard_NV4as_v4": 0.233000, + "Standard_NV6": 1.140000, + "Standard_NV6ads_A10_v5": 0.454000, + "Standard_NV72ads_A10_v5": 6.520000, + "Standard_NV8as_v4": 0.466000, + "Standard_PB12s": 1.660000, + "Standard_PB24s": 3.320000, + "Standard_PB6s": 0.830000, + } + // eastus2 + initialOnDemandPrices["eastus2"] = map[string]float64{ + "Basic_A0": 0.018000, + "Basic_A1": 0.024000, + "Basic_A2": 0.068000, + "Basic_A3": 0.176000, + "Basic_A4": 0.352000, + "DCdsv3 Type1": 5.966000, + "DCsv3 Type1": 5.069000, + "Dadsv5_Type1": 6.345000, + "Dasv4_Type1": 5.069000, + "Dasv4_Type2": 5.069000, + "Dasv5_Type1": 5.298000, + "Ddsv4_Type 1": 3.978000, + "Ddsv4_Type2": 4.723000, + "Ddsv5_Type1": 5.966000, + "Dsv3_Type3": 4.225000, + "Dsv3_Type4": 5.281000, + "Dsv4_Type1": 4.224000, + "Dsv4_Type2": 5.280000, + "Dsv5_Type1": 5.280000, + "Eadsv5_Type1": 6.917000, + "Easv4_Type1": 6.653000, + "Easv4_Type2": 6.653000, + "Easv5_Type1": 5.966000, + "Ebdsv5-Type1": 5.878000, + "Ebsv5-Type1": 5.245000, + "Edsv4_Type 1": 5.069000, + "Edsv4_Type2": 6.019000, + "Edsv5_Type1": 7.603000, + "Esv3_Type3": 4.297000, + "Esv3_Type4": 5.822000, + "Esv4_Type1": 4.435000, + "Esv4_Type2": 5.821000, + "Esv5_Type1": 6.653000, + "FXmds Type1": 4.910000, + "Fsv2 Type3": 4.276000, + "Fsv2_Type2": 3.366000, + "Fsv2_Type4": 4.675000, + "Lasv3_Type1": 6.864000, + "Lsv2_Type1": 6.864000, + "Lsv3_Type1": 7.678000, + "Mdmsv2MedMem _Type1": 29.359000, + "Mdsv2MedMem_Type1": 14.674000, + "Ms_Type1": 14.669000, + "Msm_Type1": 29.363000, + "Msmv2MedMem Type1": 28.896000, + "Msmv2_Type1": 109.060000, + "Msv2MedMem Type1": 14.211000, + "Msv2_Type1": 54.538000, + "NVasv4_Type1": 7.176000, + "NVsv3_Type1": 5.016000, + "Standard_A0": 0.020000, + "Standard_A1": 0.060000, + "Standard_A1_v2": 0.043000, + "Standard_A2": 0.120000, + "Standard_A2_v2": 0.091000, + "Standard_A2m_v2": 0.119000, + "Standard_A3": 0.240000, + "Standard_A4": 0.480000, + "Standard_A4_v2": 0.191000, + "Standard_A4m_v2": 0.249000, + "Standard_A5": 0.220000, + "Standard_A6": 0.440000, + "Standard_A7": 0.880000, + "Standard_A8_v2": 0.400000, + "Standard_A8m_v2": 0.524000, + "Standard_B12ms": 0.499000, + "Standard_B16als_v2": 0.606000, + "Standard_B16as_v2": 0.602000, + "Standard_B16ls_v2": 0.663000, + "Standard_B16ms": 0.666000, + "Standard_B16pls_v2": 0.550000, + "Standard_B16ps_v2": 0.538000, + "Standard_B16s_v2": 0.666000, + "Standard_B1ls": 0.005200, + "Standard_B1ms": 0.020800, + "Standard_B1s": 0.010400, + "Standard_B20ms": 0.832000, + "Standard_B2als_v2": 0.046800, + "Standard_B2as_v2": 0.084400, + "Standard_B2ats_v2": 0.018600, + "Standard_B2ls_v2": 0.041600, + "Standard_B2ms": 0.083200, + "Standard_B2pls_v2": 0.033600, + "Standard_B2ps_v2": 0.067200, + "Standard_B2pts_v2": 0.017600, + "Standard_B2s": 0.041600, + "Standard_B2s_v2": 0.083200, + "Standard_B2ts_v2": 0.010400, + "Standard_B32als_v2": 1.213000, + "Standard_B32as_v2": 1.203000, + "Standard_B32ls_v2": 1.179000, + "Standard_B32s_v2": 1.478000, + "Standard_B4als_v2": 0.133000, + "Standard_B4as_v2": 0.150000, + "Standard_B4ls_v2": 0.166000, + "Standard_B4ms": 0.166000, + "Standard_B4pls_v2": 0.137000, + "Standard_B4ps_v2": 0.134000, + "Standard_B4s_v2": 0.166000, + "Standard_B8als_v2": 0.266000, + "Standard_B8as_v2": 0.301000, + "Standard_B8ls_v2": 0.295000, + "Standard_B8ms": 0.333000, + "Standard_B8pls_v2": 0.275000, + "Standard_B8ps_v2": 0.306000, + "Standard_B8s_v2": 0.333000, + "Standard_D1": 0.067000, + "Standard_D11": 0.173000, + "Standard_D11_v2": 0.149000, + "Standard_D11_v2_Promo": 0.149000, + "Standard_D12": 0.347000, + "Standard_D12_v2": 0.299000, + "Standard_D12_v2_Promo": 0.299000, + "Standard_D13": 0.694000, + "Standard_D13_v2": 0.598000, + "Standard_D13_v2_Promo": 0.598000, + "Standard_D14": 1.387000, + "Standard_D14_v2": 1.196000, + "Standard_D14_v2_Promo": 1.196000, + "Standard_D15_v2": 1.495000, + "Standard_D15i_v2": 1.495000, + "Standard_D16_v3": 0.768000, + "Standard_D16_v4": 0.768000, + "Standard_D16_v5": 0.768000, + "Standard_D16a_v4": 0.768000, + "Standard_D16ads_v5": 1.560000, + "Standard_D16as_v4": 0.768000, + "Standard_D16as_v5": 0.688000, + "Standard_D16as_v5_Promo": 0.616000, + "Standard_D16d_v4": 0.904000, + "Standard_D16d_v5": 0.904000, + "Standard_D16ds_v4": 0.904000, + "Standard_D16ds_v5": 0.904000, + "Standard_D16lds_v5": 0.768000, + "Standard_D16ls_v5": 0.680000, + "Standard_D16pds_v5": 0.723000, + "Standard_D16plds_v5": 0.614000, + "Standard_D16pls_v5": 0.544000, + "Standard_D16ps_v5": 0.616000, + "Standard_D16s_v3": 0.768000, + "Standard_D16s_v4": 0.768000, + "Standard_D16s_v5": 0.768000, + "Standard_D1_v2": 0.057000, + "Standard_D2": 0.134000, + "Standard_D2_v2": 0.114000, + "Standard_D2_v2_Promo": 0.114000, + "Standard_D2_v3": 0.096000, + "Standard_D2_v4": 0.096000, + "Standard_D2_v5": 0.096000, + "Standard_D2a_v4": 0.096000, + "Standard_D2ads_v5": 0.195000, + "Standard_D2as_v4": 0.096000, + "Standard_D2as_v5": 0.178000, + "Standard_D2as_v5_Promo": 0.077000, + "Standard_D2d_v4": 0.113000, + "Standard_D2d_v5": 0.113000, + "Standard_D2ds_v4": 0.113000, + "Standard_D2ds_v5": 0.113000, + "Standard_D2lds_v5": 0.096000, + "Standard_D2ls_v5": 0.085000, + "Standard_D2pds_v5": 0.090400, + "Standard_D2plds_v5": 0.076800, + "Standard_D2pls_v5": 0.068000, + "Standard_D2ps_v5": 0.077000, + "Standard_D2s_v3": 0.096000, + "Standard_D2s_v4": 0.096000, + "Standard_D2s_v5": 0.096000, + "Standard_D3": 0.268000, + "Standard_D32-16s_v3": 1.536000, + "Standard_D32-8s_v3": 1.536000, + "Standard_D32_v3": 1.536000, + "Standard_D32_v4": 1.536000, + "Standard_D32_v5": 1.536000, + "Standard_D32a_v4": 1.536000, + "Standard_D32ads_v5": 3.120000, + "Standard_D32as_v4": 1.536000, + "Standard_D32as_v5": 1.376000, + "Standard_D32as_v5_Promo": 1.232000, + "Standard_D32d_v4": 1.808000, + "Standard_D32d_v5": 1.808000, + "Standard_D32ds_v4": 1.808000, + "Standard_D32ds_v5": 1.808000, + "Standard_D32lds_v5": 1.536000, + "Standard_D32ls_v5": 1.360000, + "Standard_D32pds_v5": 1.446000, + "Standard_D32plds_v5": 1.229000, + "Standard_D32pls_v5": 1.088000, + "Standard_D32ps_v5": 1.232000, + "Standard_D32s_v3": 1.536000, + "Standard_D32s_v4": 1.536000, + "Standard_D32s_v5": 1.536000, + "Standard_D3_v2": 0.229000, + "Standard_D3_v2_Promo": 0.229000, + "Standard_D4": 0.536000, + "Standard_D48_v3": 2.304000, + "Standard_D48_v4": 2.304000, + "Standard_D48_v5": 2.304000, + "Standard_D48a_v4": 2.304000, + "Standard_D48ads_v5": 4.680000, + "Standard_D48as_v4": 2.304000, + "Standard_D48as_v5": 4.272000, + "Standard_D48as_v5_Promo": 1.848000, + "Standard_D48d_v4": 2.712000, + "Standard_D48d_v5": 2.712000, + "Standard_D48ds_v4": 2.712000, + "Standard_D48ds_v5": 2.712000, + "Standard_D48lds_v5": 2.304000, + "Standard_D48ls_v5": 2.040000, + "Standard_D48pds_v5": 2.170000, + "Standard_D48plds_v5": 1.843000, + "Standard_D48pls_v5": 1.632000, + "Standard_D48ps_v5": 1.848000, + "Standard_D48s_v3": 2.304000, + "Standard_D48s_v4": 2.304000, + "Standard_D48s_v5": 2.304000, + "Standard_D4_v2": 0.458000, + "Standard_D4_v2_Promo": 0.458000, + "Standard_D4_v3": 0.192000, + "Standard_D4_v4": 0.192000, + "Standard_D4_v5": 0.192000, + "Standard_D4a_v4": 0.192000, + "Standard_D4ads_v5": 0.390000, + "Standard_D4as_v4": 0.192000, + "Standard_D4as_v5": 0.356000, + "Standard_D4as_v5_Promo": 0.154000, + "Standard_D4d_v4": 0.226000, + "Standard_D4d_v5": 0.226000, + "Standard_D4ds_v4": 0.226000, + "Standard_D4ds_v5": 0.226000, + "Standard_D4lds_v5": 0.192000, + "Standard_D4ls_v5": 0.170000, + "Standard_D4pds_v5": 0.181000, + "Standard_D4plds_v5": 0.154000, + "Standard_D4pls_v5": 0.136000, + "Standard_D4ps_v5": 0.154000, + "Standard_D4s_v3": 0.192000, + "Standard_D4s_v4": 0.192000, + "Standard_D4s_v5": 0.192000, + "Standard_D5_v2": 0.916000, + "Standard_D5_v2_Promo": 0.916000, + "Standard_D64-16s_v3": 3.072000, + "Standard_D64-32s_v3": 3.072000, + "Standard_D64_v3": 3.072000, + "Standard_D64_v4": 3.072000, + "Standard_D64_v5": 3.072000, + "Standard_D64a_v4": 3.072000, + "Standard_D64ads_v5": 6.240000, + "Standard_D64as_v4": 3.072000, + "Standard_D64as_v5": 5.696000, + "Standard_D64as_v5_Promo": 2.464000, + "Standard_D64d_v4": 3.616000, + "Standard_D64d_v5": 3.616000, + "Standard_D64ds_v4": 3.616000, + "Standard_D64ds_v5": 3.616000, + "Standard_D64lds_v5": 3.072000, + "Standard_D64ls_v5": 2.720000, + "Standard_D64pds_v5": 2.893000, + "Standard_D64plds_v5": 2.458000, + "Standard_D64pls_v5": 2.176000, + "Standard_D64ps_v5": 2.464000, + "Standard_D64s_v3": 3.072000, + "Standard_D64s_v4": 3.072000, + "Standard_D64s_v5": 3.072000, + "Standard_D8_v3": 0.384000, + "Standard_D8_v4": 0.384000, + "Standard_D8_v5": 0.384000, + "Standard_D8a_v4": 0.384000, + "Standard_D8ads_v5": 0.780000, + "Standard_D8as_v4": 0.384000, + "Standard_D8as_v5": 0.712000, + "Standard_D8as_v5_Promo": 0.308000, + "Standard_D8d_v4": 0.452000, + "Standard_D8d_v5": 0.452000, + "Standard_D8ds_v4": 0.452000, + "Standard_D8ds_v5": 0.452000, + "Standard_D8lds_v5": 0.384000, + "Standard_D8ls_v5": 0.340000, + "Standard_D8pds_v5": 0.362000, + "Standard_D8plds_v5": 0.307000, + "Standard_D8pls_v5": 0.272000, + "Standard_D8ps_v5": 0.308000, + "Standard_D8s_v3": 0.384000, + "Standard_D8s_v4": 0.384000, + "Standard_D8s_v5": 0.384000, + "Standard_D96_v5": 4.608000, + "Standard_D96a_v4": 4.608000, + "Standard_D96ads_v5": 4.944000, + "Standard_D96as_v4": 4.608000, + "Standard_D96as_v5": 4.128000, + "Standard_D96as_v5_Promo": 3.696000, + "Standard_D96d_v5": 5.424000, + "Standard_D96ds_v5": 5.424000, + "Standard_D96lds_v5": 4.608000, + "Standard_D96ls_v5": 4.080000, + "Standard_D96s_v5": 4.608000, + "Standard_DC16ads_v5": 0.824000, + "Standard_DC16as_v5": 0.688000, + "Standard_DC16ds_v3": 1.808000, + "Standard_DC16eds_v5": 0.904000, + "Standard_DC16es_v5": 0.768000, + "Standard_DC16s_v3": 1.536000, + "Standard_DC1ds_v3": 0.113000, + "Standard_DC1s_v3": 0.096000, + "Standard_DC24ds_v3": 2.712000, + "Standard_DC24s_v3": 2.304000, + "Standard_DC2ads_v5": 0.103000, + "Standard_DC2as_v5": 0.086000, + "Standard_DC2ds_v3": 0.226000, + "Standard_DC2eds_v5": 0.113000, + "Standard_DC2es_v5": 0.096000, + "Standard_DC2s_v3": 0.192000, + "Standard_DC32ads_v5": 1.648000, + "Standard_DC32as_v5": 1.376000, + "Standard_DC32ds_v3": 3.616000, + "Standard_DC32eds_v5": 1.808000, + "Standard_DC32es_v5": 1.536000, + "Standard_DC32s_v3": 3.072000, + "Standard_DC48ads_v5": 2.472000, + "Standard_DC48as_v5": 2.064000, + "Standard_DC48ds_v3": 5.424000, + "Standard_DC48eds_v5": 2.712000, + "Standard_DC48es_v5": 2.304000, + "Standard_DC48s_v3": 4.608000, + "Standard_DC4ads_v5": 0.206000, + "Standard_DC4as_v5": 0.172000, + "Standard_DC4ds_v3": 0.452000, + "Standard_DC4eds_v5": 0.226000, + "Standard_DC4es_v5": 0.192000, + "Standard_DC4s_v3": 0.384000, + "Standard_DC64ads_v5": 3.296000, + "Standard_DC64as_v5": 2.752000, + "Standard_DC64eds_v5": 3.616000, + "Standard_DC64es_v5": 3.072000, + "Standard_DC8ads_v5": 0.412000, + "Standard_DC8as_v5": 0.344000, + "Standard_DC8ds_v3": 0.904000, + "Standard_DC8eds_v5": 0.452000, + "Standard_DC8es_v5": 0.384000, + "Standard_DC8s_v3": 0.768000, + "Standard_DC96ads_v5": 4.944000, + "Standard_DC96as_v5": 4.128000, + "Standard_DC96eds_v5": 5.424000, + "Standard_DC96es_v5": 4.608000, + "Standard_DS1": 0.067000, + "Standard_DS11": 0.173000, + "Standard_DS11-1_v2": 0.149000, + "Standard_DS11_v2": 0.149000, + "Standard_DS11_v2_Promo": 0.149000, + "Standard_DS12": 0.347000, + "Standard_DS12-1_v2": 0.299000, + "Standard_DS12-2_v2": 0.299000, + "Standard_DS12_v2": 0.299000, + "Standard_DS12_v2_Promo": 0.299000, + "Standard_DS13": 0.694000, + "Standard_DS13-2_v2": 0.598000, + "Standard_DS13-4_v2": 0.598000, + "Standard_DS13_v2": 0.598000, + "Standard_DS13_v2_Promo": 0.598000, + "Standard_DS14": 1.387000, + "Standard_DS14-4_v2": 1.196000, + "Standard_DS14-8_v2": 1.196000, + "Standard_DS14_v2": 1.196000, + "Standard_DS14_v2_Promo": 1.196000, + "Standard_DS15_v2": 1.495000, + "Standard_DS15i_v2": 1.495000, + "Standard_DS1_v2": 0.057000, + "Standard_DS2": 0.134000, + "Standard_DS2_v2": 0.114000, + "Standard_DS2_v2_Promo": 0.114000, + "Standard_DS3": 0.268000, + "Standard_DS3_v2": 0.229000, + "Standard_DS3_v2_Promo": 0.229000, + "Standard_DS4": 0.536000, + "Standard_DS4_v2": 0.458000, + "Standard_DS4_v2_Promo": 0.458000, + "Standard_DS5_v2": 0.916000, + "Standard_DS5_v2_Promo": 0.916000, + "Standard_E104i_v5": 7.207000, + "Standard_E104id_v5": 8.237000, + "Standard_E104ids_v5": 8.237000, + "Standard_E104is_v5": 7.207000, + "Standard_E112iads_v5": 8.070000, + "Standard_E112ias_v5": 6.961000, + "Standard_E112ibds_v5": 10.287000, + "Standard_E112ibs_v5": 9.178000, + "Standard_E16-4ads_v5": 1.784000, + "Standard_E16-4as_v4": 1.008000, + "Standard_E16-4as_v5": 1.640000, + "Standard_E16-4ds_v4": 1.152000, + "Standard_E16-4ds_v5": 1.152000, + "Standard_E16-4s_v3": 1.064000, + "Standard_E16-4s_v4": 1.008000, + "Standard_E16-4s_v5": 1.008000, + "Standard_E16-8ads_v5": 1.048000, + "Standard_E16-8as_v4": 1.008000, + "Standard_E16-8as_v5": 0.904000, + "Standard_E16-8ds_v4": 1.152000, + "Standard_E16-8ds_v5": 1.152000, + "Standard_E16-8s_v3": 1.064000, + "Standard_E16-8s_v4": 1.008000, + "Standard_E16-8s_v5": 1.008000, + "Standard_E16_v3": 1.064000, + "Standard_E16_v4": 1.008000, + "Standard_E16_v5": 1.008000, + "Standard_E16a_v4": 1.008000, + "Standard_E16ads_v5": 1.048000, + "Standard_E16as_v4": 1.008000, + "Standard_E16as_v5": 0.950000, + "Standard_E16bds_v5": 1.336000, + "Standard_E16bs_v5": 1.192000, + "Standard_E16d_v4": 1.152000, + "Standard_E16d_v5": 1.152000, + "Standard_E16ds_v4": 1.152000, + "Standard_E16ds_v5": 1.152000, + "Standard_E16pds_v5": 0.922000, + "Standard_E16ps_v5": 0.806000, + "Standard_E16s_v3": 1.064000, + "Standard_E16s_v4": 1.008000, + "Standard_E16s_v5": 1.008000, + "Standard_E20_v3": 1.460000, + "Standard_E20_v4": 1.260000, + "Standard_E20_v5": 1.260000, + "Standard_E20a_v4": 1.260000, + "Standard_E20ads_v5": 1.356000, + "Standard_E20as_v4": 1.260000, + "Standard_E20as_v5": 1.176000, + "Standard_E20d_v4": 1.440000, + "Standard_E20d_v5": 1.440000, + "Standard_E20ds_v4": 1.440000, + "Standard_E20ds_v5": 1.440000, + "Standard_E20pds_v5": 1.152000, + "Standard_E20ps_v5": 1.008000, + "Standard_E20s_v3": 1.460000, + "Standard_E20s_v4": 1.260000, + "Standard_E20s_v5": 1.260000, + "Standard_E2_v3": 0.133000, + "Standard_E2_v4": 0.126000, + "Standard_E2_v5": 0.126000, + "Standard_E2a_v4": 0.126000, + "Standard_E2ads_v5": 0.131000, + "Standard_E2as_v4": 0.126000, + "Standard_E2as_v5": 0.113000, + "Standard_E2bds_v5": 0.167000, + "Standard_E2bs_v5": 0.149000, + "Standard_E2d_v4": 0.144000, + "Standard_E2d_v5": 0.144000, + "Standard_E2ds_v4": 0.144000, + "Standard_E2ds_v5": 0.144000, + "Standard_E2pds_v5": 0.115000, + "Standard_E2ps_v5": 0.101000, + "Standard_E2s_v3": 0.133000, + "Standard_E2s_v4": 0.126000, + "Standard_E2s_v5": 0.126000, + "Standard_E32-16ads_v5": 3.568000, + "Standard_E32-16as_v4": 2.016000, + "Standard_E32-16as_v5": 3.280000, + "Standard_E32-16ds_v4": 2.304000, + "Standard_E32-16ds_v5": 2.304000, + "Standard_E32-16s_v3": 2.128000, + "Standard_E32-16s_v4": 2.016000, + "Standard_E32-16s_v5": 2.016000, + "Standard_E32-8ads_v5": 3.568000, + "Standard_E32-8as_v4": 2.016000, + "Standard_E32-8as_v5": 3.280000, + "Standard_E32-8ds_v4": 2.304000, + "Standard_E32-8ds_v5": 2.304000, + "Standard_E32-8s_v3": 2.128000, + "Standard_E32-8s_v4": 2.016000, + "Standard_E32-8s_v5": 2.016000, + "Standard_E32_v3": 2.128000, + "Standard_E32_v4": 2.016000, + "Standard_E32_v5": 2.016000, + "Standard_E32a_v4": 2.016000, + "Standard_E32ads_v5": 2.142000, + "Standard_E32as_v4": 2.016000, + "Standard_E32as_v5": 1.854000, + "Standard_E32bds_v5": 2.672000, + "Standard_E32bs_v5": 2.384000, + "Standard_E32d_v4": 2.304000, + "Standard_E32d_v5": 2.304000, + "Standard_E32ds_v4": 2.304000, + "Standard_E32ds_v5": 2.304000, + "Standard_E32pds_v5": 1.843000, + "Standard_E32ps_v5": 1.613000, + "Standard_E32s_v3": 2.128000, + "Standard_E32s_v4": 2.016000, + "Standard_E32s_v5": 2.016000, + "Standard_E4-2ads_v5": 0.446000, + "Standard_E4-2as_v4": 0.252000, + "Standard_E4-2as_v5": 0.410000, + "Standard_E4-2ds_v4": 0.288000, + "Standard_E4-2ds_v5": 0.288000, + "Standard_E4-2s_v3": 0.266000, + "Standard_E4-2s_v4": 0.252000, + "Standard_E4-2s_v5": 0.252000, + "Standard_E48_v3": 3.024000, + "Standard_E48_v4": 3.024000, + "Standard_E48_v5": 3.024000, + "Standard_E48a_v4": 3.024000, + "Standard_E48ads_v5": 3.144000, + "Standard_E48as_v4": 3.024000, + "Standard_E48as_v5": 2.758000, + "Standard_E48bds_v5": 4.008000, + "Standard_E48bs_v5": 3.576000, + "Standard_E48d_v4": 3.456000, + "Standard_E48d_v5": 3.456000, + "Standard_E48ds_v4": 3.456000, + "Standard_E48ds_v5": 3.456000, + "Standard_E48s_v3": 3.024000, + "Standard_E48s_v4": 3.024000, + "Standard_E48s_v5": 3.024000, + "Standard_E4_v3": 0.266000, + "Standard_E4_v4": 0.252000, + "Standard_E4_v5": 0.252000, + "Standard_E4a_v4": 0.252000, + "Standard_E4ads_v5": 0.262000, + "Standard_E4as_v4": 0.252000, + "Standard_E4as_v5": 0.272000, + "Standard_E4bds_v5": 0.334000, + "Standard_E4bs_v5": 0.298000, + "Standard_E4d_v4": 0.288000, + "Standard_E4d_v5": 0.288000, + "Standard_E4ds_v4": 0.288000, + "Standard_E4ds_v5": 0.288000, + "Standard_E4pds_v5": 0.230000, + "Standard_E4ps_v5": 0.202000, + "Standard_E4s_v3": 0.266000, + "Standard_E4s_v4": 0.252000, + "Standard_E4s_v5": 0.252000, + "Standard_E64-16ads_v5": 4.192000, + "Standard_E64-16as_v4": 4.032000, + "Standard_E64-16as_v5": 6.560000, + "Standard_E64-16ds_v4": 4.608000, + "Standard_E64-16ds_v5": 4.608000, + "Standard_E64-16s_v3": 3.629000, + "Standard_E64-16s_v4": 4.032000, + "Standard_E64-16s_v5": 4.032000, + "Standard_E64-32ads_v5": 4.192000, + "Standard_E64-32as_v4": 4.032000, + "Standard_E64-32as_v5": 3.616000, + "Standard_E64-32ds_v4": 4.608000, + "Standard_E64-32ds_v5": 4.608000, + "Standard_E64-32s_v3": 3.629000, + "Standard_E64-32s_v4": 4.032000, + "Standard_E64-32s_v5": 4.032000, + "Standard_E64_v3": 3.629000, + "Standard_E64_v4": 4.032000, + "Standard_E64_v5": 4.032000, + "Standard_E64a_v4": 4.032000, + "Standard_E64ads_v5": 4.238000, + "Standard_E64as_v4": 4.032000, + "Standard_E64as_v5": 3.662000, + "Standard_E64bds_v5": 5.344000, + "Standard_E64bs_v5": 4.768000, + "Standard_E64d_v4": 4.608000, + "Standard_E64d_v5": 4.608000, + "Standard_E64ds_v4": 4.608000, + "Standard_E64ds_v5": 4.608000, + "Standard_E64i_v3": 3.629000, + "Standard_E64is_v3": 3.629000, + "Standard_E64s_v3": 3.629000, + "Standard_E64s_v4": 4.032000, + "Standard_E64s_v5": 4.032000, + "Standard_E8-2ads_v5": 0.524000, + "Standard_E8-2as_v4": 0.504000, + "Standard_E8-2as_v5": 0.452000, + "Standard_E8-2ds_v4": 0.576000, + "Standard_E8-2ds_v5": 0.576000, + "Standard_E8-2s_v3": 0.532000, + "Standard_E8-2s_v4": 0.504000, + "Standard_E8-2s_v5": 0.504000, + "Standard_E8-4ads_v5": 0.892000, + "Standard_E8-4as_v4": 0.504000, + "Standard_E8-4as_v5": 0.452000, + "Standard_E8-4ds_v4": 0.576000, + "Standard_E8-4ds_v5": 0.576000, + "Standard_E8-4s_v3": 0.532000, + "Standard_E8-4s_v4": 0.504000, + "Standard_E8-4s_v5": 0.504000, + "Standard_E80ids_v4": 5.760000, + "Standard_E80is_v4": 5.040000, + "Standard_E8_v3": 0.532000, + "Standard_E8_v4": 0.504000, + "Standard_E8_v5": 0.504000, + "Standard_E8a_v4": 0.504000, + "Standard_E8ads_v5": 0.524000, + "Standard_E8as_v4": 0.504000, + "Standard_E8as_v5": 0.498000, + "Standard_E8bds_v5": 0.668000, + "Standard_E8bs_v5": 0.596000, + "Standard_E8d_v4": 0.576000, + "Standard_E8d_v5": 0.576000, + "Standard_E8ds_v4": 0.576000, + "Standard_E8ds_v5": 0.576000, + "Standard_E8pds_v5": 0.461000, + "Standard_E8ps_v5": 0.403000, + "Standard_E8s_v3": 0.532000, + "Standard_E8s_v4": 0.504000, + "Standard_E8s_v5": 0.504000, + "Standard_E96-24ads_v5": 6.288000, + "Standard_E96-24as_v4": 6.048000, + "Standard_E96-24as_v5": 9.840000, + "Standard_E96-24ds_v5": 6.912000, + "Standard_E96-24s_v5": 6.048000, + "Standard_E96-48ads_v5": 6.288000, + "Standard_E96-48as_v4": 6.048000, + "Standard_E96-48as_v5": 9.840000, + "Standard_E96-48ds_v5": 6.912000, + "Standard_E96-48s_v5": 6.048000, + "Standard_E96_v5": 6.048000, + "Standard_E96a_v4": 6.048000, + "Standard_E96ads_v5": 6.334000, + "Standard_E96as_v4": 6.048000, + "Standard_E96as_v5": 5.424000, + "Standard_E96bds_v5": 8.016000, + "Standard_E96bs_v5": 7.152000, + "Standard_E96d_v5": 6.912000, + "Standard_E96ds_v5": 6.912000, + "Standard_E96iads_v5": 6.917000, + "Standard_E96ias_v5": 5.966000, + "Standard_E96s_v5": 6.048000, + "Standard_EC16ads_v5": 1.048000, + "Standard_EC16as_v5": 0.904000, + "Standard_EC16eds_v5": 1.152000, + "Standard_EC16es_v5": 1.008000, + "Standard_EC20ads_v5": 1.310000, + "Standard_EC20as_v5": 1.130000, + "Standard_EC2ads_v5": 0.131000, + "Standard_EC2as_v5": 0.113000, + "Standard_EC2eds_v5": 0.144000, + "Standard_EC2es_v5": 0.126000, + "Standard_EC32ads_v5": 2.096000, + "Standard_EC32as_v5": 1.808000, + "Standard_EC32eds_v5": 2.304000, + "Standard_EC32es_v5": 2.016000, + "Standard_EC48ads_v5": 3.144000, + "Standard_EC48as_v5": 2.712000, + "Standard_EC48eds_v5": 3.456000, + "Standard_EC48es_v5": 3.024000, + "Standard_EC4ads_v5": 0.262000, + "Standard_EC4as_v5": 0.226000, + "Standard_EC4eds_v5": 0.288000, + "Standard_EC4es_v5": 0.252000, + "Standard_EC64ads_v5": 4.192000, + "Standard_EC64as_v5": 3.616000, + "Standard_EC64eds_v5": 4.608000, + "Standard_EC64es_v5": 4.032000, + "Standard_EC8ads_v5": 0.524000, + "Standard_EC8as_v5": 0.452000, + "Standard_EC8eds_v5": 0.576000, + "Standard_EC8es_v5": 0.504000, + "Standard_EC96ads_v5": 6.288000, + "Standard_EC96as_v5": 5.424000, + "Standard_EC96eds_v5": 6.912000, + "Standard_EC96es_v5": 6.048000, + "Standard_EC96iads_v5": 6.917000, + "Standard_EC96ias_v5": 5.966000, + "Standard_F1": 0.049700, + "Standard_F16": 0.796000, + "Standard_F16s": 0.796000, + "Standard_F16s_v2": 0.677000, + "Standard_F1s": 0.049700, + "Standard_F2": 0.099000, + "Standard_F2s": 0.099000, + "Standard_F2s_v2": 0.084600, + "Standard_F32s_v2": 1.353000, + "Standard_F4": 0.199000, + "Standard_F48s_v2": 2.030000, + "Standard_F4s": 0.199000, + "Standard_F4s_v2": 0.169000, + "Standard_F64s_v2": 2.706000, + "Standard_F72s_v2": 3.045000, + "Standard_F8": 0.398000, + "Standard_F8s": 0.398000, + "Standard_F8s_v2": 0.338000, + "Standard_FX12mds": 1.116000, + "Standard_FX24mds": 2.232000, + "Standard_FX36mds": 3.348000, + "Standard_FX48mds": 4.464000, + "Standard_FX4mds": 0.372000, + "Standard_G1": 0.550000, + "Standard_G2": 1.100000, + "Standard_G3": 2.200000, + "Standard_G4": 4.400000, + "Standard_G5": 7.820000, + "Standard_GS1": 0.550000, + "Standard_GS2": 1.100000, + "Standard_GS3": 2.200000, + "Standard_GS4": 4.400000, + "Standard_GS4-4": 4.400000, + "Standard_GS4-8": 4.400000, + "Standard_GS5": 7.820000, + "Standard_GS5-16": 7.820000, + "Standard_GS5-8": 7.820000, + "Standard_HB176-144rs_v4": 7.200000, + "Standard_HB176-24rs_v4": 7.200000, + "Standard_HB176-48rs_v4": 7.200000, + "Standard_HB176-96rs_v4": 7.200000, + "Standard_HB176rs_v4": 7.200000, + "Standard_HX176-144rs": 8.640000, + "Standard_HX176-24rs": 8.640000, + "Standard_HX176-48rs": 8.640000, + "Standard_HX176-96rs": 8.640000, + "Standard_HX176rs": 8.640000, + "Standard_L16as_v3": 1.248000, + "Standard_L16s": 1.373000, + "Standard_L16s_v2": 1.248000, + "Standard_L16s_v3": 1.396000, + "Standard_L32as_v3": 2.496000, + "Standard_L32s": 2.746000, + "Standard_L32s_v2": 2.496000, + "Standard_L32s_v3": 2.792000, + "Standard_L48as_v3": 3.744000, + "Standard_L48s_v2": 3.744000, + "Standard_L48s_v3": 4.188000, + "Standard_L4s": 0.343000, + "Standard_L64as_v3": 4.992000, + "Standard_L64s_v2": 4.992000, + "Standard_L64s_v3": 5.584000, + "Standard_L80as_v3": 6.240000, + "Standard_L80s_v2": 6.240000, + "Standard_L80s_v3": 6.980000, + "Standard_L88is_v2": 6.864000, + "Standard_L8as_v3": 0.624000, + "Standard_L8s": 0.686000, + "Standard_L8s_v2": 0.624000, + "Standard_L8s_v3": 0.698000, + "Standard_M128": 13.338000, + "Standard_M128-32ms": 26.688000, + "Standard_M128-64ms": 26.688000, + "Standard_M128dms_v2": 26.690000, + "Standard_M128ds_v2": 13.340000, + "Standard_M128m": 26.688000, + "Standard_M128ms": 26.688000, + "Standard_M128ms_v2": 26.269000, + "Standard_M128s": 13.338000, + "Standard_M128s_v2": 12.919000, + "Standard_M12ds_v3": 0.000000, + "Standard_M12s_v3": 0.000000, + "Standard_M16-4ms": 3.073000, + "Standard_M16-8ms": 3.073000, + "Standard_M16ms": 3.073000, + "Standard_M16s": 2.387000, + "Standard_M176ds_3_v3": 0.000000, + "Standard_M176s_3_v3": 0.000000, + "Standard_M192idms_v2": 32.064000, + "Standard_M192ids_v2": 16.032000, + "Standard_M192ims_v2": 31.643000, + "Standard_M192is_v2": 15.611000, + "Standard_M208-104ms_v2": 44.620000, + "Standard_M208-104s_v2": 22.310000, + "Standard_M208-52ms_v2": 44.620000, + "Standard_M208-52s_v2": 22.310000, + "Standard_M208ms_v2": 44.620000, + "Standard_M208s_v2": 22.310000, + "Standard_M24ds_v3": 0.000000, + "Standard_M24s_v3": 0.000000, + "Standard_M32-16ms": 6.146000, + "Standard_M32-8ms": 6.146000, + "Standard_M32dms_v2": 6.146000, + "Standard_M32ls": 2.873000, + "Standard_M32ms": 6.146000, + "Standard_M32ms_v2": 6.041000, + "Standard_M32s": 3.335000, + "Standard_M32ts": 2.707000, + "Standard_M416-104ms_v2": 99.150000, + "Standard_M416-104s_v2": 49.580000, + "Standard_M416-208ms_v2": 99.150000, + "Standard_M416-208s_v2": 49.580000, + "Standard_M416is_v2": 49.580000, + "Standard_M416ms_v2": 99.150000, + "Standard_M416s_8_v2": 66.110000, + "Standard_M416s_v2": 49.580000, + "Standard_M48ds_1_v3": 0.000000, + "Standard_M48s_1_v3": 0.000000, + "Standard_M64": 6.669000, + "Standard_M64-16ms": 10.337000, + "Standard_M64-32ms": 10.337000, + "Standard_M64dms_v2": 10.340000, + "Standard_M64ds_v2": 6.669000, + "Standard_M64ls": 5.415000, + "Standard_M64m": 10.337000, + "Standard_M64ms": 10.337000, + "Standard_M64ms_v2": 10.130000, + "Standard_M64s": 6.669000, + "Standard_M64s_v2": 6.459000, + "Standard_M8-2ms": 1.536500, + "Standard_M8-4ms": 1.536500, + "Standard_M8ms": 1.536500, + "Standard_M96ds_1_v3": 0.000000, + "Standard_M96ds_2_v3": 0.000000, + "Standard_M96s_1_v3": 0.000000, + "Standard_M96s_2_v3": 0.000000, + "Standard_NC12": 1.800000, + "Standard_NC12s_v3": 6.120000, + "Standard_NC16as_T4_v3": 1.204000, + "Standard_NC24": 3.600000, + "Standard_NC24ads_A100_v4": 3.673000, + "Standard_NC24r": 3.960000, + "Standard_NC24rs_v3": 13.464000, + "Standard_NC24s_v3": 12.240000, + "Standard_NC48ads_A100_v4": 7.346000, + "Standard_NC4as_T4_v3": 0.526000, + "Standard_NC6": 0.900000, + "Standard_NC64as_T4_v3": 4.352000, + "Standard_NC6s_v3": 3.060000, + "Standard_NC8as_T4_v3": 0.752000, + "Standard_NC96ads_A100_v4": 14.692000, + "Standard_ND96ams_A100_v4": 32.770000, + "Standard_ND96amsr_A100_v4": 32.770000, + "Standard_NG16ads_V620_v1": 2.190000, + "Standard_NG32adms_V620": 5.694000, + "Standard_NG32ads_V620_v1": 4.380000, + "Standard_NG8ads_V620_v1": 1.095000, + "Standard_NV12": 2.403000, + "Standard_NV12ads_A10_v5": 0.908000, + "Standard_NV12s_v3": 1.140000, + "Standard_NV16as_v4": 0.932000, + "Standard_NV18ads_A10_v5": 1.600000, + "Standard_NV24": 4.807000, + "Standard_NV24s_v3": 2.280000, + "Standard_NV32as_v4": 1.864000, + "Standard_NV36adms_A10_v5": 4.520000, + "Standard_NV36ads_A10_v5": 3.200000, + "Standard_NV48s_v3": 4.560000, + "Standard_NV4as_v4": 0.233000, + "Standard_NV6": 1.202000, + "Standard_NV6ads_A10_v5": 0.454000, + "Standard_NV72ads_A10_v5": 6.520000, + "Standard_NV8as_v4": 0.466000, + } + // francecentral + initialOnDemandPrices["francecentral"] = map[string]float64{ + "Basic_A0": 0.020000, + "Basic_A1": 0.026000, + "Basic_A2": 0.101000, + "Basic_A3": 0.224000, + "Basic_A4": 0.449000, + "Dadsv5_Type1": 7.454000, + "Dasv4_Type1": 5.914000, + "Dasv4_Type2": 6.160000, + "Dasv5_Type1": 6.222000, + "Ddsv4_Type 1": 4.646000, + "Ddsv4_Type2": 5.518000, + "Ddsv5_Type1": 6.970000, + "Dsv3_Type3": 4.936000, + "Dsv3_Type4": 6.170000, + "Dsv4_Type1": 4.928000, + "Dsv4_Type2": 5.914000, + "Dsv5_Type1": 6.160000, + "Eadsv5_Type1": 8.078000, + "Easv4_Type1": 7.814000, + "Easv5_Type1": 7.022000, + "Ebdsv5-Type1": 6.952000, + "Ebsv5-Type1": 6.213000, + "Edsv4_Type 1": 5.949000, + "Edsv4_Type2": 7.064000, + "Edsv5_Type1": 8.923000, + "Esv3_Type3": 5.043000, + "Esv3_Type4": 6.832000, + "Esv4_Type1": 5.209000, + "Esv4_Type2": 6.838000, + "Esv5_Type1": 7.814000, + "Fsv2 Type3": 4.666000, + "Fsv2_Type2": 4.000000, + "Fsv2_Type4": 5.333000, + "Lsv2_Type1": 7.964000, + "Lsv3_Type1": 8.888000, + "Mdmsv2MedMem _Type1": 36.992000, + "Mdsv2MedMem_Type1": 18.489000, + "Ms_Type1": 18.483000, + "Msm_Type1": 36.998000, + "Msmv2MedMem Type1": 36.409000, + "Msv2MedMem Type1": 17.906000, + "NVsv3_Type1": 6.270000, + "Standard_A0": 0.022000, + "Standard_A1": 0.066000, + "Standard_A1_v2": 0.050000, + "Standard_A2": 0.132000, + "Standard_A2_v2": 0.106000, + "Standard_A2m_v2": 0.178000, + "Standard_A3": 0.264000, + "Standard_A4": 0.528000, + "Standard_A4_v2": 0.222000, + "Standard_A4m_v2": 0.374000, + "Standard_A5": 0.297000, + "Standard_A6": 0.594000, + "Standard_A7": 1.188000, + "Standard_A8_v2": 0.467000, + "Standard_A8m_v2": 0.786000, + "Standard_B12ms": 0.566000, + "Standard_B16als_v2": 0.676000, + "Standard_B16as_v2": 0.680000, + "Standard_B16ls_v2": 0.669000, + "Standard_B16ms": 0.755000, + "Standard_B16pls_v2": 0.606000, + "Standard_B16ps_v2": 0.602000, + "Standard_B16s_v2": 0.755000, + "Standard_B1ls": 0.005900, + "Standard_B1ms": 0.023600, + "Standard_B1s": 0.011800, + "Standard_B20ms": 0.944000, + "Standard_B2als_v2": 0.042500, + "Standard_B2as_v2": 0.085000, + "Standard_B2ats_v2": 0.019800, + "Standard_B2ls_v2": 0.056400, + "Standard_B2ms": 0.094400, + "Standard_B2pls_v2": 0.037600, + "Standard_B2ps_v2": 0.084400, + "Standard_B2pts_v2": 0.009400, + "Standard_B2s": 0.047200, + "Standard_B2s_v2": 0.094400, + "Standard_B2ts_v2": 0.021000, + "Standard_B32als_v2": 1.204000, + "Standard_B32as_v2": 1.360000, + "Standard_B32ls_v2": 1.485000, + "Standard_B32s_v2": 1.510000, + "Standard_B4als_v2": 0.151000, + "Standard_B4as_v2": 0.170000, + "Standard_B4ls_v2": 0.186000, + "Standard_B4ms": 0.189000, + "Standard_B4pls_v2": 0.152000, + "Standard_B4ps_v2": 0.150000, + "Standard_B4s_v2": 0.189000, + "Standard_B8als_v2": 0.301000, + "Standard_B8as_v2": 0.377000, + "Standard_B8ls_v2": 0.371000, + "Standard_B8ms": 0.378000, + "Standard_B8pls_v2": 0.266000, + "Standard_B8ps_v2": 0.338000, + "Standard_B8s_v2": 0.414000, + "Standard_D11_v2": 0.234000, + "Standard_D12_v2": 0.469000, + "Standard_D13_v2": 0.937000, + "Standard_D14_v2": 1.874000, + "Standard_D15_v2": 2.343000, + "Standard_D15i_v2": 2.343000, + "Standard_D16_v3": 0.896000, + "Standard_D16_v4": 0.896000, + "Standard_D16_v5": 0.896000, + "Standard_D16a_v4": 0.896000, + "Standard_D16ads_v5": 0.968000, + "Standard_D16as_v4": 0.896000, + "Standard_D16as_v5": 1.544000, + "Standard_D16d_v4": 1.056000, + "Standard_D16d_v5": 1.056000, + "Standard_D16ds_v4": 1.056000, + "Standard_D16ds_v5": 1.056000, + "Standard_D16lds_v5": 0.920000, + "Standard_D16ls_v5": 0.808000, + "Standard_D16pds_v5": 0.848000, + "Standard_D16plds_v5": 0.740000, + "Standard_D16pls_v5": 0.648000, + "Standard_D16ps_v5": 0.720000, + "Standard_D16s_v3": 0.896000, + "Standard_D16s_v4": 0.896000, + "Standard_D16s_v5": 0.896000, + "Standard_D1_v2": 0.087700, + "Standard_D2_v2": 0.175000, + "Standard_D2_v3": 0.112000, + "Standard_D2_v4": 0.112000, + "Standard_D2_v5": 0.112000, + "Standard_D2a_v4": 0.112000, + "Standard_D2ads_v5": 0.213000, + "Standard_D2as_v4": 0.112000, + "Standard_D2as_v5": 0.101000, + "Standard_D2d_v4": 0.132000, + "Standard_D2d_v5": 0.132000, + "Standard_D2ds_v4": 0.132000, + "Standard_D2ds_v5": 0.132000, + "Standard_D2lds_v5": 0.115000, + "Standard_D2ls_v5": 0.101000, + "Standard_D2pds_v5": 0.106000, + "Standard_D2plds_v5": 0.092500, + "Standard_D2pls_v5": 0.081000, + "Standard_D2ps_v5": 0.090000, + "Standard_D2s_v3": 0.112000, + "Standard_D2s_v4": 0.112000, + "Standard_D2s_v5": 0.112000, + "Standard_D32-16s_v3": 1.792000, + "Standard_D32-8s_v3": 1.792000, + "Standard_D32_v3": 1.792000, + "Standard_D32_v4": 1.792000, + "Standard_D32_v5": 1.792000, + "Standard_D32a_v4": 1.792000, + "Standard_D32ads_v5": 3.408000, + "Standard_D32as_v4": 1.792000, + "Standard_D32as_v5": 3.088000, + "Standard_D32d_v4": 2.112000, + "Standard_D32d_v5": 2.112000, + "Standard_D32ds_v4": 2.112000, + "Standard_D32ds_v5": 2.112000, + "Standard_D32lds_v5": 1.840000, + "Standard_D32ls_v5": 1.616000, + "Standard_D32pds_v5": 1.696000, + "Standard_D32plds_v5": 1.480000, + "Standard_D32pls_v5": 1.296000, + "Standard_D32ps_v5": 1.440000, + "Standard_D32s_v3": 1.792000, + "Standard_D32s_v4": 1.792000, + "Standard_D32s_v5": 1.792000, + "Standard_D3_v2": 0.351000, + "Standard_D48_v3": 2.688000, + "Standard_D48_v4": 2.688000, + "Standard_D48_v5": 2.688000, + "Standard_D48a_v4": 2.688000, + "Standard_D48ads_v5": 2.904000, + "Standard_D48as_v4": 2.688000, + "Standard_D48as_v5": 4.632000, + "Standard_D48d_v4": 3.168000, + "Standard_D48d_v5": 3.168000, + "Standard_D48ds_v4": 3.168000, + "Standard_D48ds_v5": 3.168000, + "Standard_D48lds_v5": 2.760000, + "Standard_D48ls_v5": 2.424000, + "Standard_D48pds_v5": 2.544000, + "Standard_D48plds_v5": 2.220000, + "Standard_D48pls_v5": 1.944000, + "Standard_D48ps_v5": 2.160000, + "Standard_D48s_v3": 2.688000, + "Standard_D48s_v4": 2.688000, + "Standard_D48s_v5": 2.688000, + "Standard_D4_v2": 0.702000, + "Standard_D4_v3": 0.224000, + "Standard_D4_v4": 0.224000, + "Standard_D4_v5": 0.224000, + "Standard_D4a_v4": 0.224000, + "Standard_D4ads_v5": 0.426000, + "Standard_D4as_v4": 0.224000, + "Standard_D4as_v5": 0.202000, + "Standard_D4d_v4": 0.264000, + "Standard_D4d_v5": 0.264000, + "Standard_D4ds_v4": 0.264000, + "Standard_D4ds_v5": 0.264000, + "Standard_D4lds_v5": 0.230000, + "Standard_D4ls_v5": 0.202000, + "Standard_D4pds_v5": 0.212000, + "Standard_D4plds_v5": 0.185000, + "Standard_D4pls_v5": 0.162000, + "Standard_D4ps_v5": 0.180000, + "Standard_D4s_v3": 0.224000, + "Standard_D4s_v4": 0.224000, + "Standard_D4s_v5": 0.224000, + "Standard_D5_v2": 1.404000, + "Standard_D64-16s_v3": 3.584000, + "Standard_D64-32s_v3": 3.584000, + "Standard_D64_v3": 3.584000, + "Standard_D64_v4": 3.584000, + "Standard_D64_v5": 3.584000, + "Standard_D64a_v4": 3.584000, + "Standard_D64ads_v5": 3.872000, + "Standard_D64as_v4": 3.584000, + "Standard_D64as_v5": 3.232000, + "Standard_D64d_v4": 4.224000, + "Standard_D64d_v5": 4.224000, + "Standard_D64ds_v4": 4.224000, + "Standard_D64ds_v5": 4.224000, + "Standard_D64lds_v5": 3.680000, + "Standard_D64ls_v5": 3.232000, + "Standard_D64pds_v5": 3.392000, + "Standard_D64plds_v5": 2.960000, + "Standard_D64pls_v5": 2.592000, + "Standard_D64ps_v5": 2.880000, + "Standard_D64s_v3": 3.584000, + "Standard_D64s_v4": 3.584000, + "Standard_D64s_v5": 3.584000, + "Standard_D8_v3": 0.448000, + "Standard_D8_v4": 0.448000, + "Standard_D8_v5": 0.448000, + "Standard_D8a_v4": 0.448000, + "Standard_D8ads_v5": 0.484000, + "Standard_D8as_v4": 0.448000, + "Standard_D8as_v5": 0.772000, + "Standard_D8d_v4": 0.528000, + "Standard_D8d_v5": 0.528000, + "Standard_D8ds_v4": 0.528000, + "Standard_D8ds_v5": 0.528000, + "Standard_D8lds_v5": 0.460000, + "Standard_D8ls_v5": 0.404000, + "Standard_D8pds_v5": 0.424000, + "Standard_D8plds_v5": 0.370000, + "Standard_D8pls_v5": 0.324000, + "Standard_D8ps_v5": 0.360000, + "Standard_D8s_v3": 0.448000, + "Standard_D8s_v4": 0.448000, + "Standard_D8s_v5": 0.448000, + "Standard_D96_v5": 5.376000, + "Standard_D96a_v4": 5.376000, + "Standard_D96ads_v5": 5.808000, + "Standard_D96as_v4": 5.376000, + "Standard_D96as_v5": 9.264000, + "Standard_D96d_v5": 6.336000, + "Standard_D96ds_v5": 6.336000, + "Standard_D96lds_v5": 5.520000, + "Standard_D96ls_v5": 4.848000, + "Standard_D96s_v5": 5.376000, + "Standard_DS11-1_v2": 0.234000, + "Standard_DS11_v2": 0.234000, + "Standard_DS12-1_v2": 0.469000, + "Standard_DS12-2_v2": 0.469000, + "Standard_DS12_v2": 0.469000, + "Standard_DS13-2_v2": 0.937000, + "Standard_DS13-4_v2": 0.937000, + "Standard_DS13_v2": 0.937000, + "Standard_DS14-4_v2": 1.874000, + "Standard_DS14-8_v2": 1.874000, + "Standard_DS14_v2": 1.874000, + "Standard_DS15_v2": 2.343000, + "Standard_DS15i_v2": 2.343000, + "Standard_DS1_v2": 0.087700, + "Standard_DS2_v2": 0.175000, + "Standard_DS3_v2": 0.351000, + "Standard_DS4_v2": 0.702000, + "Standard_DS5_v2": 1.404000, + "Standard_E104i_v5": 8.466000, + "Standard_E104id_v5": 9.667000, + "Standard_E104ids_v5": 9.667000, + "Standard_E104is_v5": 8.466000, + "Standard_E112iads_v5": 9.425000, + "Standard_E112ias_v5": 8.193000, + "Standard_E112ibds_v5": 12.166000, + "Standard_E112ibs_v5": 10.872000, + "Standard_E16-4ads_v5": 1.224000, + "Standard_E16-4as_v4": 1.184000, + "Standard_E16-4as_v5": 1.800000, + "Standard_E16-4ds_v4": 1.352000, + "Standard_E16-4ds_v5": 1.352000, + "Standard_E16-4s_v3": 1.248000, + "Standard_E16-4s_v4": 1.184000, + "Standard_E16-4s_v5": 1.184000, + "Standard_E16-8ads_v5": 1.960000, + "Standard_E16-8as_v4": 1.184000, + "Standard_E16-8as_v5": 1.064000, + "Standard_E16-8ds_v4": 1.352000, + "Standard_E16-8ds_v5": 1.352000, + "Standard_E16-8s_v3": 1.248000, + "Standard_E16-8s_v4": 1.184000, + "Standard_E16-8s_v5": 1.184000, + "Standard_E16_v3": 1.248000, + "Standard_E16_v4": 1.184000, + "Standard_E16_v5": 1.184000, + "Standard_E16a_v4": 1.184000, + "Standard_E16ads_v5": 1.270000, + "Standard_E16as_v4": 1.184000, + "Standard_E16as_v5": 1.110000, + "Standard_E16bds_v5": 1.580000, + "Standard_E16bs_v5": 1.412000, + "Standard_E16d_v4": 1.352000, + "Standard_E16d_v5": 1.352000, + "Standard_E16ds_v4": 1.352000, + "Standard_E16ds_v5": 1.352000, + "Standard_E16pds_v5": 1.080000, + "Standard_E16ps_v5": 0.944000, + "Standard_E16s_v3": 1.248000, + "Standard_E16s_v4": 1.184000, + "Standard_E16s_v5": 1.184000, + "Standard_E20_v3": 1.560000, + "Standard_E20_v4": 1.480000, + "Standard_E20_v5": 1.480000, + "Standard_E20a_v4": 1.480000, + "Standard_E20ads_v5": 1.576000, + "Standard_E20as_v4": 1.480000, + "Standard_E20as_v5": 1.376000, + "Standard_E20d_v4": 1.690000, + "Standard_E20d_v5": 1.690000, + "Standard_E20ds_v4": 1.690000, + "Standard_E20ds_v5": 1.690000, + "Standard_E20pds_v5": 1.350000, + "Standard_E20ps_v5": 1.180000, + "Standard_E20s_v3": 1.560000, + "Standard_E20s_v4": 1.480000, + "Standard_E20s_v5": 1.480000, + "Standard_E2_v3": 0.156000, + "Standard_E2_v4": 0.148000, + "Standard_E2_v5": 0.148000, + "Standard_E2a_v4": 0.148000, + "Standard_E2ads_v5": 0.153000, + "Standard_E2as_v4": 0.148000, + "Standard_E2as_v5": 0.179000, + "Standard_E2bds_v5": 0.198000, + "Standard_E2bs_v5": 0.177000, + "Standard_E2d_v4": 0.169000, + "Standard_E2d_v5": 0.169000, + "Standard_E2ds_v4": 0.169000, + "Standard_E2ds_v5": 0.169000, + "Standard_E2pds_v5": 0.135000, + "Standard_E2ps_v5": 0.118000, + "Standard_E2s_v3": 0.156000, + "Standard_E2s_v4": 0.148000, + "Standard_E2s_v5": 0.148000, + "Standard_E32-16ads_v5": 2.448000, + "Standard_E32-16as_v4": 2.368000, + "Standard_E32-16as_v5": 2.128000, + "Standard_E32-16ds_v4": 2.704000, + "Standard_E32-16ds_v5": 2.704000, + "Standard_E32-16s_v3": 2.496000, + "Standard_E32-16s_v4": 2.368000, + "Standard_E32-16s_v5": 2.368000, + "Standard_E32-8ads_v5": 3.920000, + "Standard_E32-8as_v4": 2.368000, + "Standard_E32-8as_v5": 3.600000, + "Standard_E32-8ds_v4": 2.704000, + "Standard_E32-8ds_v5": 2.704000, + "Standard_E32-8s_v3": 2.496000, + "Standard_E32-8s_v4": 2.368000, + "Standard_E32-8s_v5": 2.368000, + "Standard_E32_v3": 2.496000, + "Standard_E32_v4": 2.368000, + "Standard_E32_v5": 2.368000, + "Standard_E32a_v4": 2.368000, + "Standard_E32ads_v5": 2.448000, + "Standard_E32as_v4": 2.368000, + "Standard_E32as_v5": 2.174000, + "Standard_E32bds_v5": 3.160000, + "Standard_E32bs_v5": 2.824000, + "Standard_E32d_v4": 2.704000, + "Standard_E32d_v5": 2.704000, + "Standard_E32ds_v4": 2.704000, + "Standard_E32ds_v5": 2.704000, + "Standard_E32pds_v5": 2.160000, + "Standard_E32ps_v5": 1.888000, + "Standard_E32s_v3": 2.496000, + "Standard_E32s_v4": 2.368000, + "Standard_E32s_v5": 2.368000, + "Standard_E4-2ads_v5": 0.306000, + "Standard_E4-2as_v4": 0.296000, + "Standard_E4-2as_v5": 0.450000, + "Standard_E4-2ds_v4": 0.338000, + "Standard_E4-2ds_v5": 0.338000, + "Standard_E4-2s_v3": 0.312000, + "Standard_E4-2s_v4": 0.296000, + "Standard_E4-2s_v5": 0.296000, + "Standard_E48_v3": 3.740000, + "Standard_E48_v4": 3.552000, + "Standard_E48_v5": 3.552000, + "Standard_E48a_v4": 3.552000, + "Standard_E48ads_v5": 3.672000, + "Standard_E48as_v4": 3.552000, + "Standard_E48as_v5": 3.238000, + "Standard_E48bds_v5": 4.740000, + "Standard_E48bs_v5": 4.236000, + "Standard_E48d_v4": 4.056000, + "Standard_E48d_v5": 4.056000, + "Standard_E48ds_v4": 4.056000, + "Standard_E48ds_v5": 4.056000, + "Standard_E48s_v3": 3.740000, + "Standard_E48s_v4": 3.552000, + "Standard_E48s_v5": 3.552000, + "Standard_E4_v3": 0.312000, + "Standard_E4_v4": 0.296000, + "Standard_E4_v5": 0.296000, + "Standard_E4a_v4": 0.296000, + "Standard_E4ads_v5": 0.352000, + "Standard_E4as_v4": 0.296000, + "Standard_E4as_v5": 0.312000, + "Standard_E4bds_v5": 0.395000, + "Standard_E4bs_v5": 0.353000, + "Standard_E4d_v4": 0.338000, + "Standard_E4d_v5": 0.338000, + "Standard_E4ds_v4": 0.338000, + "Standard_E4ds_v5": 0.338000, + "Standard_E4pds_v5": 0.270000, + "Standard_E4ps_v5": 0.236000, + "Standard_E4s_v3": 0.312000, + "Standard_E4s_v4": 0.296000, + "Standard_E4s_v5": 0.296000, + "Standard_E64-16ads_v5": 4.896000, + "Standard_E64-16as_v4": 4.736000, + "Standard_E64-16as_v5": 7.200000, + "Standard_E64-16ds_v4": 5.408000, + "Standard_E64-16ds_v5": 5.408000, + "Standard_E64-16s_v3": 4.488000, + "Standard_E64-16s_v4": 4.736000, + "Standard_E64-16s_v5": 4.736000, + "Standard_E64-32ads_v5": 7.840000, + "Standard_E64-32as_v4": 4.736000, + "Standard_E64-32as_v5": 7.200000, + "Standard_E64-32ds_v4": 5.408000, + "Standard_E64-32ds_v5": 5.408000, + "Standard_E64-32s_v3": 4.488000, + "Standard_E64-32s_v4": 4.736000, + "Standard_E64-32s_v5": 4.736000, + "Standard_E64_v3": 4.488000, + "Standard_E64_v4": 4.736000, + "Standard_E64_v5": 4.736000, + "Standard_E64a_v4": 4.736000, + "Standard_E64ads_v5": 4.896000, + "Standard_E64as_v4": 4.736000, + "Standard_E64as_v5": 4.256000, + "Standard_E64bds_v5": 6.320000, + "Standard_E64bs_v5": 5.648000, + "Standard_E64d_v4": 5.408000, + "Standard_E64d_v5": 5.408000, + "Standard_E64ds_v4": 5.408000, + "Standard_E64ds_v5": 5.408000, + "Standard_E64i_v3": 4.488000, + "Standard_E64is_v3": 4.488000, + "Standard_E64s_v3": 4.488000, + "Standard_E64s_v4": 4.736000, + "Standard_E64s_v5": 4.736000, + "Standard_E8-2ads_v5": 0.612000, + "Standard_E8-2as_v4": 0.592000, + "Standard_E8-2as_v5": 0.900000, + "Standard_E8-2ds_v4": 0.676000, + "Standard_E8-2ds_v5": 0.676000, + "Standard_E8-2s_v3": 0.624000, + "Standard_E8-2s_v4": 0.592000, + "Standard_E8-2s_v5": 0.592000, + "Standard_E8-4ads_v5": 0.612000, + "Standard_E8-4as_v4": 0.592000, + "Standard_E8-4as_v5": 0.532000, + "Standard_E8-4ds_v4": 0.676000, + "Standard_E8-4ds_v5": 0.676000, + "Standard_E8-4s_v3": 0.624000, + "Standard_E8-4s_v4": 0.592000, + "Standard_E8-4s_v5": 0.592000, + "Standard_E80ids_v4": 6.760000, + "Standard_E80is_v4": 5.920000, + "Standard_E8_v3": 0.624000, + "Standard_E8_v4": 0.592000, + "Standard_E8_v5": 0.592000, + "Standard_E8a_v4": 0.592000, + "Standard_E8ads_v5": 0.612000, + "Standard_E8as_v4": 0.592000, + "Standard_E8as_v5": 0.578000, + "Standard_E8bds_v5": 0.790000, + "Standard_E8bs_v5": 0.706000, + "Standard_E8d_v4": 0.676000, + "Standard_E8d_v5": 0.676000, + "Standard_E8ds_v4": 0.676000, + "Standard_E8ds_v5": 0.676000, + "Standard_E8pds_v5": 0.540000, + "Standard_E8ps_v5": 0.472000, + "Standard_E8s_v3": 0.624000, + "Standard_E8s_v4": 0.592000, + "Standard_E8s_v5": 0.592000, + "Standard_E96-24ads_v5": 11.760000, + "Standard_E96-24as_v4": 7.104000, + "Standard_E96-24as_v5": 6.384000, + "Standard_E96-24ds_v5": 8.112000, + "Standard_E96-24s_v5": 7.104000, + "Standard_E96-48ads_v5": 11.760000, + "Standard_E96-48as_v4": 7.104000, + "Standard_E96-48as_v5": 10.800000, + "Standard_E96-48ds_v5": 8.112000, + "Standard_E96-48s_v5": 7.104000, + "Standard_E96_v5": 7.104000, + "Standard_E96a_v4": 7.104000, + "Standard_E96ads_v5": 7.344000, + "Standard_E96as_v4": 7.104000, + "Standard_E96as_v5": 6.430000, + "Standard_E96bds_v5": 9.480000, + "Standard_E96bs_v5": 8.472000, + "Standard_E96d_v5": 8.112000, + "Standard_E96ds_v5": 8.112000, + "Standard_E96iads_v5": 8.078000, + "Standard_E96ias_v4": 7.104000, + "Standard_E96ias_v5": 7.022000, + "Standard_E96s_v5": 7.104000, + "Standard_F1": 0.050000, + "Standard_F16": 0.808000, + "Standard_F16s": 0.808000, + "Standard_F16s_v2": 0.808000, + "Standard_F1s": 0.050000, + "Standard_F2": 0.101000, + "Standard_F2s": 0.101000, + "Standard_F2s_v2": 0.101000, + "Standard_F32s_v2": 1.616000, + "Standard_F4": 0.202000, + "Standard_F48s_v2": 2.424000, + "Standard_F4s": 0.202000, + "Standard_F4s_v2": 0.202000, + "Standard_F64s_v2": 3.232000, + "Standard_F72s_v2": 3.636000, + "Standard_F8": 0.404000, + "Standard_F8s": 0.404000, + "Standard_F8s_v2": 0.404000, + "Standard_L112ias_v3": 11.150000, + "Standard_L16as_v3": 1.448000, + "Standard_L16s_v2": 1.448000, + "Standard_L16s_v3": 1.616000, + "Standard_L32as_v3": 2.896000, + "Standard_L32s_v2": 2.896000, + "Standard_L32s_v3": 3.232000, + "Standard_L48as_v3": 4.344000, + "Standard_L48s_v2": 4.344000, + "Standard_L48s_v3": 4.848000, + "Standard_L64as_v3": 5.792000, + "Standard_L64s_v2": 5.792000, + "Standard_L64s_v3": 6.464000, + "Standard_L80as_v3": 7.240000, + "Standard_L80s_v2": 7.240000, + "Standard_L80s_v3": 8.080000, + "Standard_L8as_v3": 0.724000, + "Standard_L8s_v2": 0.724000, + "Standard_L8s_v3": 0.808000, + "Standard_L96ias_v3": 9.557000, + "Standard_M128": 16.806000, + "Standard_M128-32ms": 33.627000, + "Standard_M128-64ms": 33.627000, + "Standard_M128dms_v2": 33.629000, + "Standard_M128ds_v2": 16.808000, + "Standard_M128m": 33.627000, + "Standard_M128ms": 33.627000, + "Standard_M128ms_v2": 33.099000, + "Standard_M128s": 16.806000, + "Standard_M128s_v2": 16.278000, + "Standard_M16-4ms": 3.872000, + "Standard_M16-8ms": 3.872000, + "Standard_M16ms": 3.872000, + "Standard_M16s": 3.008000, + "Standard_M192idms_v2": 40.400000, + "Standard_M192ids_v2": 20.200000, + "Standard_M192ims_v2": 39.870000, + "Standard_M192is_v2": 19.670000, + "Standard_M32-16ms": 7.744000, + "Standard_M32-8ms": 7.744000, + "Standard_M32dms_v2": 7.744000, + "Standard_M32ls": 3.370000, + "Standard_M32ms": 7.744000, + "Standard_M32ms_v2": 7.611000, + "Standard_M32s": 4.202000, + "Standard_M32ts": 3.175000, + "Standard_M64": 8.403000, + "Standard_M64-16ms": 13.025000, + "Standard_M64-32ms": 13.025000, + "Standard_M64dms_v2": 13.028000, + "Standard_M64ds_v2": 8.403000, + "Standard_M64ls": 6.351000, + "Standard_M64m": 13.025000, + "Standard_M64ms": 13.025000, + "Standard_M64ms_v2": 12.763000, + "Standard_M64s": 8.403000, + "Standard_M64s_v2": 8.138000, + "Standard_M8-2ms": 1.936000, + "Standard_M8-4ms": 1.936000, + "Standard_M8ms": 1.936000, + "Standard_NC12s_v3": 7.160000, + "Standard_NC24ads_A100_v4": 4.591000, + "Standard_NC24rs_v3": 15.753000, + "Standard_NC24s_v3": 14.321000, + "Standard_NC48ads_A100_v4": 9.183000, + "Standard_NC6s_v3": 3.580000, + "Standard_NC96ads_A100_v4": 18.365000, + "Standard_ND96ams_A100_v4": 40.962000, + "Standard_ND96amsr_A100_v4": 40.962000, + "Standard_NV12ads_A10_v5": 1.135000, + "Standard_NV12s_v3": 1.425000, + "Standard_NV18ads_A10_v5": 2.000000, + "Standard_NV24s_v3": 2.850000, + "Standard_NV36adms_A10_v5": 5.650000, + "Standard_NV36ads_A10_v5": 4.000000, + "Standard_NV48s_v3": 5.700000, + "Standard_NV6ads_A10_v5": 0.567000, + "Standard_NV72ads_A10_v5": 8.150000, + } + // francesouth + initialOnDemandPrices["francesouth"] = map[string]float64{ + "Basic_A0": 0.026000, + "Basic_A1": 0.033800, + "Basic_A2": 0.131300, + "Basic_A3": 0.291200, + "Basic_A4": 0.583700, + "Dadsv5_Type1": 9.690000, + "Dasv4_Type1": 8.008000, + "Dasv4_Type2": 8.008000, + "Dasv5_Type1": 8.088000, + "Ddsv4_Type 1": 6.046000, + "Ddsv4_Type2": 7.180000, + "Ddsv5_Type1": 9.060000, + "Dsv3_Type3": 6.422000, + "Dsv3_Type4": 8.028000, + "Dsv4_Type1": 6.420000, + "Dsv4_Type2": 8.026000, + "Dsv5_Type1": 8.008000, + "Eadsv5_Type1": 10.502000, + "Easv4_Type1": 10.159000, + "Easv4_Type2": 10.159000, + "Easv5_Type1": 9.129000, + "Ebdsv5-Type1": 9.038000, + "Ebsv5-Type1": 8.077000, + "Edsv4_Type 1": 7.705000, + "Edsv4_Type2": 9.149000, + "Edsv5_Type1": 11.600000, + "Esv3_Type3": 6.531000, + "Esv3_Type4": 8.849000, + "Esv4_Type1": 6.742000, + "Esv4_Type2": 8.848000, + "Esv5_Type1": 10.159000, + "Fsv2 Type3": 6.538000, + "Fsv2_Type2": 5.116000, + "Fsv2_Type4": 7.106000, + "Lasv3_Type1": 10.353000, + "Lsv3_Type1": 11.554000, + "Standard_A0": 0.028600, + "Standard_A1": 0.085800, + "Standard_A1_v2": 0.065000, + "Standard_A2": 0.171600, + "Standard_A2_v2": 0.137800, + "Standard_A2m_v2": 0.231400, + "Standard_A3": 0.343200, + "Standard_A4": 0.686400, + "Standard_A4_v2": 0.288600, + "Standard_A4m_v2": 0.486200, + "Standard_A5": 0.386100, + "Standard_A6": 0.772200, + "Standard_A7": 1.544400, + "Standard_A8_v2": 0.607100, + "Standard_A8m_v2": 1.021800, + "Standard_B12ms": 0.811000, + "Standard_B16als_v2": 0.783000, + "Standard_B16as_v2": 0.958000, + "Standard_B16ls_v2": 0.943000, + "Standard_B16ms": 1.082000, + "Standard_B16s_v2": 1.055000, + "Standard_B1ls": 0.008450, + "Standard_B1ms": 0.033800, + "Standard_B1s": 0.016900, + "Standard_B20ms": 1.352000, + "Standard_B2als_v2": 0.055300, + "Standard_B2as_v2": 0.120000, + "Standard_B2ats_v2": 0.013800, + "Standard_B2ls_v2": 0.070600, + "Standard_B2ms": 0.135000, + "Standard_B2s": 0.067600, + "Standard_B2s_v2": 0.123000, + "Standard_B2ts_v2": 0.024500, + "Standard_B32als_v2": 1.713000, + "Standard_B32as_v2": 1.768000, + "Standard_B32ls_v2": 1.739000, + "Standard_B32s_v2": 1.964000, + "Standard_B4als_v2": 0.196000, + "Standard_B4as_v2": 0.239000, + "Standard_B4ls_v2": 0.217000, + "Standard_B4ms": 0.270000, + "Standard_B4s_v2": 0.245000, + "Standard_B8als_v2": 0.428000, + "Standard_B8as_v2": 0.442000, + "Standard_B8ls_v2": 0.472000, + "Standard_B8ms": 0.541000, + "Standard_B8s_v2": 0.528000, + "Standard_D11_v2": 0.304000, + "Standard_D12_v2": 0.608000, + "Standard_D13_v2": 1.217000, + "Standard_D14_v2": 2.434000, + "Standard_D15_v2": 3.042000, + "Standard_D15i_v2": 3.042000, + "Standard_D16_v3": 1.164800, + "Standard_D16_v4": 1.167000, + "Standard_D16_v5": 1.165000, + "Standard_D16a_v4": 1.165000, + "Standard_D16ads_v5": 1.994000, + "Standard_D16as_v4": 1.165000, + "Standard_D16as_v5": 1.050000, + "Standard_D16d_v4": 1.374000, + "Standard_D16d_v5": 1.373000, + "Standard_D16ds_v4": 1.374000, + "Standard_D16ds_v5": 1.373000, + "Standard_D16lds_v5": 1.196000, + "Standard_D16ls_v5": 1.050000, + "Standard_D16s_v3": 1.164800, + "Standard_D16s_v4": 1.167000, + "Standard_D16s_v5": 1.165000, + "Standard_D1_v2": 0.114000, + "Standard_D2_v2": 0.228000, + "Standard_D2_v3": 0.145600, + "Standard_D2_v4": 0.146000, + "Standard_D2_v5": 0.146000, + "Standard_D2a_v4": 0.146000, + "Standard_D2ads_v5": 0.157000, + "Standard_D2as_v4": 0.146000, + "Standard_D2as_v5": 0.223000, + "Standard_D2d_v4": 0.172000, + "Standard_D2d_v5": 0.172000, + "Standard_D2ds_v4": 0.172000, + "Standard_D2ds_v5": 0.172000, + "Standard_D2lds_v5": 0.150000, + "Standard_D2ls_v5": 0.131000, + "Standard_D2s_v3": 0.145600, + "Standard_D2s_v4": 0.146000, + "Standard_D2s_v5": 0.146000, + "Standard_D32-16s_v3": 2.329600, + "Standard_D32-8s_v3": 2.329600, + "Standard_D32_v3": 2.329600, + "Standard_D32_v4": 2.335000, + "Standard_D32_v5": 2.330000, + "Standard_D32a_v4": 2.330000, + "Standard_D32ads_v5": 3.989000, + "Standard_D32as_v4": 2.330000, + "Standard_D32as_v5": 3.573000, + "Standard_D32d_v4": 2.748000, + "Standard_D32d_v5": 2.746000, + "Standard_D32ds_v4": 2.748000, + "Standard_D32ds_v5": 2.746000, + "Standard_D32lds_v5": 2.392000, + "Standard_D32ls_v5": 2.101000, + "Standard_D32s_v3": 2.329600, + "Standard_D32s_v4": 2.335000, + "Standard_D32s_v5": 2.330000, + "Standard_D3_v2": 0.456000, + "Standard_D48_v3": 3.494000, + "Standard_D48_v4": 3.502000, + "Standard_D48_v5": 3.494000, + "Standard_D48a_v4": 3.494000, + "Standard_D48ads_v5": 3.775000, + "Standard_D48as_v4": 3.494000, + "Standard_D48as_v5": 3.151000, + "Standard_D48d_v4": 4.122000, + "Standard_D48d_v5": 4.118000, + "Standard_D48ds_v4": 4.122000, + "Standard_D48ds_v5": 4.118000, + "Standard_D48lds_v5": 3.588000, + "Standard_D48ls_v5": 3.151000, + "Standard_D48s_v3": 3.494000, + "Standard_D48s_v4": 3.502000, + "Standard_D48s_v5": 3.494000, + "Standard_D4_v2": 0.912000, + "Standard_D4_v3": 0.291000, + "Standard_D4_v4": 0.292000, + "Standard_D4_v5": 0.291000, + "Standard_D4a_v4": 0.291000, + "Standard_D4ads_v5": 0.499000, + "Standard_D4as_v4": 0.291000, + "Standard_D4as_v5": 0.263000, + "Standard_D4d_v4": 0.344000, + "Standard_D4d_v5": 0.343000, + "Standard_D4ds_v4": 0.344000, + "Standard_D4ds_v5": 0.343000, + "Standard_D4lds_v5": 0.299000, + "Standard_D4ls_v5": 0.263000, + "Standard_D4s_v3": 0.291000, + "Standard_D4s_v4": 0.292000, + "Standard_D4s_v5": 0.291000, + "Standard_D5_v2": 1.824000, + "Standard_D64-16s_v3": 4.659000, + "Standard_D64-32s_v3": 4.659000, + "Standard_D64_v3": 4.659000, + "Standard_D64_v4": 4.669000, + "Standard_D64_v5": 4.659000, + "Standard_D64a_v4": 4.659000, + "Standard_D64ads_v5": 5.034000, + "Standard_D64as_v4": 4.659000, + "Standard_D64as_v5": 7.146000, + "Standard_D64d_v4": 5.496000, + "Standard_D64d_v5": 5.491000, + "Standard_D64ds_v4": 5.496000, + "Standard_D64ds_v5": 5.491000, + "Standard_D64lds_v5": 4.784000, + "Standard_D64ls_v5": 4.202000, + "Standard_D64s_v3": 4.659000, + "Standard_D64s_v4": 4.669000, + "Standard_D64s_v5": 4.659000, + "Standard_D8_v3": 0.582000, + "Standard_D8_v4": 0.584000, + "Standard_D8_v5": 0.582000, + "Standard_D8a_v4": 0.582000, + "Standard_D8ads_v5": 0.997000, + "Standard_D8as_v4": 0.582000, + "Standard_D8as_v5": 0.893000, + "Standard_D8d_v4": 0.687000, + "Standard_D8d_v5": 0.686000, + "Standard_D8ds_v4": 0.687000, + "Standard_D8ds_v5": 0.686000, + "Standard_D8lds_v5": 0.598000, + "Standard_D8ls_v5": 0.525000, + "Standard_D8s_v3": 0.582000, + "Standard_D8s_v4": 0.584000, + "Standard_D8s_v5": 0.582000, + "Standard_D96_v5": 6.989000, + "Standard_D96a_v4": 6.989000, + "Standard_D96ads_v5": 7.550000, + "Standard_D96as_v4": 6.989000, + "Standard_D96as_v5": 10.718000, + "Standard_D96d_v5": 8.237000, + "Standard_D96ds_v5": 8.237000, + "Standard_D96lds_v5": 7.176000, + "Standard_D96ls_v5": 6.302000, + "Standard_D96s_v5": 6.989000, + "Standard_DS11-1_v2": 0.304000, + "Standard_DS11_v2": 0.304000, + "Standard_DS12-1_v2": 0.608000, + "Standard_DS12-2_v2": 0.608000, + "Standard_DS12_v2": 0.608000, + "Standard_DS13-2_v2": 1.217000, + "Standard_DS13-4_v2": 1.217000, + "Standard_DS13_v2": 1.217000, + "Standard_DS14-4_v2": 2.434000, + "Standard_DS14-8_v2": 2.434000, + "Standard_DS14_v2": 2.434000, + "Standard_DS15_v2": 3.042000, + "Standard_DS15i_v2": 3.042000, + "Standard_DS1_v2": 0.114000, + "Standard_DS2_v2": 0.228000, + "Standard_DS3_v2": 0.456000, + "Standard_DS4_v2": 0.912000, + "Standard_DS5_v2": 1.824000, + "Standard_E104i_v5": 11.005000, + "Standard_E104id_v5": 12.567000, + "Standard_E104ids_v5": 12.567000, + "Standard_E104is_v5": 11.005000, + "Standard_E112iads_v5": 12.252000, + "Standard_E112ias_v5": 10.651000, + "Standard_E112ibds_v5": 15.816000, + "Standard_E112ibs_v5": 14.134000, + "Standard_E16-4ads_v5": 1.591000, + "Standard_E16-4as_v4": 1.539000, + "Standard_E16-4as_v5": 1.383000, + "Standard_E16-4ds_v4": 1.751000, + "Standard_E16-4ds_v5": 1.758000, + "Standard_E16-4s_v3": 1.621000, + "Standard_E16-4s_v4": 1.532000, + "Standard_E16-4s_v5": 1.539000, + "Standard_E16-8ads_v5": 2.327000, + "Standard_E16-8as_v4": 1.539000, + "Standard_E16-8as_v5": 2.119000, + "Standard_E16-8ds_v4": 1.751000, + "Standard_E16-8ds_v5": 1.758000, + "Standard_E16-8s_v3": 1.621000, + "Standard_E16-8s_v4": 1.532000, + "Standard_E16-8s_v5": 1.539000, + "Standard_E16_v3": 1.621000, + "Standard_E16_v4": 1.532000, + "Standard_E16_v5": 1.539000, + "Standard_E16a_v4": 1.539000, + "Standard_E16ads_v5": 1.637000, + "Standard_E16as_v4": 1.539000, + "Standard_E16as_v5": 1.429000, + "Standard_E16bds_v5": 2.054000, + "Standard_E16bs_v5": 1.836000, + "Standard_E16d_v4": 1.751000, + "Standard_E16d_v5": 1.758000, + "Standard_E16ds_v4": 1.751000, + "Standard_E16ds_v5": 1.758000, + "Standard_E16s_v3": 1.621000, + "Standard_E16s_v4": 1.532000, + "Standard_E16s_v5": 1.539000, + "Standard_E20_v3": 2.028000, + "Standard_E20_v4": 1.915000, + "Standard_E20_v5": 1.924000, + "Standard_E20a_v4": 1.924000, + "Standard_E20ads_v5": 1.989000, + "Standard_E20as_v4": 1.924000, + "Standard_E20as_v5": 1.729000, + "Standard_E20d_v4": 2.189000, + "Standard_E20d_v5": 2.197000, + "Standard_E20ds_v4": 2.189000, + "Standard_E20ds_v5": 2.197000, + "Standard_E20s_v3": 2.028000, + "Standard_E20s_v4": 1.915000, + "Standard_E20s_v5": 1.924000, + "Standard_E2_v3": 0.202800, + "Standard_E2_v4": 0.192000, + "Standard_E2_v5": 0.192000, + "Standard_E2a_v4": 0.192000, + "Standard_E2ads_v5": 0.199000, + "Standard_E2as_v4": 0.192000, + "Standard_E2as_v5": 0.173000, + "Standard_E2bds_v5": 0.257000, + "Standard_E2bs_v5": 0.229000, + "Standard_E2d_v4": 0.219000, + "Standard_E2d_v5": 0.220000, + "Standard_E2ds_v4": 0.219000, + "Standard_E2ds_v5": 0.220000, + "Standard_E2s_v3": 0.202800, + "Standard_E2s_v4": 0.192000, + "Standard_E2s_v5": 0.192000, + "Standard_E32-16ads_v5": 3.182000, + "Standard_E32-16as_v4": 3.078000, + "Standard_E32-16as_v5": 2.766000, + "Standard_E32-16ds_v4": 3.502000, + "Standard_E32-16ds_v5": 3.515000, + "Standard_E32-16s_v3": 3.244800, + "Standard_E32-16s_v4": 3.064000, + "Standard_E32-16s_v5": 3.078000, + "Standard_E32-8ads_v5": 3.182000, + "Standard_E32-8as_v4": 3.078000, + "Standard_E32-8as_v5": 4.238000, + "Standard_E32-8ds_v4": 3.502000, + "Standard_E32-8ds_v5": 3.515000, + "Standard_E32-8s_v3": 3.244800, + "Standard_E32-8s_v4": 3.064000, + "Standard_E32-8s_v5": 3.078000, + "Standard_E32_v3": 3.244800, + "Standard_E32_v4": 3.064000, + "Standard_E32_v5": 3.078000, + "Standard_E32a_v4": 3.078000, + "Standard_E32ads_v5": 3.182000, + "Standard_E32as_v4": 3.078000, + "Standard_E32as_v5": 2.766000, + "Standard_E32bds_v5": 4.108000, + "Standard_E32bs_v5": 3.671000, + "Standard_E32d_v4": 3.502000, + "Standard_E32d_v5": 3.515000, + "Standard_E32ds_v4": 3.502000, + "Standard_E32ds_v5": 3.515000, + "Standard_E32s_v3": 3.244800, + "Standard_E32s_v4": 3.064000, + "Standard_E32s_v5": 3.078000, + "Standard_E4-2ads_v5": 0.582000, + "Standard_E4-2as_v4": 0.385000, + "Standard_E4-2as_v5": 0.530000, + "Standard_E4-2ds_v4": 0.438000, + "Standard_E4-2ds_v5": 0.439000, + "Standard_E4-2s_v3": 0.405600, + "Standard_E4-2s_v4": 0.383000, + "Standard_E4-2s_v5": 0.385000, + "Standard_E48_v3": 4.863000, + "Standard_E48_v4": 4.596000, + "Standard_E48_v5": 4.618000, + "Standard_E48a_v4": 4.618000, + "Standard_E48ads_v5": 4.774000, + "Standard_E48as_v4": 4.618000, + "Standard_E48as_v5": 4.196000, + "Standard_E48bds_v5": 6.162000, + "Standard_E48bs_v5": 5.507000, + "Standard_E48d_v4": 5.253000, + "Standard_E48d_v5": 5.273000, + "Standard_E48ds_v4": 5.253000, + "Standard_E48ds_v5": 5.273000, + "Standard_E48s_v3": 4.863000, + "Standard_E48s_v4": 4.596000, + "Standard_E48s_v5": 4.618000, + "Standard_E4_v3": 0.405600, + "Standard_E4_v4": 0.383000, + "Standard_E4_v5": 0.385000, + "Standard_E4a_v4": 0.385000, + "Standard_E4ads_v5": 0.444000, + "Standard_E4as_v4": 0.385000, + "Standard_E4as_v5": 0.346000, + "Standard_E4bds_v5": 0.514000, + "Standard_E4bs_v5": 0.459000, + "Standard_E4d_v4": 0.438000, + "Standard_E4d_v5": 0.439000, + "Standard_E4ds_v4": 0.438000, + "Standard_E4ds_v5": 0.439000, + "Standard_E4s_v3": 0.405600, + "Standard_E4s_v4": 0.383000, + "Standard_E4s_v5": 0.385000, + "Standard_E64-16ads_v5": 9.309000, + "Standard_E64-16as_v4": 6.157000, + "Standard_E64-16as_v5": 5.533000, + "Standard_E64-16ds_v4": 7.004000, + "Standard_E64-16ds_v5": 7.030000, + "Standard_E64-16s_v3": 5.835000, + "Standard_E64-16s_v4": 6.129000, + "Standard_E64-16s_v5": 6.157000, + "Standard_E64-32ads_v5": 9.309000, + "Standard_E64-32as_v4": 6.157000, + "Standard_E64-32as_v5": 8.477000, + "Standard_E64-32ds_v4": 7.004000, + "Standard_E64-32ds_v5": 7.030000, + "Standard_E64-32s_v3": 5.835000, + "Standard_E64-32s_v4": 6.129000, + "Standard_E64-32s_v5": 6.157000, + "Standard_E64_v3": 5.835000, + "Standard_E64_v4": 6.129000, + "Standard_E64_v5": 6.157000, + "Standard_E64a_v4": 6.157000, + "Standard_E64ads_v5": 6.411000, + "Standard_E64as_v4": 6.157000, + "Standard_E64as_v5": 5.579000, + "Standard_E64bds_v5": 8.216000, + "Standard_E64bs_v5": 7.342000, + "Standard_E64d_v4": 7.004000, + "Standard_E64d_v5": 7.030000, + "Standard_E64ds_v4": 7.004000, + "Standard_E64ds_v5": 7.030000, + "Standard_E64i_v3": 5.835000, + "Standard_E64is_v3": 5.835000, + "Standard_E64s_v3": 5.835000, + "Standard_E64s_v4": 6.129000, + "Standard_E64s_v5": 6.157000, + "Standard_E8-2ads_v5": 1.164000, + "Standard_E8-2as_v4": 0.770000, + "Standard_E8-2as_v5": 0.692000, + "Standard_E8-2ds_v4": 0.876000, + "Standard_E8-2ds_v5": 0.879000, + "Standard_E8-2s_v3": 0.810000, + "Standard_E8-2s_v4": 0.766000, + "Standard_E8-2s_v5": 0.770000, + "Standard_E8-4ads_v5": 0.796000, + "Standard_E8-4as_v4": 0.770000, + "Standard_E8-4as_v5": 0.692000, + "Standard_E8-4ds_v4": 0.876000, + "Standard_E8-4ds_v5": 0.879000, + "Standard_E8-4s_v3": 0.810000, + "Standard_E8-4s_v4": 0.766000, + "Standard_E8-4s_v5": 0.770000, + "Standard_E80ids_v4": 8.755000, + "Standard_E80is_v4": 7.661000, + "Standard_E8_v3": 0.810000, + "Standard_E8_v4": 0.766000, + "Standard_E8_v5": 0.770000, + "Standard_E8a_v4": 0.770000, + "Standard_E8ads_v5": 0.842000, + "Standard_E8as_v4": 0.770000, + "Standard_E8as_v5": 0.738000, + "Standard_E8bds_v5": 1.027000, + "Standard_E8bs_v5": 0.918000, + "Standard_E8d_v4": 0.876000, + "Standard_E8d_v5": 0.879000, + "Standard_E8ds_v4": 0.876000, + "Standard_E8ds_v5": 0.879000, + "Standard_E8s_v3": 0.810000, + "Standard_E8s_v4": 0.766000, + "Standard_E8s_v5": 0.770000, + "Standard_E96-24ads_v5": 9.547000, + "Standard_E96-24as_v4": 9.235000, + "Standard_E96-24as_v5": 8.299000, + "Standard_E96-24ds_v5": 10.546000, + "Standard_E96-24s_v5": 9.235000, + "Standard_E96-48ads_v5": 9.547000, + "Standard_E96-48as_v4": 9.235000, + "Standard_E96-48as_v5": 8.299000, + "Standard_E96-48ds_v5": 10.546000, + "Standard_E96-48s_v5": 9.235000, + "Standard_E96_v5": 9.235000, + "Standard_E96a_v4": 9.235000, + "Standard_E96ads_v5": 9.547000, + "Standard_E96as_v4": 9.235000, + "Standard_E96as_v5": 8.299000, + "Standard_E96bds_v5": 12.324000, + "Standard_E96bs_v5": 11.014000, + "Standard_E96d_v5": 10.546000, + "Standard_E96ds_v5": 10.546000, + "Standard_E96iads_v5": 10.502000, + "Standard_E96ias_v4": 9.235000, + "Standard_E96ias_v5": 9.129000, + "Standard_E96s_v5": 9.235000, + "Standard_F1": 0.065600, + "Standard_F16": 1.050000, + "Standard_F16s": 1.050000, + "Standard_F16s_v2": 1.047000, + "Standard_F1s": 0.065600, + "Standard_F2": 0.131000, + "Standard_F2s": 0.131000, + "Standard_F2s_v2": 0.131000, + "Standard_F32s_v2": 2.094000, + "Standard_F4": 0.262000, + "Standard_F48s_v2": 3.142000, + "Standard_F4s": 0.262000, + "Standard_F4s_v2": 0.262000, + "Standard_F64s_v2": 4.189000, + "Standard_F72s_v2": 4.712000, + "Standard_F8": 0.525000, + "Standard_F8s": 0.525000, + "Standard_F8s_v2": 0.524000, + "Standard_L16as_v3": 1.882000, + "Standard_L16s_v3": 2.101000, + "Standard_L32as_v3": 3.765000, + "Standard_L32s_v3": 4.202000, + "Standard_L48as_v3": 5.647000, + "Standard_L48s_v3": 6.302000, + "Standard_L64as_v3": 7.530000, + "Standard_L64s_v3": 8.403000, + "Standard_L80as_v3": 9.412000, + "Standard_L80s_v3": 10.504000, + "Standard_L8as_v3": 0.941000, + "Standard_L8s_v3": 1.050000, + } + // germanynorth + initialOnDemandPrices["germanynorth"] = map[string]float64{ + "DCdsv3 Type1": 4.668000, + "DCsv3 Type1": 3.947000, + "Dadsv5_Type1": 10.010000, + "Dasv4_Type1": 8.222000, + "Dasv4_Type2": 8.222000, + "Dasv5_Type1": 8.328000, + "Ddsv5_Type1": 9.335000, + "Dsv3_Type3": 6.549000, + "Dsv3_Type4": 8.186000, + "Dsv5_Type1": 8.223000, + "Eadsv5_Type1": 10.845000, + "Easv4_Type1": 10.433000, + "Easv4_Type2": 10.433000, + "Easv5_Type1": 9.404000, + "Ebdsv5-Type1": 9.106000, + "Ebsv5-Type1": 8.145000, + "Edsv5_Type1": 11.875000, + "Esv3_Type3": 6.660000, + "Esv3_Type4": 9.024000, + "Esv5_Type1": 10.433000, + "Fsv2_Type2": 4.993000, + "Lasv3_Type1": 10.639000, + "Lsv3_Type1": 11.898000, + "Standard_A1_v2": 0.053300, + "Standard_A2_v2": 0.107000, + "Standard_A2m_v2": 0.161000, + "Standard_A4_v2": 0.213000, + "Standard_A4m_v2": 0.322000, + "Standard_A8_v2": 0.426000, + "Standard_A8m_v2": 0.645000, + "Standard_B12ms": 0.749000, + "Standard_B16als_v2": 0.869000, + "Standard_B16as_v2": 0.972000, + "Standard_B16ls_v2": 0.884000, + "Standard_B16ms": 0.998000, + "Standard_B16s_v2": 0.998000, + "Standard_B1ls": 0.007800, + "Standard_B1ms": 0.031200, + "Standard_B1s": 0.015600, + "Standard_B20ms": 1.248000, + "Standard_B2als_v2": 0.056200, + "Standard_B2as_v2": 0.122000, + "Standard_B2ats_v2": 0.014000, + "Standard_B2ls_v2": 0.071600, + "Standard_B2ms": 0.125000, + "Standard_B2s": 0.062400, + "Standard_B2s_v2": 0.134000, + "Standard_B2ts_v2": 0.024800, + "Standard_B32als_v2": 1.591000, + "Standard_B32as_v2": 1.944000, + "Standard_B32ls_v2": 1.915000, + "Standard_B32s_v2": 2.144000, + "Standard_B4als_v2": 0.217000, + "Standard_B4as_v2": 0.225000, + "Standard_B4ls_v2": 0.240000, + "Standard_B4ms": 0.250000, + "Standard_B4s_v2": 0.250000, + "Standard_B8als_v2": 0.398000, + "Standard_B8as_v2": 0.486000, + "Standard_B8ls_v2": 0.479000, + "Standard_B8ms": 0.499000, + "Standard_B8s_v2": 0.536000, + "Standard_D11_v2": 0.216000, + "Standard_D12_v2": 0.432000, + "Standard_D13_v2": 0.863000, + "Standard_D14_v2": 1.726000, + "Standard_D15_v2": 2.158000, + "Standard_D15i_v2": 2.158000, + "Standard_D16_v3": 1.196000, + "Standard_D16_v4": 1.196000, + "Standard_D16_v5": 1.196000, + "Standard_D16a_v4": 1.196000, + "Standard_D16ads_v5": 2.036000, + "Standard_D16as_v4": 1.196000, + "Standard_D16as_v5": 1.082000, + "Standard_D16d_v4": 1.408000, + "Standard_D16d_v5": 1.414000, + "Standard_D16ds_v4": 1.408000, + "Standard_D16ds_v5": 1.414000, + "Standard_D16lds_v5": 1.154000, + "Standard_D16ls_v5": 1.009000, + "Standard_D16s_v3": 1.196000, + "Standard_D16s_v4": 1.196000, + "Standard_D16s_v5": 1.196000, + "Standard_D1_v2": 0.087100, + "Standard_D2_v2": 0.174000, + "Standard_D2_v3": 0.150000, + "Standard_D2_v4": 0.150000, + "Standard_D2_v5": 0.150000, + "Standard_D2a_v4": 0.150000, + "Standard_D2ads_v5": 0.255000, + "Standard_D2as_v4": 0.150000, + "Standard_D2as_v5": 0.227000, + "Standard_D2d_v4": 0.176000, + "Standard_D2d_v5": 0.177000, + "Standard_D2ds_v4": 0.176000, + "Standard_D2ds_v5": 0.177000, + "Standard_D2lds_v5": 0.144000, + "Standard_D2ls_v5": 0.126000, + "Standard_D2s_v3": 0.150000, + "Standard_D2s_v4": 0.150000, + "Standard_D2s_v5": 0.150000, + "Standard_D32-16s_v3": 2.392000, + "Standard_D32-8s_v3": 2.392000, + "Standard_D32_v3": 2.392000, + "Standard_D32_v4": 2.392000, + "Standard_D32_v5": 2.392000, + "Standard_D32a_v4": 2.392000, + "Standard_D32ads_v5": 2.600000, + "Standard_D32as_v4": 2.392000, + "Standard_D32as_v5": 3.635000, + "Standard_D32d_v4": 2.816000, + "Standard_D32d_v5": 2.829000, + "Standard_D32ds_v4": 2.816000, + "Standard_D32ds_v5": 2.829000, + "Standard_D32lds_v5": 2.309000, + "Standard_D32ls_v5": 2.018000, + "Standard_D32s_v3": 2.392000, + "Standard_D32s_v4": 2.392000, + "Standard_D32s_v5": 2.392000, + "Standard_D3_v2": 0.348000, + "Standard_D48_v3": 3.588000, + "Standard_D48_v4": 3.588000, + "Standard_D48_v5": 3.588000, + "Standard_D48a_v4": 3.588000, + "Standard_D48ads_v5": 3.900000, + "Standard_D48as_v4": 3.588000, + "Standard_D48as_v5": 3.245000, + "Standard_D48d_v4": 4.223000, + "Standard_D48d_v5": 4.243000, + "Standard_D48ds_v4": 4.223000, + "Standard_D48ds_v5": 4.243000, + "Standard_D48lds_v5": 3.463000, + "Standard_D48ls_v5": 3.026000, + "Standard_D48s_v3": 3.588000, + "Standard_D48s_v4": 3.588000, + "Standard_D48s_v5": 3.588000, + "Standard_D4_v2": 0.697000, + "Standard_D4_v3": 0.299000, + "Standard_D4_v4": 0.299000, + "Standard_D4_v5": 0.299000, + "Standard_D4a_v4": 0.299000, + "Standard_D4ads_v5": 0.325000, + "Standard_D4as_v4": 0.299000, + "Standard_D4as_v5": 0.270000, + "Standard_D4d_v4": 0.352000, + "Standard_D4d_v5": 0.354000, + "Standard_D4ds_v4": 0.352000, + "Standard_D4ds_v5": 0.354000, + "Standard_D4lds_v5": 0.289000, + "Standard_D4ls_v5": 0.252000, + "Standard_D4s_v3": 0.299000, + "Standard_D4s_v4": 0.299000, + "Standard_D4s_v5": 0.299000, + "Standard_D5_v2": 1.394000, + "Standard_D64-16s_v3": 4.784000, + "Standard_D64-32s_v3": 4.784000, + "Standard_D64_v3": 4.784000, + "Standard_D64_v4": 4.784000, + "Standard_D64_v5": 4.784000, + "Standard_D64a_v4": 4.784000, + "Standard_D64ads_v5": 8.144000, + "Standard_D64as_v4": 4.784000, + "Standard_D64as_v5": 7.270000, + "Standard_D64d_v4": 5.631000, + "Standard_D64d_v5": 5.658000, + "Standard_D64ds_v4": 5.631000, + "Standard_D64ds_v5": 5.658000, + "Standard_D64lds_v5": 4.618000, + "Standard_D64ls_v5": 4.035000, + "Standard_D64s_v3": 4.784000, + "Standard_D64s_v4": 4.784000, + "Standard_D64s_v5": 4.784000, + "Standard_D8_v3": 0.598000, + "Standard_D8_v4": 0.598000, + "Standard_D8_v5": 0.598000, + "Standard_D8a_v4": 0.598000, + "Standard_D8ads_v5": 0.650000, + "Standard_D8as_v4": 0.598000, + "Standard_D8as_v5": 0.909000, + "Standard_D8d_v4": 0.704000, + "Standard_D8d_v5": 0.707000, + "Standard_D8ds_v4": 0.704000, + "Standard_D8ds_v5": 0.707000, + "Standard_D8lds_v5": 0.577000, + "Standard_D8ls_v5": 0.504000, + "Standard_D8s_v3": 0.598000, + "Standard_D8s_v4": 0.598000, + "Standard_D8s_v5": 0.598000, + "Standard_D96_v5": 7.176000, + "Standard_D96a_v4": 7.176000, + "Standard_D96ads_v5": 12.216000, + "Standard_D96as_v4": 7.176000, + "Standard_D96as_v5": 10.906000, + "Standard_D96d_v5": 8.486000, + "Standard_D96ds_v5": 8.486000, + "Standard_D96lds_v5": 6.926000, + "Standard_D96ls_v5": 6.053000, + "Standard_D96s_v5": 7.176000, + "Standard_DC16ds_v3": 1.414000, + "Standard_DC16s_v3": 1.196000, + "Standard_DC1ds_v3": 0.088400, + "Standard_DC1s_v3": 0.074800, + "Standard_DC24ds_v3": 2.122000, + "Standard_DC24s_v3": 1.794000, + "Standard_DC2ds_v3": 0.177000, + "Standard_DC2s_v3": 0.150000, + "Standard_DC32ds_v3": 2.829000, + "Standard_DC32s_v3": 2.392000, + "Standard_DC48ds_v3": 4.243000, + "Standard_DC48s_v3": 3.588000, + "Standard_DC4ds_v3": 0.354000, + "Standard_DC4s_v3": 0.299000, + "Standard_DC8ds_v3": 0.707000, + "Standard_DC8s_v3": 0.598000, + "Standard_DS11-1_v2": 0.216000, + "Standard_DS11_v2": 0.216000, + "Standard_DS12-1_v2": 0.432000, + "Standard_DS12-2_v2": 0.432000, + "Standard_DS12_v2": 0.432000, + "Standard_DS13-2_v2": 0.863000, + "Standard_DS13-4_v2": 0.863000, + "Standard_DS13_v2": 0.863000, + "Standard_DS14-4_v2": 1.726000, + "Standard_DS14-8_v2": 1.726000, + "Standard_DS14_v2": 1.726000, + "Standard_DS15_v2": 2.158000, + "Standard_DS15i_v2": 2.158000, + "Standard_DS1_v2": 0.087100, + "Standard_DS2_v2": 0.174000, + "Standard_DS3_v2": 0.348000, + "Standard_DS4_v2": 0.697000, + "Standard_DS5_v2": 1.394000, + "Standard_E104i_v5": 11.303000, + "Standard_E104id_v5": 12.864000, + "Standard_E104ids_v5": 12.864000, + "Standard_E104is_v5": 11.303000, + "Standard_E112iads_v5": 12.653000, + "Standard_E112ias_v5": 10.971000, + "Standard_E112ibds_v5": 15.936000, + "Standard_E112ibs_v5": 14.254000, + "Standard_E16-4ads_v5": 1.643000, + "Standard_E16-4as_v4": 1.581000, + "Standard_E16-4as_v5": 1.425000, + "Standard_E16-4ds_v4": 1.807000, + "Standard_E16-4ds_v5": 1.799000, + "Standard_E16-4s_v3": 1.581000, + "Standard_E16-4s_v4": 1.581000, + "Standard_E16-4s_v5": 1.581000, + "Standard_E16-8ads_v5": 1.643000, + "Standard_E16-8as_v4": 1.581000, + "Standard_E16-8as_v5": 2.161000, + "Standard_E16-8ds_v4": 1.807000, + "Standard_E16-8ds_v5": 1.799000, + "Standard_E16-8s_v3": 1.581000, + "Standard_E16-8s_v4": 1.581000, + "Standard_E16-8s_v5": 1.581000, + "Standard_E16_v3": 1.581000, + "Standard_E16_v4": 1.581000, + "Standard_E16_v5": 1.581000, + "Standard_E16a_v4": 1.581000, + "Standard_E16ads_v5": 1.689000, + "Standard_E16as_v4": 1.581000, + "Standard_E16as_v5": 1.425000, + "Standard_E16bds_v5": 2.070000, + "Standard_E16bs_v5": 1.851000, + "Standard_E16d_v4": 1.807000, + "Standard_E16d_v5": 1.799000, + "Standard_E16ds_v4": 1.807000, + "Standard_E16ds_v5": 1.799000, + "Standard_E16s_v3": 1.581000, + "Standard_E16s_v4": 1.581000, + "Standard_E16s_v5": 1.581000, + "Standard_E20_v3": 1.976000, + "Standard_E20_v4": 1.976000, + "Standard_E20_v5": 1.976000, + "Standard_E20a_v4": 1.976000, + "Standard_E20ads_v5": 2.054000, + "Standard_E20as_v4": 1.976000, + "Standard_E20as_v5": 1.781000, + "Standard_E20d_v4": 2.259000, + "Standard_E20d_v5": 2.249000, + "Standard_E20ds_v4": 2.259000, + "Standard_E20ds_v5": 2.249000, + "Standard_E20s_v3": 1.976000, + "Standard_E20s_v4": 1.976000, + "Standard_E20s_v5": 1.976000, + "Standard_E2_v3": 0.198000, + "Standard_E2_v4": 0.198000, + "Standard_E2_v5": 0.198000, + "Standard_E2a_v4": 0.198000, + "Standard_E2ads_v5": 0.251000, + "Standard_E2as_v4": 0.198000, + "Standard_E2as_v5": 0.178000, + "Standard_E2bds_v5": 0.259000, + "Standard_E2bs_v5": 0.231000, + "Standard_E2d_v4": 0.226000, + "Standard_E2d_v5": 0.225000, + "Standard_E2ds_v4": 0.226000, + "Standard_E2ds_v5": 0.225000, + "Standard_E2s_v3": 0.198000, + "Standard_E2s_v4": 0.198000, + "Standard_E2s_v5": 0.198000, + "Standard_E32-16ads_v5": 4.758000, + "Standard_E32-16as_v4": 3.162000, + "Standard_E32-16as_v5": 4.322000, + "Standard_E32-16ds_v4": 3.614000, + "Standard_E32-16ds_v5": 3.598000, + "Standard_E32-16s_v3": 3.162000, + "Standard_E32-16s_v4": 3.162000, + "Standard_E32-16s_v5": 3.162000, + "Standard_E32-8ads_v5": 4.758000, + "Standard_E32-8as_v4": 3.162000, + "Standard_E32-8as_v5": 4.322000, + "Standard_E32-8ds_v4": 3.614000, + "Standard_E32-8ds_v5": 3.598000, + "Standard_E32-8s_v3": 3.162000, + "Standard_E32-8s_v4": 3.162000, + "Standard_E32-8s_v5": 3.162000, + "Standard_E32_v3": 3.162000, + "Standard_E32_v4": 3.162000, + "Standard_E32_v5": 3.162000, + "Standard_E32a_v4": 3.162000, + "Standard_E32ads_v5": 3.332000, + "Standard_E32as_v4": 3.162000, + "Standard_E32as_v5": 2.896000, + "Standard_E32bds_v5": 4.139000, + "Standard_E32bs_v5": 3.702000, + "Standard_E32d_v4": 3.614000, + "Standard_E32d_v5": 3.598000, + "Standard_E32ds_v4": 3.614000, + "Standard_E32ds_v5": 3.598000, + "Standard_E32s_v3": 3.162000, + "Standard_E32s_v4": 3.162000, + "Standard_E32s_v5": 3.162000, + "Standard_E4-2ads_v5": 0.595000, + "Standard_E4-2as_v4": 0.395000, + "Standard_E4-2as_v5": 0.356000, + "Standard_E4-2ds_v4": 0.452000, + "Standard_E4-2ds_v5": 0.450000, + "Standard_E4-2s_v3": 0.395000, + "Standard_E4-2s_v4": 0.395000, + "Standard_E4-2s_v5": 0.395000, + "Standard_E48_v3": 4.743000, + "Standard_E48_v4": 4.743000, + "Standard_E48_v5": 4.742000, + "Standard_E48a_v4": 4.742000, + "Standard_E48ads_v5": 4.930000, + "Standard_E48as_v4": 4.742000, + "Standard_E48as_v5": 4.274000, + "Standard_E48bds_v5": 6.209000, + "Standard_E48bs_v5": 5.554000, + "Standard_E48d_v4": 5.421000, + "Standard_E48d_v5": 5.398000, + "Standard_E48ds_v4": 5.421000, + "Standard_E48ds_v5": 5.398000, + "Standard_E48s_v3": 4.743000, + "Standard_E48s_v4": 4.743000, + "Standard_E48s_v5": 4.742000, + "Standard_E4_v3": 0.395000, + "Standard_E4_v4": 0.395000, + "Standard_E4_v5": 0.395000, + "Standard_E4a_v4": 0.395000, + "Standard_E4ads_v5": 0.457000, + "Standard_E4as_v4": 0.395000, + "Standard_E4as_v5": 0.356000, + "Standard_E4bds_v5": 0.517000, + "Standard_E4bs_v5": 0.463000, + "Standard_E4d_v4": 0.452000, + "Standard_E4d_v5": 0.450000, + "Standard_E4ds_v4": 0.452000, + "Standard_E4ds_v5": 0.450000, + "Standard_E4s_v3": 0.395000, + "Standard_E4s_v4": 0.395000, + "Standard_E4s_v5": 0.395000, + "Standard_E64-16ads_v5": 6.573000, + "Standard_E64-16as_v4": 6.323000, + "Standard_E64-16as_v5": 8.643000, + "Standard_E64-16ds_v4": 7.228000, + "Standard_E64-16ds_v5": 7.197000, + "Standard_E64-16s_v3": 5.691000, + "Standard_E64-16s_v4": 6.324000, + "Standard_E64-16s_v5": 6.323000, + "Standard_E64-32ads_v5": 9.517000, + "Standard_E64-32as_v4": 6.323000, + "Standard_E64-32as_v5": 5.699000, + "Standard_E64-32ds_v4": 7.228000, + "Standard_E64-32ds_v5": 7.197000, + "Standard_E64-32s_v3": 5.691000, + "Standard_E64-32s_v4": 6.324000, + "Standard_E64-32s_v5": 6.323000, + "Standard_E64_v3": 5.691000, + "Standard_E64_v4": 6.324000, + "Standard_E64_v5": 6.323000, + "Standard_E64a_v4": 6.323000, + "Standard_E64ads_v5": 6.573000, + "Standard_E64as_v4": 6.323000, + "Standard_E64as_v5": 5.699000, + "Standard_E64bds_v5": 8.278000, + "Standard_E64bs_v5": 7.405000, + "Standard_E64d_v4": 7.228000, + "Standard_E64d_v5": 7.197000, + "Standard_E64ds_v4": 7.228000, + "Standard_E64ds_v5": 7.197000, + "Standard_E64i_v3": 5.691000, + "Standard_E64is_v3": 5.691000, + "Standard_E64s_v3": 5.691000, + "Standard_E64s_v4": 6.324000, + "Standard_E64s_v5": 6.323000, + "Standard_E8-2ads_v5": 0.822000, + "Standard_E8-2as_v4": 0.790000, + "Standard_E8-2as_v5": 0.712000, + "Standard_E8-2ds_v4": 0.903000, + "Standard_E8-2ds_v5": 0.900000, + "Standard_E8-2s_v3": 0.790000, + "Standard_E8-2s_v4": 0.791000, + "Standard_E8-2s_v5": 0.790000, + "Standard_E8-4ads_v5": 0.822000, + "Standard_E8-4as_v4": 0.790000, + "Standard_E8-4as_v5": 0.712000, + "Standard_E8-4ds_v4": 0.903000, + "Standard_E8-4ds_v5": 0.900000, + "Standard_E8-4s_v3": 0.790000, + "Standard_E8-4s_v4": 0.791000, + "Standard_E8-4s_v5": 0.790000, + "Standard_E80ids_v4": 9.035000, + "Standard_E80is_v4": 7.905000, + "Standard_E8_v3": 0.790000, + "Standard_E8_v4": 0.791000, + "Standard_E8_v5": 0.790000, + "Standard_E8a_v4": 0.790000, + "Standard_E8ads_v5": 0.822000, + "Standard_E8as_v4": 0.790000, + "Standard_E8as_v5": 0.758000, + "Standard_E8bds_v5": 1.035000, + "Standard_E8bs_v5": 0.926000, + "Standard_E8d_v4": 0.903000, + "Standard_E8d_v5": 0.900000, + "Standard_E8ds_v4": 0.903000, + "Standard_E8ds_v5": 0.900000, + "Standard_E8s_v3": 0.790000, + "Standard_E8s_v4": 0.791000, + "Standard_E8s_v5": 0.790000, + "Standard_E96-24ads_v5": 9.859000, + "Standard_E96-24as_v4": 9.485000, + "Standard_E96-24as_v5": 8.549000, + "Standard_E96-24ds_v5": 10.795000, + "Standard_E96-24s_v5": 9.485000, + "Standard_E96-48ads_v5": 14.275000, + "Standard_E96-48as_v4": 9.485000, + "Standard_E96-48as_v5": 8.549000, + "Standard_E96-48ds_v5": 10.795000, + "Standard_E96-48s_v5": 9.485000, + "Standard_E96_v5": 9.485000, + "Standard_E96a_v4": 9.485000, + "Standard_E96ads_v5": 9.859000, + "Standard_E96as_v4": 9.485000, + "Standard_E96as_v5": 8.549000, + "Standard_E96bds_v5": 12.418000, + "Standard_E96bs_v5": 11.107000, + "Standard_E96d_v5": 10.795000, + "Standard_E96ds_v5": 10.795000, + "Standard_E96iads_v5": 10.845000, + "Standard_E96ias_v4": 9.485000, + "Standard_E96ias_v5": 9.404000, + "Standard_E96s_v5": 9.485000, + "Standard_F1": 0.074100, + "Standard_F16": 1.186000, + "Standard_F16s": 1.186000, + "Standard_F16s_v2": 1.009000, + "Standard_F1s": 0.074100, + "Standard_F2": 0.148000, + "Standard_F2s": 0.148000, + "Standard_F2s_v2": 0.126000, + "Standard_F32s_v2": 2.018000, + "Standard_F4": 0.296000, + "Standard_F48s_v2": 3.027000, + "Standard_F4s": 0.296000, + "Standard_F4s_v2": 0.252000, + "Standard_F64s_v2": 4.035000, + "Standard_F72s_v2": 4.540000, + "Standard_F8": 0.593000, + "Standard_F8s": 0.593000, + "Standard_F8s_v2": 0.504000, + "Standard_L16as_v3": 1.934000, + "Standard_L16s_v3": 2.163000, + "Standard_L32as_v3": 3.869000, + "Standard_L32s_v3": 4.326000, + "Standard_L48as_v3": 5.803000, + "Standard_L48s_v3": 6.490000, + "Standard_L64as_v3": 7.738000, + "Standard_L64s_v3": 8.653000, + "Standard_L80as_v3": 9.672000, + "Standard_L80s_v3": 10.816000, + "Standard_L8as_v3": 0.967000, + "Standard_L8s_v3": 1.082000, + "Standard_M128": 24.276000, + "Standard_M128-32ms": 48.547000, + "Standard_M128-64ms": 48.547000, + "Standard_M128m": 48.547000, + "Standard_M128ms": 48.547000, + "Standard_M128s": 24.276000, + "Standard_M16-4ms": 5.593000, + "Standard_M16-8ms": 5.593000, + "Standard_M16ms": 5.593000, + "Standard_M16s": 4.344000, + "Standard_M32-16ms": 11.186000, + "Standard_M32-8ms": 11.186000, + "Standard_M32ls": 4.482000, + "Standard_M32ms": 11.186000, + "Standard_M32s": 6.070000, + "Standard_M32ts": 4.222000, + "Standard_M64": 12.138000, + "Standard_M64-16ms": 18.813000, + "Standard_M64-32ms": 18.813000, + "Standard_M64ls": 8.447000, + "Standard_M64m": 18.813000, + "Standard_M64ms": 18.813000, + "Standard_M64s": 12.138000, + "Standard_M8-2ms": 2.796000, + "Standard_M8-4ms": 2.796000, + "Standard_M8ms": 2.796000, + "Standard_ND96ams_A100_v4": 55.381000, + "Standard_ND96amsr_A100_v4": 55.381000, + } + // germanywestcentral + initialOnDemandPrices["germanywestcentral"] = map[string]float64{ + "DCdsv3 Type1": 3.590000, + "DCsv3 Type1": 3.036000, + "Dadsv5_Type1": 7.700000, + "Dasv4_Type1": 6.325000, + "Dasv4_Type2": 6.325000, + "Dasv5_Type1": 6.406000, + "Ddsv4_Type 1": 4.787000, + "Ddsv4_Type2": 5.685000, + "Ddsv5_Type1": 7.181000, + "Dsv3_Type3": 5.086000, + "Dsv3_Type4": 6.357000, + "Dsv4_Type1": 5.060000, + "Dsv4_Type2": 6.072000, + "Dsv5_Type1": 6.325000, + "Eadsv5_Type1": 8.342000, + "Easv4_Type1": 8.026000, + "Easv4_Type2": 8.026000, + "Easv5_Type1": 7.234000, + "Ebdsv5-Type1": 7.005000, + "Ebsv5-Type1": 6.266000, + "Edsv4_Type 1": 6.090000, + "Edsv4_Type2": 7.231000, + "Edsv5_Type1": 9.134000, + "Esv3_Type3": 5.162000, + "Esv3_Type4": 6.994000, + "Esv4_Type1": 5.350000, + "Esv4_Type2": 7.022000, + "Esv5_Type1": 8.026000, + "Fsv2 Type3": 4.481000, + "Fsv2_Type2": 3.841000, + "Fsv2_Type4": 5.122000, + "Lsv2_Type1": 8.184000, + "Lsv3_Type1": 9.152000, + "Mdmsv2MedMem _Type1": 41.103000, + "Mdsv2MedMem_Type1": 20.544000, + "Ms_Type1": 20.536000, + "Msm_Type1": 41.108000, + "Msmv2MedMem Type1": 40.455000, + "Msmv2_Type1": 130.880000, + "Msv2MedMem Type1": 19.896000, + "Msv2_Type1": 65.446000, + "Standard_A1_v2": 0.041000, + "Standard_A2_v2": 0.087000, + "Standard_A2m_v2": 0.124000, + "Standard_A4_v2": 0.183000, + "Standard_A4m_v2": 0.260000, + "Standard_A8_v2": 0.383000, + "Standard_A8m_v2": 0.546000, + "Standard_B12ms": 0.576000, + "Standard_B16als_v2": 0.612000, + "Standard_B16as_v2": 0.691000, + "Standard_B16ls_v2": 0.754000, + "Standard_B16ms": 0.768000, + "Standard_B16pls_v2": 0.544000, + "Standard_B16ps_v2": 0.614000, + "Standard_B16s_v2": 0.842000, + "Standard_B1ls": 0.006000, + "Standard_B1ms": 0.024000, + "Standard_B1s": 0.012000, + "Standard_B20ms": 0.960000, + "Standard_B2als_v2": 0.043200, + "Standard_B2as_v2": 0.095600, + "Standard_B2ats_v2": 0.020000, + "Standard_B2ls_v2": 0.048000, + "Standard_B2ms": 0.096000, + "Standard_B2pls_v2": 0.047600, + "Standard_B2ps_v2": 0.086000, + "Standard_B2pts_v2": 0.018800, + "Standard_B2s": 0.048000, + "Standard_B2s_v2": 0.096000, + "Standard_B2ts_v2": 0.021200, + "Standard_B32als_v2": 1.371000, + "Standard_B32as_v2": 1.382000, + "Standard_B32ls_v2": 1.507000, + "Standard_B32s_v2": 1.536000, + "Standard_B4als_v2": 0.171000, + "Standard_B4as_v2": 0.191000, + "Standard_B4ls_v2": 0.188000, + "Standard_B4ms": 0.192000, + "Standard_B4pls_v2": 0.136000, + "Standard_B4ps_v2": 0.154000, + "Standard_B4s_v2": 0.192000, + "Standard_B8als_v2": 0.343000, + "Standard_B8as_v2": 0.346000, + "Standard_B8ls_v2": 0.377000, + "Standard_B8ms": 0.384000, + "Standard_B8pls_v2": 0.309000, + "Standard_B8ps_v2": 0.307000, + "Standard_B8s_v2": 0.384000, + "Standard_D11_v2": 0.166000, + "Standard_D12_v2": 0.332000, + "Standard_D13_v2": 0.664000, + "Standard_D14_v2": 1.328000, + "Standard_D15_v2": 1.660000, + "Standard_D15i_v2": 1.660000, + "Standard_D16_v3": 0.920000, + "Standard_D16_v4": 0.920000, + "Standard_D16_v5": 0.920000, + "Standard_D16ads_v5": 1.736000, + "Standard_D16as_v5": 0.832000, + "Standard_D16d_v4": 1.088000, + "Standard_D16d_v5": 1.088000, + "Standard_D16ds_v4": 1.088000, + "Standard_D16ds_v5": 1.088000, + "Standard_D16lds_v5": 0.888000, + "Standard_D16ls_v5": 0.776000, + "Standard_D16pds_v5": 0.872000, + "Standard_D16plds_v5": 0.712000, + "Standard_D16pls_v5": 0.621000, + "Standard_D16ps_v5": 0.736000, + "Standard_D16s_v3": 0.920000, + "Standard_D16s_v4": 0.920000, + "Standard_D16s_v5": 0.920000, + "Standard_D1_v2": 0.067000, + "Standard_D2_v2": 0.133000, + "Standard_D2_v3": 0.115000, + "Standard_D2_v4": 0.115000, + "Standard_D2_v5": 0.115000, + "Standard_D2ads_v5": 0.217000, + "Standard_D2as_v5": 0.104000, + "Standard_D2d_v4": 0.136000, + "Standard_D2d_v5": 0.136000, + "Standard_D2ds_v4": 0.136000, + "Standard_D2ds_v5": 0.136000, + "Standard_D2lds_v5": 0.111000, + "Standard_D2ls_v5": 0.097000, + "Standard_D2pds_v5": 0.109000, + "Standard_D2plds_v5": 0.089000, + "Standard_D2pls_v5": 0.077600, + "Standard_D2ps_v5": 0.092000, + "Standard_D2s_v3": 0.115000, + "Standard_D2s_v4": 0.115000, + "Standard_D2s_v5": 0.115000, + "Standard_D32-16s_v3": 1.840000, + "Standard_D32-8s_v3": 1.840000, + "Standard_D32_v3": 1.840000, + "Standard_D32_v4": 1.840000, + "Standard_D32_v5": 1.840000, + "Standard_D32ads_v5": 2.000000, + "Standard_D32as_v5": 3.136000, + "Standard_D32d_v4": 2.176000, + "Standard_D32d_v5": 2.176000, + "Standard_D32ds_v4": 2.176000, + "Standard_D32ds_v5": 2.176000, + "Standard_D32lds_v5": 1.776000, + "Standard_D32ls_v5": 1.552000, + "Standard_D32pds_v5": 1.744000, + "Standard_D32plds_v5": 1.424000, + "Standard_D32pls_v5": 1.242000, + "Standard_D32ps_v5": 1.472000, + "Standard_D32s_v3": 1.840000, + "Standard_D32s_v4": 1.840000, + "Standard_D32s_v5": 1.840000, + "Standard_D3_v2": 0.266000, + "Standard_D48_v3": 2.760000, + "Standard_D48_v4": 2.760000, + "Standard_D48_v5": 2.760000, + "Standard_D48ads_v5": 3.000000, + "Standard_D48as_v5": 4.704000, + "Standard_D48d_v4": 3.264000, + "Standard_D48d_v5": 3.264000, + "Standard_D48ds_v4": 3.264000, + "Standard_D48ds_v5": 3.264000, + "Standard_D48lds_v5": 2.664000, + "Standard_D48ls_v5": 2.328000, + "Standard_D48pds_v5": 2.616000, + "Standard_D48plds_v5": 2.136000, + "Standard_D48pls_v5": 1.862000, + "Standard_D48ps_v5": 2.208000, + "Standard_D48s_v3": 2.760000, + "Standard_D48s_v4": 2.760000, + "Standard_D48s_v5": 2.760000, + "Standard_D4_v2": 0.532000, + "Standard_D4_v3": 0.230000, + "Standard_D4_v4": 0.230000, + "Standard_D4_v5": 0.230000, + "Standard_D4ads_v5": 0.250000, + "Standard_D4as_v5": 0.392000, + "Standard_D4d_v4": 0.272000, + "Standard_D4d_v5": 0.272000, + "Standard_D4ds_v4": 0.272000, + "Standard_D4ds_v5": 0.272000, + "Standard_D4lds_v5": 0.222000, + "Standard_D4ls_v5": 0.194000, + "Standard_D4pds_v5": 0.218000, + "Standard_D4plds_v5": 0.178000, + "Standard_D4pls_v5": 0.155000, + "Standard_D4ps_v5": 0.184000, + "Standard_D4s_v3": 0.230000, + "Standard_D4s_v4": 0.230000, + "Standard_D4s_v5": 0.230000, + "Standard_D5_v2": 1.064000, + "Standard_D64-16s_v3": 3.680000, + "Standard_D64-32s_v3": 3.680000, + "Standard_D64_v3": 3.680000, + "Standard_D64_v4": 3.680000, + "Standard_D64_v5": 3.680000, + "Standard_D64ads_v5": 4.000000, + "Standard_D64as_v5": 3.328000, + "Standard_D64d_v4": 4.352000, + "Standard_D64d_v5": 4.352000, + "Standard_D64ds_v4": 4.352000, + "Standard_D64ds_v5": 4.352000, + "Standard_D64lds_v5": 3.552000, + "Standard_D64ls_v5": 3.104000, + "Standard_D64pds_v5": 3.488000, + "Standard_D64plds_v5": 2.848000, + "Standard_D64pls_v5": 2.483000, + "Standard_D64ps_v5": 2.944000, + "Standard_D64s_v3": 3.680000, + "Standard_D64s_v4": 3.680000, + "Standard_D64s_v5": 3.680000, + "Standard_D8_v3": 0.460000, + "Standard_D8_v4": 0.460000, + "Standard_D8_v5": 0.460000, + "Standard_D8ads_v5": 0.500000, + "Standard_D8as_v5": 0.416000, + "Standard_D8d_v4": 0.544000, + "Standard_D8d_v5": 0.544000, + "Standard_D8ds_v4": 0.544000, + "Standard_D8ds_v5": 0.544000, + "Standard_D8lds_v5": 0.444000, + "Standard_D8ls_v5": 0.388000, + "Standard_D8pds_v5": 0.436000, + "Standard_D8plds_v5": 0.356000, + "Standard_D8pls_v5": 0.310000, + "Standard_D8ps_v5": 0.368000, + "Standard_D8s_v3": 0.460000, + "Standard_D8s_v4": 0.460000, + "Standard_D8s_v5": 0.460000, + "Standard_D96_v5": 5.520000, + "Standard_D96ads_v5": 10.416000, + "Standard_D96as_v5": 4.992000, + "Standard_D96d_v5": 6.528000, + "Standard_D96ds_v5": 6.528000, + "Standard_D96lds_v5": 5.328000, + "Standard_D96ls_v5": 4.656000, + "Standard_D96s_v5": 5.520000, + "Standard_DC16ads_v5": 1.000000, + "Standard_DC16as_v5": 0.832000, + "Standard_DC16ds_v3": 1.088000, + "Standard_DC16s_v3": 0.920000, + "Standard_DC1ds_v3": 0.068000, + "Standard_DC1s_v3": 0.057500, + "Standard_DC24ds_v3": 1.632000, + "Standard_DC24s_v3": 1.380000, + "Standard_DC2ads_v5": 0.125000, + "Standard_DC2as_v5": 0.104000, + "Standard_DC2ds_v3": 0.136000, + "Standard_DC2s_v3": 0.115000, + "Standard_DC32ads_v5": 2.000000, + "Standard_DC32as_v5": 1.664000, + "Standard_DC32ds_v3": 2.176000, + "Standard_DC32s_v3": 1.840000, + "Standard_DC48ads_v5": 3.000000, + "Standard_DC48as_v5": 2.496000, + "Standard_DC48ds_v3": 3.264000, + "Standard_DC48s_v3": 2.760000, + "Standard_DC4ads_v5": 0.250000, + "Standard_DC4as_v5": 0.208000, + "Standard_DC4ds_v3": 0.272000, + "Standard_DC4s_v3": 0.230000, + "Standard_DC64ads_v5": 4.000000, + "Standard_DC64as_v5": 3.328000, + "Standard_DC8ads_v5": 0.500000, + "Standard_DC8as_v5": 0.416000, + "Standard_DC8ds_v3": 0.544000, + "Standard_DC8s_v3": 0.460000, + "Standard_DC96ads_v5": 6.000000, + "Standard_DC96as_v5": 4.992000, + "Standard_DS11-1_v2": 0.166000, + "Standard_DS11_v2": 0.166000, + "Standard_DS12-1_v2": 0.332000, + "Standard_DS12-2_v2": 0.332000, + "Standard_DS12_v2": 0.332000, + "Standard_DS13-2_v2": 0.664000, + "Standard_DS13-4_v2": 0.664000, + "Standard_DS13_v2": 0.664000, + "Standard_DS14-4_v2": 1.328000, + "Standard_DS14-8_v2": 1.328000, + "Standard_DS14_v2": 1.328000, + "Standard_DS15_v2": 1.660000, + "Standard_DS15i_v2": 1.660000, + "Standard_DS1_v2": 0.067000, + "Standard_DS2_v2": 0.133000, + "Standard_DS3_v2": 0.266000, + "Standard_DS4_v2": 0.532000, + "Standard_DS5_v2": 1.064000, + "Standard_E104i_v5": 8.694000, + "Standard_E104id_v5": 9.896000, + "Standard_E104ids_v5": 9.896000, + "Standard_E104is_v5": 8.694000, + "Standard_E112iads_v5": 9.733000, + "Standard_E112ias_v5": 8.439000, + "Standard_E112ibds_v5": 12.258000, + "Standard_E112ibs_v5": 10.965000, + "Standard_E16-4ads_v5": 1.264000, + "Standard_E16-4as_v5": 1.832000, + "Standard_E16-4ds_v4": 1.384000, + "Standard_E16-4ds_v5": 1.384000, + "Standard_E16-4s_v3": 1.216000, + "Standard_E16-4s_v4": 1.216000, + "Standard_E16-4s_v5": 1.216000, + "Standard_E16-8ads_v5": 2.000000, + "Standard_E16-8as_v5": 1.096000, + "Standard_E16-8ds_v4": 1.384000, + "Standard_E16-8ds_v5": 1.384000, + "Standard_E16-8s_v3": 1.216000, + "Standard_E16-8s_v4": 1.216000, + "Standard_E16-8s_v5": 1.216000, + "Standard_E16_v3": 1.216000, + "Standard_E16_v4": 1.216000, + "Standard_E16_v5": 1.216000, + "Standard_E16ads_v5": 1.310000, + "Standard_E16as_v5": 1.142000, + "Standard_E16bds_v5": 1.592000, + "Standard_E16bs_v5": 1.424000, + "Standard_E16d_v4": 1.384000, + "Standard_E16d_v5": 1.384000, + "Standard_E16ds_v4": 1.384000, + "Standard_E16ds_v5": 1.384000, + "Standard_E16pds_v5": 1.104000, + "Standard_E16ps_v5": 0.973000, + "Standard_E16s_v3": 1.216000, + "Standard_E16s_v4": 1.216000, + "Standard_E16s_v5": 1.216000, + "Standard_E20_v3": 1.520000, + "Standard_E20_v4": 1.520000, + "Standard_E20_v5": 1.520000, + "Standard_E20ads_v5": 1.626000, + "Standard_E20as_v5": 1.370000, + "Standard_E20d_v4": 1.730000, + "Standard_E20d_v5": 1.730000, + "Standard_E20ds_v4": 1.730000, + "Standard_E20ds_v5": 1.730000, + "Standard_E20pds_v5": 1.380000, + "Standard_E20ps_v5": 1.216000, + "Standard_E20s_v3": 1.520000, + "Standard_E20s_v4": 1.520000, + "Standard_E20s_v5": 1.520000, + "Standard_E2_v3": 0.152000, + "Standard_E2_v4": 0.152000, + "Standard_E2_v5": 0.152000, + "Standard_E2ads_v5": 0.204000, + "Standard_E2as_v5": 0.183000, + "Standard_E2bds_v5": 0.199000, + "Standard_E2bs_v5": 0.178000, + "Standard_E2d_v4": 0.173000, + "Standard_E2d_v5": 0.173000, + "Standard_E2ds_v4": 0.173000, + "Standard_E2ds_v5": 0.173000, + "Standard_E2pds_v5": 0.138000, + "Standard_E2ps_v5": 0.122000, + "Standard_E2s_v3": 0.152000, + "Standard_E2s_v4": 0.152000, + "Standard_E2s_v5": 0.152000, + "Standard_E32-16ads_v5": 4.000000, + "Standard_E32-16as_v5": 3.664000, + "Standard_E32-16ds_v4": 2.768000, + "Standard_E32-16ds_v5": 2.768000, + "Standard_E32-16s_v3": 2.432000, + "Standard_E32-16s_v4": 2.432000, + "Standard_E32-16s_v5": 2.432000, + "Standard_E32-8ads_v5": 4.000000, + "Standard_E32-8as_v5": 3.664000, + "Standard_E32-8ds_v4": 2.768000, + "Standard_E32-8ds_v5": 2.768000, + "Standard_E32-8s_v3": 2.432000, + "Standard_E32-8s_v4": 2.432000, + "Standard_E32-8s_v5": 2.432000, + "Standard_E32_v3": 2.432000, + "Standard_E32_v4": 2.432000, + "Standard_E32_v5": 2.432000, + "Standard_E32ads_v5": 2.528000, + "Standard_E32as_v5": 2.238000, + "Standard_E32bds_v5": 3.184000, + "Standard_E32bs_v5": 2.848000, + "Standard_E32d_v4": 2.768000, + "Standard_E32d_v5": 2.768000, + "Standard_E32ds_v4": 2.768000, + "Standard_E32ds_v5": 2.768000, + "Standard_E32pds_v5": 2.208000, + "Standard_E32ps_v5": 1.946000, + "Standard_E32s_v3": 2.432000, + "Standard_E32s_v4": 2.432000, + "Standard_E32s_v5": 2.432000, + "Standard_E4-2ads_v5": 0.500000, + "Standard_E4-2as_v5": 0.274000, + "Standard_E4-2ds_v4": 0.346000, + "Standard_E4-2ds_v5": 0.346000, + "Standard_E4-2s_v3": 0.304000, + "Standard_E4-2s_v4": 0.304000, + "Standard_E4-2s_v5": 0.304000, + "Standard_E48_v3": 3.648000, + "Standard_E48_v4": 3.648000, + "Standard_E48_v5": 3.648000, + "Standard_E48ads_v5": 3.838000, + "Standard_E48as_v5": 3.334000, + "Standard_E48bds_v5": 4.776000, + "Standard_E48bs_v5": 4.272000, + "Standard_E48d_v4": 4.152000, + "Standard_E48d_v5": 4.152000, + "Standard_E48ds_v4": 4.152000, + "Standard_E48ds_v5": 4.152000, + "Standard_E48s_v3": 3.648000, + "Standard_E48s_v4": 3.648000, + "Standard_E48s_v5": 3.648000, + "Standard_E4_v3": 0.304000, + "Standard_E4_v4": 0.304000, + "Standard_E4_v5": 0.304000, + "Standard_E4ads_v5": 0.362000, + "Standard_E4as_v5": 0.274000, + "Standard_E4bds_v5": 0.398000, + "Standard_E4bs_v5": 0.356000, + "Standard_E4d_v4": 0.346000, + "Standard_E4d_v5": 0.346000, + "Standard_E4ds_v4": 0.346000, + "Standard_E4ds_v5": 0.346000, + "Standard_E4pds_v5": 0.276000, + "Standard_E4ps_v5": 0.243000, + "Standard_E4s_v3": 0.304000, + "Standard_E4s_v4": 0.304000, + "Standard_E4s_v5": 0.304000, + "Standard_E64-16ads_v5": 5.056000, + "Standard_E64-16as_v5": 7.328000, + "Standard_E64-16ds_v4": 5.536000, + "Standard_E64-16ds_v5": 5.536000, + "Standard_E64-16s_v3": 4.378000, + "Standard_E64-16s_v4": 4.864000, + "Standard_E64-16s_v5": 4.864000, + "Standard_E64-32ads_v5": 8.000000, + "Standard_E64-32as_v5": 7.328000, + "Standard_E64-32ds_v4": 5.536000, + "Standard_E64-32ds_v5": 5.536000, + "Standard_E64-32s_v3": 4.378000, + "Standard_E64-32s_v4": 4.864000, + "Standard_E64-32s_v5": 4.864000, + "Standard_E64_v3": 4.378000, + "Standard_E64_v4": 4.864000, + "Standard_E64_v5": 4.864000, + "Standard_E64ads_v5": 5.102000, + "Standard_E64as_v5": 4.430000, + "Standard_E64bds_v5": 6.368000, + "Standard_E64bs_v5": 5.696000, + "Standard_E64d_v4": 5.536000, + "Standard_E64d_v5": 5.536000, + "Standard_E64ds_v4": 5.536000, + "Standard_E64ds_v5": 5.536000, + "Standard_E64i_v3": 4.378000, + "Standard_E64is_v3": 4.378000, + "Standard_E64s_v3": 4.378000, + "Standard_E64s_v4": 4.864000, + "Standard_E64s_v5": 4.864000, + "Standard_E8-2ads_v5": 0.632000, + "Standard_E8-2as_v5": 0.916000, + "Standard_E8-2ds_v4": 0.692000, + "Standard_E8-2ds_v5": 0.692000, + "Standard_E8-2s_v3": 0.608000, + "Standard_E8-2s_v4": 0.608000, + "Standard_E8-2s_v5": 0.608000, + "Standard_E8-4ads_v5": 1.000000, + "Standard_E8-4as_v5": 0.916000, + "Standard_E8-4ds_v4": 0.692000, + "Standard_E8-4ds_v5": 0.692000, + "Standard_E8-4s_v3": 0.608000, + "Standard_E8-4s_v4": 0.608000, + "Standard_E8-4s_v5": 0.608000, + "Standard_E80ids_v4": 6.920000, + "Standard_E80is_v4": 6.080000, + "Standard_E8_v3": 0.608000, + "Standard_E8_v4": 0.608000, + "Standard_E8_v5": 0.608000, + "Standard_E8ads_v5": 0.632000, + "Standard_E8as_v5": 0.548000, + "Standard_E8bds_v5": 0.796000, + "Standard_E8bs_v5": 0.712000, + "Standard_E8d_v4": 0.692000, + "Standard_E8d_v5": 0.692000, + "Standard_E8ds_v4": 0.692000, + "Standard_E8ds_v5": 0.692000, + "Standard_E8pds_v5": 0.552000, + "Standard_E8ps_v5": 0.486000, + "Standard_E8s_v3": 0.608000, + "Standard_E8s_v4": 0.608000, + "Standard_E8s_v5": 0.608000, + "Standard_E96-24ads_v5": 7.584000, + "Standard_E96-24as_v5": 10.992000, + "Standard_E96-24ds_v5": 8.304000, + "Standard_E96-24s_v5": 7.296000, + "Standard_E96-48ads_v5": 7.584000, + "Standard_E96-48as_v5": 10.992000, + "Standard_E96-48ds_v5": 8.304000, + "Standard_E96-48s_v5": 7.296000, + "Standard_E96_v5": 7.296000, + "Standard_E96ads_v5": 7.630000, + "Standard_E96as_v5": 6.622000, + "Standard_E96bds_v5": 9.552000, + "Standard_E96bs_v5": 8.544000, + "Standard_E96d_v5": 8.304000, + "Standard_E96ds_v5": 8.304000, + "Standard_E96iads_v5": 8.342000, + "Standard_E96ias_v5": 7.234000, + "Standard_E96s_v5": 7.296000, + "Standard_EC16ads_v5": 1.264000, + "Standard_EC16as_v5": 1.096000, + "Standard_EC20ads_v5": 1.580000, + "Standard_EC20as_v5": 1.370000, + "Standard_EC2ads_v5": 0.158000, + "Standard_EC2as_v5": 0.137000, + "Standard_EC32ads_v5": 2.528000, + "Standard_EC32as_v5": 2.192000, + "Standard_EC48ads_v5": 3.792000, + "Standard_EC48as_v5": 3.288000, + "Standard_EC4ads_v5": 0.316000, + "Standard_EC4as_v5": 0.274000, + "Standard_EC64ads_v5": 5.056000, + "Standard_EC64as_v5": 4.384000, + "Standard_EC8ads_v5": 0.632000, + "Standard_EC8as_v5": 0.548000, + "Standard_EC96ads_v5": 7.584000, + "Standard_EC96as_v5": 6.576000, + "Standard_EC96iads_v5": 8.342000, + "Standard_EC96ias_v5": 7.234000, + "Standard_F1": 0.057000, + "Standard_F16": 0.909000, + "Standard_F16s": 0.909000, + "Standard_F16s_v2": 0.776000, + "Standard_F1s": 0.057000, + "Standard_F2": 0.114000, + "Standard_F2s": 0.114000, + "Standard_F2s_v2": 0.097000, + "Standard_F32s_v2": 1.552000, + "Standard_F4": 0.227000, + "Standard_F48s_v2": 2.328000, + "Standard_F4s": 0.227000, + "Standard_F4s_v2": 0.194000, + "Standard_F64s_v2": 3.104000, + "Standard_F72s_v2": 3.492000, + "Standard_F8": 0.454000, + "Standard_F8s": 0.454000, + "Standard_F8s_v2": 0.388000, + "Standard_L16s_v2": 1.488000, + "Standard_L16s_v3": 1.664000, + "Standard_L32s_v2": 2.976000, + "Standard_L32s_v3": 3.328000, + "Standard_L48s_v2": 4.464000, + "Standard_L48s_v3": 4.992000, + "Standard_L64s_v2": 5.952000, + "Standard_L64s_v3": 6.656000, + "Standard_L80s_v2": 7.440000, + "Standard_L80s_v3": 8.320000, + "Standard_L8s_v2": 0.744000, + "Standard_L8s_v3": 0.832000, + "Standard_M128": 18.674000, + "Standard_M128-32ms": 37.344000, + "Standard_M128-64ms": 37.344000, + "Standard_M128dms_v2": 37.366000, + "Standard_M128ds_v2": 18.676000, + "Standard_M128m": 37.344000, + "Standard_M128ms": 37.344000, + "Standard_M128ms_v2": 36.777000, + "Standard_M128s": 18.674000, + "Standard_M128s_v2": 18.087000, + "Standard_M16-4ms": 4.302000, + "Standard_M16-8ms": 4.302000, + "Standard_M16ms": 4.302000, + "Standard_M16s": 3.342000, + "Standard_M192idms_v2": 44.889000, + "Standard_M192ids_v2": 22.445000, + "Standard_M192ims_v2": 44.300000, + "Standard_M192is_v2": 21.856000, + "Standard_M208-104ms_v2": 53.544000, + "Standard_M208-104s_v2": 26.772000, + "Standard_M208-52ms_v2": 53.544000, + "Standard_M208-52s_v2": 26.772000, + "Standard_M208ms_v2": 53.544000, + "Standard_M208s_v2": 26.772000, + "Standard_M32-16ms": 8.604000, + "Standard_M32-8ms": 8.604000, + "Standard_M32dms_v2": 8.604000, + "Standard_M32ls": 3.448000, + "Standard_M32ms": 8.604000, + "Standard_M32ms_v2": 8.457000, + "Standard_M32s": 4.669000, + "Standard_M32ts": 3.248000, + "Standard_M416-104ms_v2": 118.980000, + "Standard_M416-104s_v2": 59.496000, + "Standard_M416-208ms_v2": 118.980000, + "Standard_M416-208s_v2": 59.496000, + "Standard_M416is_v2": 59.496000, + "Standard_M416ms_v2": 118.980000, + "Standard_M416s_8_v2": 79.750000, + "Standard_M416s_v2": 59.496000, + "Standard_M64": 9.337000, + "Standard_M64-16ms": 14.472000, + "Standard_M64-32ms": 14.472000, + "Standard_M64dms_v2": 14.476000, + "Standard_M64ds_v2": 9.337000, + "Standard_M64ls": 6.498000, + "Standard_M64m": 14.472000, + "Standard_M64ms": 14.472000, + "Standard_M64ms_v2": 14.181000, + "Standard_M64s": 9.337000, + "Standard_M64s_v2": 9.042000, + "Standard_M8-2ms": 2.151000, + "Standard_M8-4ms": 2.151000, + "Standard_M8ms": 2.151000, + "Standard_NC16as_T4_v3": 1.504000, + "Standard_NC4as_T4_v3": 0.658000, + "Standard_NC64as_T4_v3": 5.428000, + "Standard_NC8as_T4_v3": 0.939000, + "Standard_NV12ads_A10_v5": 1.180000, + "Standard_NV18ads_A10_v5": 2.080000, + "Standard_NV36adms_A10_v5": 5.876000, + "Standard_NV36ads_A10_v5": 4.160000, + "Standard_NV6ads_A10_v5": 0.590000, + "Standard_NV72ads_A10_v5": 8.476000, + } + // japaneast + initialOnDemandPrices["japaneast"] = map[string]float64{ + "Basic_A0": 0.022000, + "Basic_A1": 0.032000, + "Basic_A2": 0.109000, + "Basic_A3": 0.276000, + "Basic_A4": 0.552000, + "DCdsv3 Type1": 7.709000, + "DCsv3 Type1": 6.547000, + "Dadsv5_Type1": 8.254000, + "Dasv4_Type1": 6.547000, + "Dasv4_Type2": 6.820000, + "Dasv5_Type1": 6.899000, + "Ddsv4_Type 1": 5.139000, + "Ddsv4_Type2": 6.103000, + "Ddsv5_Type1": 7.709000, + "Dsv3_Type3": 5.456000, + "Dsv3_Type4": 6.820000, + "Dsv4_Type1": 5.456000, + "Dsv4_Type2": 6.547000, + "Dsv5_Type1": 6.820000, + "Eadsv5_Type1": 8.395000, + "Easv4_Type1": 8.025000, + "Easv4_Type2": 8.026000, + "Easv5_Type1": 7.234000, + "Ebdsv5-Type1": 7.146000, + "Ebsv5-Type1": 6.371000, + "Edsv4_Type 1": 6.125000, + "Edsv4_Type2": 7.273000, + "Edsv5_Type1": 9.187000, + "Esv3_Type3": 5.182000, + "Esv3_Type4": 7.021000, + "Esv4_Type1": 5.350000, + "Esv4_Type2": 7.022000, + "Esv5_Type1": 8.026000, + "FXmds Type1": 5.993000, + "Fsv2 Type3": 4.943000, + "Fsv2_Type2": 4.237000, + "Fsv2_Type4": 5.650000, + "Lsv2_Type1": 8.052000, + "Lsv3_Type1": 8.998000, + "Mdmsv2MedMem _Type1": 42.571000, + "Mdsv2MedMem_Type1": 21.277000, + "Ms_Type1": 21.270000, + "Msm_Type1": 42.577000, + "Msmv2MedMem Type1": 41.899000, + "Msmv2_Type1": 130.880000, + "Msv2MedMem Type1": 20.606000, + "Msv2_Type1": 65.446000, + "NVasv4_Type1": 10.406000, + "NVsv3_Type1": 6.952000, + "Standard_A0": 0.024000, + "Standard_A1": 0.081000, + "Standard_A1_v2": 0.054000, + "Standard_A2": 0.162000, + "Standard_A2_v2": 0.113000, + "Standard_A2m_v2": 0.153000, + "Standard_A3": 0.324000, + "Standard_A4": 0.648000, + "Standard_A4_v2": 0.238000, + "Standard_A4m_v2": 0.322000, + "Standard_A5": 0.281000, + "Standard_A6": 0.562000, + "Standard_A7": 1.124000, + "Standard_A8_v2": 0.500000, + "Standard_A8m_v2": 0.677000, + "Standard_B12ms": 0.653000, + "Standard_B16als_v2": 0.768000, + "Standard_B16as_v2": 0.858000, + "Standard_B16ls_v2": 0.844000, + "Standard_B16ms": 0.870000, + "Standard_B16pls_v2": 0.686000, + "Standard_B16ps_v2": 0.765000, + "Standard_B16s_v2": 0.870000, + "Standard_B1ls": 0.006800, + "Standard_B1ms": 0.027200, + "Standard_B1s": 0.013600, + "Standard_B20ms": 1.088000, + "Standard_B2als_v2": 0.049000, + "Standard_B2as_v2": 0.098000, + "Standard_B2ats_v2": 0.012300, + "Standard_B2ls_v2": 0.054400, + "Standard_B2ms": 0.109000, + "Standard_B2pls_v2": 0.052400, + "Standard_B2ps_v2": 0.095600, + "Standard_B2pts_v2": 0.020000, + "Standard_B2s": 0.054400, + "Standard_B2s_v2": 0.109000, + "Standard_B2ts_v2": 0.022800, + "Standard_B32als_v2": 1.536000, + "Standard_B32as_v2": 1.715000, + "Standard_B32ls_v2": 1.689000, + "Standard_B32s_v2": 1.888000, + "Standard_B4als_v2": 0.174000, + "Standard_B4as_v2": 0.214000, + "Standard_B4ls_v2": 0.211000, + "Standard_B4ms": 0.218000, + "Standard_B4pls_v2": 0.171000, + "Standard_B4ps_v2": 0.191000, + "Standard_B4s_v2": 0.236000, + "Standard_B8als_v2": 0.384000, + "Standard_B8as_v2": 0.429000, + "Standard_B8ls_v2": 0.422000, + "Standard_B8ms": 0.435000, + "Standard_B8pls_v2": 0.343000, + "Standard_B8ps_v2": 0.382000, + "Standard_B8s_v2": 0.472000, + "Standard_D1": 0.110000, + "Standard_D11": 0.242000, + "Standard_D11_v2": 0.229000, + "Standard_D11_v2_Promo": 0.229000, + "Standard_D12": 0.483000, + "Standard_D12_v2": 0.459000, + "Standard_D12_v2_Promo": 0.459000, + "Standard_D13": 0.966000, + "Standard_D13_v2": 0.918000, + "Standard_D13_v2_Promo": 0.918000, + "Standard_D14": 1.932000, + "Standard_D14_v2": 1.835000, + "Standard_D14_v2_Promo": 1.835000, + "Standard_D15_v2": 2.294000, + "Standard_D15i_v2": 2.294000, + "Standard_D16_v3": 1.032000, + "Standard_D16_v4": 0.992000, + "Standard_D16_v5": 0.992000, + "Standard_D16a_v4": 0.992000, + "Standard_D16ads_v5": 1.808000, + "Standard_D16as_v4": 0.992000, + "Standard_D16as_v5": 1.632000, + "Standard_D16as_v5_Promo": 0.792000, + "Standard_D16d_v4": 1.168000, + "Standard_D16d_v5": 1.168000, + "Standard_D16ds_v4": 1.168000, + "Standard_D16ds_v5": 1.168000, + "Standard_D16lds_v5": 0.976000, + "Standard_D16ls_v5": 0.856000, + "Standard_D16pds_v5": 0.936000, + "Standard_D16plds_v5": 0.784000, + "Standard_D16pls_v5": 0.685000, + "Standard_D16ps_v5": 0.792000, + "Standard_D16s_v3": 1.032000, + "Standard_D16s_v4": 0.992000, + "Standard_D16s_v5": 0.992000, + "Standard_D1_v2": 0.102000, + "Standard_D2": 0.221000, + "Standard_D2_v2": 0.205000, + "Standard_D2_v2_Promo": 0.205000, + "Standard_D2_v3": 0.129000, + "Standard_D2_v4": 0.124000, + "Standard_D2_v5": 0.124000, + "Standard_D2a_v4": 0.124000, + "Standard_D2ads_v5": 0.134000, + "Standard_D2as_v4": 0.124000, + "Standard_D2as_v5": 0.204000, + "Standard_D2as_v5_Promo": 0.099000, + "Standard_D2d_v4": 0.146000, + "Standard_D2d_v5": 0.146000, + "Standard_D2ds_v4": 0.146000, + "Standard_D2ds_v5": 0.146000, + "Standard_D2lds_v5": 0.122000, + "Standard_D2ls_v5": 0.107000, + "Standard_D2pds_v5": 0.117000, + "Standard_D2plds_v5": 0.098000, + "Standard_D2pls_v5": 0.085600, + "Standard_D2ps_v5": 0.099000, + "Standard_D2s_v3": 0.129000, + "Standard_D2s_v4": 0.124000, + "Standard_D2s_v5": 0.124000, + "Standard_D3": 0.442000, + "Standard_D32-16s_v3": 2.064000, + "Standard_D32-8s_v3": 2.064000, + "Standard_D32_v3": 2.064000, + "Standard_D32_v4": 1.984000, + "Standard_D32_v5": 1.984000, + "Standard_D32a_v4": 1.984000, + "Standard_D32ads_v5": 2.144000, + "Standard_D32as_v4": 1.984000, + "Standard_D32as_v5": 3.264000, + "Standard_D32as_v5_Promo": 1.584000, + "Standard_D32d_v4": 2.336000, + "Standard_D32d_v5": 2.336000, + "Standard_D32ds_v4": 2.336000, + "Standard_D32ds_v5": 2.336000, + "Standard_D32lds_v5": 1.952000, + "Standard_D32ls_v5": 1.712000, + "Standard_D32pds_v5": 1.872000, + "Standard_D32plds_v5": 1.568000, + "Standard_D32pls_v5": 1.370000, + "Standard_D32ps_v5": 1.584000, + "Standard_D32s_v3": 2.064000, + "Standard_D32s_v4": 1.984000, + "Standard_D32s_v5": 1.984000, + "Standard_D3_v2": 0.409000, + "Standard_D3_v2_Promo": 0.409000, + "Standard_D4": 0.883000, + "Standard_D48_v3": 3.096000, + "Standard_D48_v4": 2.976000, + "Standard_D48_v5": 2.976000, + "Standard_D48a_v4": 2.976000, + "Standard_D48ads_v5": 5.424000, + "Standard_D48as_v4": 2.976000, + "Standard_D48as_v5": 4.896000, + "Standard_D48as_v5_Promo": 2.376000, + "Standard_D48d_v4": 3.504000, + "Standard_D48d_v5": 3.504000, + "Standard_D48ds_v4": 3.504000, + "Standard_D48ds_v5": 3.504000, + "Standard_D48lds_v5": 2.928000, + "Standard_D48ls_v5": 2.568000, + "Standard_D48pds_v5": 2.808000, + "Standard_D48plds_v5": 2.352000, + "Standard_D48pls_v5": 2.054000, + "Standard_D48ps_v5": 2.376000, + "Standard_D48s_v3": 3.096000, + "Standard_D48s_v4": 2.976000, + "Standard_D48s_v5": 2.976000, + "Standard_D4_v2": 0.818000, + "Standard_D4_v2_Promo": 0.818000, + "Standard_D4_v3": 0.258000, + "Standard_D4_v4": 0.248000, + "Standard_D4_v5": 0.248000, + "Standard_D4a_v4": 0.248000, + "Standard_D4ads_v5": 0.268000, + "Standard_D4as_v4": 0.248000, + "Standard_D4as_v5": 0.408000, + "Standard_D4as_v5_Promo": 0.198000, + "Standard_D4d_v4": 0.292000, + "Standard_D4d_v5": 0.292000, + "Standard_D4ds_v4": 0.292000, + "Standard_D4ds_v5": 0.292000, + "Standard_D4lds_v5": 0.244000, + "Standard_D4ls_v5": 0.214000, + "Standard_D4pds_v5": 0.234000, + "Standard_D4plds_v5": 0.196000, + "Standard_D4pls_v5": 0.171000, + "Standard_D4ps_v5": 0.198000, + "Standard_D4s_v3": 0.258000, + "Standard_D4s_v4": 0.248000, + "Standard_D4s_v5": 0.248000, + "Standard_D5_v2": 1.636000, + "Standard_D5_v2_Promo": 1.636000, + "Standard_D64-16s_v3": 4.128000, + "Standard_D64-32s_v3": 4.128000, + "Standard_D64_v3": 4.128000, + "Standard_D64_v4": 3.968000, + "Standard_D64_v5": 3.968000, + "Standard_D64a_v4": 3.968000, + "Standard_D64ads_v5": 4.288000, + "Standard_D64as_v4": 3.968000, + "Standard_D64as_v5": 6.528000, + "Standard_D64as_v5_Promo": 3.168000, + "Standard_D64d_v4": 4.672000, + "Standard_D64d_v5": 4.672000, + "Standard_D64ds_v4": 4.672000, + "Standard_D64ds_v5": 4.672000, + "Standard_D64lds_v5": 3.904000, + "Standard_D64ls_v5": 3.424000, + "Standard_D64pds_v5": 3.744000, + "Standard_D64plds_v5": 3.136000, + "Standard_D64pls_v5": 2.739000, + "Standard_D64ps_v5": 3.168000, + "Standard_D64s_v3": 4.128000, + "Standard_D64s_v4": 3.968000, + "Standard_D64s_v5": 3.968000, + "Standard_D8_v3": 0.516000, + "Standard_D8_v4": 0.496000, + "Standard_D8_v5": 0.496000, + "Standard_D8a_v4": 0.496000, + "Standard_D8ads_v5": 0.904000, + "Standard_D8as_v4": 0.496000, + "Standard_D8as_v5": 0.448000, + "Standard_D8as_v5_Promo": 0.396000, + "Standard_D8d_v4": 0.584000, + "Standard_D8d_v5": 0.584000, + "Standard_D8ds_v4": 0.584000, + "Standard_D8ds_v5": 0.584000, + "Standard_D8lds_v5": 0.488000, + "Standard_D8ls_v5": 0.428000, + "Standard_D8pds_v5": 0.468000, + "Standard_D8plds_v5": 0.392000, + "Standard_D8pls_v5": 0.342000, + "Standard_D8ps_v5": 0.396000, + "Standard_D8s_v3": 0.516000, + "Standard_D8s_v4": 0.496000, + "Standard_D8s_v5": 0.496000, + "Standard_D96_v5": 5.952000, + "Standard_D96a_v4": 5.952000, + "Standard_D96ads_v5": 6.432000, + "Standard_D96as_v4": 5.952000, + "Standard_D96as_v5": 9.792000, + "Standard_D96as_v5_Promo": 4.752000, + "Standard_D96d_v5": 7.008000, + "Standard_D96ds_v5": 7.008000, + "Standard_D96lds_v5": 5.856000, + "Standard_D96ls_v5": 5.136000, + "Standard_D96s_v5": 5.952000, + "Standard_DC16ads_v5": 1.072000, + "Standard_DC16as_v5": 0.896000, + "Standard_DC16ds_v3": 2.336000, + "Standard_DC16s_v3": 1.984000, + "Standard_DC1ds_v3": 0.146000, + "Standard_DC1s_v3": 0.124000, + "Standard_DC24ds_v3": 3.504000, + "Standard_DC24s_v3": 2.976000, + "Standard_DC2ads_v5": 0.134000, + "Standard_DC2as_v5": 0.112000, + "Standard_DC2ds_v3": 0.292000, + "Standard_DC2s_v3": 0.248000, + "Standard_DC32ads_v5": 2.144000, + "Standard_DC32as_v5": 1.792000, + "Standard_DC32ds_v3": 4.672000, + "Standard_DC32s_v3": 3.968000, + "Standard_DC48ads_v5": 3.216000, + "Standard_DC48as_v5": 2.688000, + "Standard_DC48ds_v3": 7.008000, + "Standard_DC48s_v3": 5.952000, + "Standard_DC4ads_v5": 0.268000, + "Standard_DC4as_v5": 0.224000, + "Standard_DC4ds_v3": 0.584000, + "Standard_DC4s_v3": 0.496000, + "Standard_DC64ads_v5": 4.288000, + "Standard_DC64as_v5": 3.584000, + "Standard_DC8ads_v5": 0.536000, + "Standard_DC8as_v5": 0.448000, + "Standard_DC8ds_v3": 1.168000, + "Standard_DC8s_v3": 0.992000, + "Standard_DC96ads_v5": 6.432000, + "Standard_DC96as_v5": 5.376000, + "Standard_DS1": 0.110000, + "Standard_DS11": 0.242000, + "Standard_DS11-1_v2": 0.229000, + "Standard_DS11_v2": 0.229000, + "Standard_DS11_v2_Promo": 0.229000, + "Standard_DS12": 0.483000, + "Standard_DS12-1_v2": 0.459000, + "Standard_DS12-2_v2": 0.459000, + "Standard_DS12_v2": 0.459000, + "Standard_DS12_v2_Promo": 0.459000, + "Standard_DS13": 0.966000, + "Standard_DS13-2_v2": 0.918000, + "Standard_DS13-4_v2": 0.918000, + "Standard_DS13_v2": 0.918000, + "Standard_DS13_v2_Promo": 0.918000, + "Standard_DS14": 1.932000, + "Standard_DS14-4_v2": 1.835000, + "Standard_DS14-8_v2": 1.835000, + "Standard_DS14_v2": 1.835000, + "Standard_DS14_v2_Promo": 1.835000, + "Standard_DS15_v2": 2.294000, + "Standard_DS15i_v2": 2.294000, + "Standard_DS1_v2": 0.102000, + "Standard_DS2": 0.221000, + "Standard_DS2_v2": 0.205000, + "Standard_DS2_v2_Promo": 0.205000, + "Standard_DS3": 0.442000, + "Standard_DS3_v2": 0.409000, + "Standard_DS3_v2_Promo": 0.409000, + "Standard_DS4": 0.883000, + "Standard_DS4_v2": 0.818000, + "Standard_DS4_v2_Promo": 0.818000, + "Standard_DS5_v2": 1.636000, + "Standard_DS5_v2_Promo": 1.636000, + "Standard_E104i_v5": 8.694000, + "Standard_E104id_v5": 9.953000, + "Standard_E104ids_v5": 9.953000, + "Standard_E104is_v5": 8.694000, + "Standard_E112iads_v5": 9.794000, + "Standard_E112ias_v5": 8.439000, + "Standard_E112ibds_v5": 12.505000, + "Standard_E112ibs_v5": 11.150000, + "Standard_E16-4ads_v5": 2.008000, + "Standard_E16-4as_v4": 1.216000, + "Standard_E16-4as_v5": 1.096000, + "Standard_E16-4ds_v4": 1.392000, + "Standard_E16-4ds_v5": 1.392000, + "Standard_E16-4s_v3": 1.280000, + "Standard_E16-4s_v4": 1.216000, + "Standard_E16-4s_v5": 1.216000, + "Standard_E16-8ads_v5": 1.272000, + "Standard_E16-8as_v4": 1.216000, + "Standard_E16-8as_v5": 1.096000, + "Standard_E16-8ds_v4": 1.392000, + "Standard_E16-8ds_v5": 1.392000, + "Standard_E16-8s_v3": 1.280000, + "Standard_E16-8s_v4": 1.216000, + "Standard_E16-8s_v5": 1.216000, + "Standard_E16_v3": 1.280000, + "Standard_E16_v4": 1.216000, + "Standard_E16_v5": 1.216000, + "Standard_E16a_v4": 1.216000, + "Standard_E16ads_v5": 1.272000, + "Standard_E16as_v4": 1.216000, + "Standard_E16as_v5": 1.142000, + "Standard_E16bds_v5": 1.624000, + "Standard_E16bs_v5": 1.448000, + "Standard_E16d_v4": 1.392000, + "Standard_E16d_v5": 1.392000, + "Standard_E16ds_v4": 1.392000, + "Standard_E16ds_v5": 1.392000, + "Standard_E16pds_v5": 1.112000, + "Standard_E16ps_v5": 0.973000, + "Standard_E16s_v3": 1.280000, + "Standard_E16s_v4": 1.216000, + "Standard_E16s_v5": 1.216000, + "Standard_E20_v3": 1.600000, + "Standard_E20_v4": 1.520000, + "Standard_E20_v5": 1.520000, + "Standard_E20a_v4": 1.520000, + "Standard_E20ads_v5": 1.590000, + "Standard_E20as_v4": 1.520000, + "Standard_E20as_v5": 1.370000, + "Standard_E20d_v4": 1.740000, + "Standard_E20d_v5": 1.740000, + "Standard_E20ds_v4": 1.740000, + "Standard_E20ds_v5": 1.740000, + "Standard_E20pds_v5": 1.390000, + "Standard_E20ps_v5": 1.216000, + "Standard_E20s_v3": 1.600000, + "Standard_E20s_v4": 1.520000, + "Standard_E20s_v5": 1.520000, + "Standard_E2_v3": 0.160000, + "Standard_E2_v4": 0.152000, + "Standard_E2_v5": 0.152000, + "Standard_E2a_v4": 0.152000, + "Standard_E2ads_v5": 0.205000, + "Standard_E2as_v4": 0.152000, + "Standard_E2as_v5": 0.137000, + "Standard_E2bds_v5": 0.203000, + "Standard_E2bs_v5": 0.181000, + "Standard_E2d_v4": 0.174000, + "Standard_E2d_v5": 0.174000, + "Standard_E2ds_v4": 0.174000, + "Standard_E2ds_v5": 0.174000, + "Standard_E2pds_v5": 0.139000, + "Standard_E2ps_v5": 0.122000, + "Standard_E2s_v3": 0.160000, + "Standard_E2s_v4": 0.152000, + "Standard_E2s_v5": 0.152000, + "Standard_E32-16ads_v5": 2.544000, + "Standard_E32-16as_v4": 2.432000, + "Standard_E32-16as_v5": 2.192000, + "Standard_E32-16ds_v4": 2.784000, + "Standard_E32-16ds_v5": 2.784000, + "Standard_E32-16s_v3": 2.560000, + "Standard_E32-16s_v4": 2.432000, + "Standard_E32-16s_v5": 2.432000, + "Standard_E32-8ads_v5": 2.544000, + "Standard_E32-8as_v4": 2.432000, + "Standard_E32-8as_v5": 2.192000, + "Standard_E32-8ds_v4": 2.784000, + "Standard_E32-8ds_v5": 2.784000, + "Standard_E32-8s_v3": 2.560000, + "Standard_E32-8s_v4": 2.432000, + "Standard_E32-8s_v5": 2.432000, + "Standard_E32_v3": 2.560000, + "Standard_E32_v4": 2.432000, + "Standard_E32_v5": 2.432000, + "Standard_E32a_v4": 2.432000, + "Standard_E32ads_v5": 2.590000, + "Standard_E32as_v4": 2.432000, + "Standard_E32as_v5": 2.238000, + "Standard_E32bds_v5": 3.248000, + "Standard_E32bs_v5": 2.896000, + "Standard_E32d_v4": 2.784000, + "Standard_E32d_v5": 2.784000, + "Standard_E32ds_v4": 2.784000, + "Standard_E32ds_v5": 2.784000, + "Standard_E32pds_v5": 2.224000, + "Standard_E32ps_v5": 1.946000, + "Standard_E32s_v3": 2.560000, + "Standard_E32s_v4": 2.432000, + "Standard_E32s_v5": 2.432000, + "Standard_E4-2ads_v5": 0.318000, + "Standard_E4-2as_v4": 0.304000, + "Standard_E4-2as_v5": 0.458000, + "Standard_E4-2ds_v4": 0.348000, + "Standard_E4-2ds_v5": 0.348000, + "Standard_E4-2s_v3": 0.320000, + "Standard_E4-2s_v4": 0.304000, + "Standard_E4-2s_v5": 0.304000, + "Standard_E48_v3": 3.647000, + "Standard_E48_v4": 3.648000, + "Standard_E48_v5": 3.648000, + "Standard_E48a_v4": 3.648000, + "Standard_E48ads_v5": 3.816000, + "Standard_E48as_v4": 3.648000, + "Standard_E48as_v5": 3.288000, + "Standard_E48bds_v5": 4.872000, + "Standard_E48bs_v5": 4.344000, + "Standard_E48d_v4": 4.176000, + "Standard_E48d_v5": 4.176000, + "Standard_E48ds_v4": 4.176000, + "Standard_E48ds_v5": 4.176000, + "Standard_E48s_v3": 3.647000, + "Standard_E48s_v4": 3.648000, + "Standard_E48s_v5": 3.648000, + "Standard_E4_v3": 0.320000, + "Standard_E4_v4": 0.304000, + "Standard_E4_v5": 0.304000, + "Standard_E4a_v4": 0.304000, + "Standard_E4ads_v5": 0.318000, + "Standard_E4as_v4": 0.304000, + "Standard_E4as_v5": 0.274000, + "Standard_E4bds_v5": 0.406000, + "Standard_E4bs_v5": 0.362000, + "Standard_E4d_v4": 0.348000, + "Standard_E4d_v5": 0.348000, + "Standard_E4ds_v4": 0.348000, + "Standard_E4ds_v5": 0.348000, + "Standard_E4pds_v5": 0.278000, + "Standard_E4ps_v5": 0.243000, + "Standard_E4s_v3": 0.320000, + "Standard_E4s_v4": 0.304000, + "Standard_E4s_v5": 0.304000, + "Standard_E64-16ads_v5": 5.088000, + "Standard_E64-16as_v4": 4.864000, + "Standard_E64-16as_v5": 7.328000, + "Standard_E64-16ds_v4": 5.568000, + "Standard_E64-16ds_v5": 5.568000, + "Standard_E64-16s_v3": 4.376000, + "Standard_E64-16s_v4": 4.864000, + "Standard_E64-16s_v5": 4.864000, + "Standard_E64-32ads_v5": 8.032000, + "Standard_E64-32as_v4": 4.864000, + "Standard_E64-32as_v5": 7.328000, + "Standard_E64-32ds_v4": 5.568000, + "Standard_E64-32ds_v5": 5.568000, + "Standard_E64-32s_v3": 4.376000, + "Standard_E64-32s_v4": 4.864000, + "Standard_E64-32s_v5": 4.864000, + "Standard_E64_v3": 4.376000, + "Standard_E64_v4": 4.864000, + "Standard_E64_v5": 4.864000, + "Standard_E64a_v4": 4.864000, + "Standard_E64ads_v5": 5.088000, + "Standard_E64as_v4": 4.864000, + "Standard_E64as_v5": 4.384000, + "Standard_E64bds_v5": 6.496000, + "Standard_E64bs_v5": 5.792000, + "Standard_E64d_v4": 5.568000, + "Standard_E64d_v5": 5.568000, + "Standard_E64ds_v4": 5.568000, + "Standard_E64ds_v5": 5.568000, + "Standard_E64i_v3": 4.376000, + "Standard_E64is_v3": 4.376000, + "Standard_E64s_v3": 4.376000, + "Standard_E64s_v4": 4.864000, + "Standard_E64s_v5": 4.864000, + "Standard_E8-2ads_v5": 0.636000, + "Standard_E8-2as_v4": 0.608000, + "Standard_E8-2as_v5": 0.548000, + "Standard_E8-2ds_v4": 0.696000, + "Standard_E8-2ds_v5": 0.696000, + "Standard_E8-2s_v3": 0.640000, + "Standard_E8-2s_v4": 0.608000, + "Standard_E8-2s_v5": 0.608000, + "Standard_E8-4ads_v5": 1.004000, + "Standard_E8-4as_v4": 0.608000, + "Standard_E8-4as_v5": 0.916000, + "Standard_E8-4ds_v4": 0.696000, + "Standard_E8-4ds_v5": 0.696000, + "Standard_E8-4s_v3": 0.640000, + "Standard_E8-4s_v4": 0.608000, + "Standard_E8-4s_v5": 0.608000, + "Standard_E80ids_v4": 6.960000, + "Standard_E80is_v4": 6.080000, + "Standard_E8_v3": 0.640000, + "Standard_E8_v4": 0.608000, + "Standard_E8_v5": 0.608000, + "Standard_E8a_v4": 0.608000, + "Standard_E8ads_v5": 0.682000, + "Standard_E8as_v4": 0.608000, + "Standard_E8as_v5": 0.594000, + "Standard_E8bds_v5": 0.812000, + "Standard_E8bs_v5": 0.724000, + "Standard_E8d_v4": 0.696000, + "Standard_E8d_v5": 0.696000, + "Standard_E8ds_v4": 0.696000, + "Standard_E8ds_v5": 0.696000, + "Standard_E8pds_v5": 0.556000, + "Standard_E8ps_v5": 0.486000, + "Standard_E8s_v3": 0.640000, + "Standard_E8s_v4": 0.608000, + "Standard_E8s_v5": 0.608000, + "Standard_E96-24ads_v5": 12.048000, + "Standard_E96-24as_v4": 7.296000, + "Standard_E96-24as_v5": 6.576000, + "Standard_E96-24ds_v5": 8.352000, + "Standard_E96-24s_v5": 7.296000, + "Standard_E96-48ads_v5": 12.048000, + "Standard_E96-48as_v4": 7.296000, + "Standard_E96-48as_v5": 6.576000, + "Standard_E96-48ds_v5": 8.352000, + "Standard_E96-48s_v5": 7.296000, + "Standard_E96_v5": 7.296000, + "Standard_E96a_v4": 7.296000, + "Standard_E96ads_v5": 7.632000, + "Standard_E96as_v4": 7.296000, + "Standard_E96as_v5": 6.576000, + "Standard_E96bds_v5": 9.744000, + "Standard_E96bs_v5": 8.688000, + "Standard_E96d_v5": 8.352000, + "Standard_E96ds_v5": 8.352000, + "Standard_E96iads_v5": 8.395000, + "Standard_E96ias_v5": 7.234000, + "Standard_E96s_v5": 7.296000, + "Standard_EC16ads_v5": 1.272000, + "Standard_EC16as_v5": 1.096000, + "Standard_EC20ads_v5": 1.590000, + "Standard_EC20as_v5": 1.370000, + "Standard_EC2ads_v5": 0.159000, + "Standard_EC2as_v5": 0.137000, + "Standard_EC32ads_v5": 2.544000, + "Standard_EC32as_v5": 2.192000, + "Standard_EC48ads_v5": 3.816000, + "Standard_EC48as_v5": 3.288000, + "Standard_EC4ads_v5": 0.318000, + "Standard_EC4as_v5": 0.274000, + "Standard_EC64ads_v5": 5.088000, + "Standard_EC64as_v5": 4.384000, + "Standard_EC8ads_v5": 0.636000, + "Standard_EC8as_v5": 0.548000, + "Standard_EC96ads_v5": 7.632000, + "Standard_EC96as_v5": 6.576000, + "Standard_EC96iads_v5": 8.395000, + "Standard_EC96ias_v5": 7.234000, + "Standard_F1": 0.063000, + "Standard_F16": 1.008000, + "Standard_F16s": 1.008000, + "Standard_F16s_v2": 0.856000, + "Standard_F1s": 0.063000, + "Standard_F2": 0.126000, + "Standard_F2s": 0.126000, + "Standard_F2s_v2": 0.107000, + "Standard_F32s_v2": 1.712000, + "Standard_F4": 0.252000, + "Standard_F48s_v2": 2.568000, + "Standard_F4s": 0.252000, + "Standard_F4s_v2": 0.214000, + "Standard_F64s_v2": 3.424000, + "Standard_F72s_v2": 3.852000, + "Standard_F8": 0.504000, + "Standard_F8s": 0.504000, + "Standard_F8s_v2": 0.428000, + "Standard_FX12mds": 1.362000, + "Standard_FX24mds": 2.724000, + "Standard_FX36mds": 4.086000, + "Standard_FX48mds": 5.448000, + "Standard_FX4mds": 0.454000, + "Standard_G1": 0.576000, + "Standard_G2": 1.151000, + "Standard_G3": 2.302000, + "Standard_G4": 4.602000, + "Standard_G5": 9.205000, + "Standard_GS1": 0.576000, + "Standard_GS2": 1.151000, + "Standard_GS3": 2.302000, + "Standard_GS4": 4.602000, + "Standard_GS4-4": 4.602000, + "Standard_GS4-8": 4.602000, + "Standard_GS5": 9.205000, + "Standard_GS5-16": 9.205000, + "Standard_GS5-8": 9.205000, + "Standard_H16": 2.291000, + "Standard_H16m": 3.070000, + "Standard_H16mr": 3.377000, + "Standard_H16r": 2.520000, + "Standard_H8": 1.145000, + "Standard_H8m": 1.535000, + "Standard_HB120-16rs_v2": 5.220000, + "Standard_HB120-16rs_v3": 5.220000, + "Standard_HB120-32rs_v2": 5.220000, + "Standard_HB120-32rs_v3": 5.220000, + "Standard_HB120-64rs_v2": 5.220000, + "Standard_HB120-64rs_v3": 5.220000, + "Standard_HB120-96rs_v2": 5.220000, + "Standard_HB120-96rs_v3": 5.220000, + "Standard_HB120rs_v2": 5.220000, + "Standard_HB120rs_v3": 5.220000, + "Standard_HC44-16rs": 4.594000, + "Standard_HC44-32rs": 4.594000, + "Standard_HC44rs": 4.594000, + "Standard_L112ias_v3": 11.273000, + "Standard_L16as_v3": 1.464000, + "Standard_L16s": 1.464000, + "Standard_L16s_v2": 1.464000, + "Standard_L16s_v3": 1.636000, + "Standard_L32as_v3": 2.928000, + "Standard_L32s": 2.928000, + "Standard_L32s_v2": 2.928000, + "Standard_L32s_v3": 3.272000, + "Standard_L48as_v3": 4.392000, + "Standard_L48s_v2": 4.392000, + "Standard_L48s_v3": 4.908000, + "Standard_L4s": 0.366000, + "Standard_L64as_v3": 5.856000, + "Standard_L64s_v2": 5.856000, + "Standard_L64s_v3": 6.544000, + "Standard_L80as_v3": 7.320000, + "Standard_L80s_v2": 7.320000, + "Standard_L80s_v3": 8.180000, + "Standard_L88is_v2": 8.052000, + "Standard_L8as_v3": 0.732000, + "Standard_L8s": 0.732000, + "Standard_L8s_v2": 0.732000, + "Standard_L8s_v3": 0.818000, + "Standard_L96ias_v3": 9.662000, + "Standard_M128": 19.340000, + "Standard_M128-32ms": 38.698000, + "Standard_M128-64ms": 38.698000, + "Standard_M128dms_v2": 38.701000, + "Standard_M128ds_v2": 19.343000, + "Standard_M128m": 38.698000, + "Standard_M128ms": 38.698000, + "Standard_M128ms_v2": 38.090000, + "Standard_M128s": 19.340000, + "Standard_M128s_v2": 18.733000, + "Standard_M16-4ms": 4.455900, + "Standard_M16-8ms": 4.455900, + "Standard_M16ms": 4.455900, + "Standard_M16s": 3.461000, + "Standard_M192idms_v2": 46.493000, + "Standard_M192ids_v2": 23.246000, + "Standard_M192ims_v2": 45.882000, + "Standard_M192is_v2": 22.636000, + "Standard_M208-104ms_v2": 53.544000, + "Standard_M208-104s_v2": 26.772000, + "Standard_M208-52ms_v2": 53.544000, + "Standard_M208-52s_v2": 26.772000, + "Standard_M208ms_v2": 53.544000, + "Standard_M208s_v2": 26.772000, + "Standard_M32-16ms": 8.911700, + "Standard_M32-8ms": 8.911700, + "Standard_M32dms_v2": 8.912000, + "Standard_M32ls": 3.447600, + "Standard_M32ms": 8.911700, + "Standard_M32ms_v2": 8.759000, + "Standard_M32s": 4.835800, + "Standard_M32ts": 3.248000, + "Standard_M416-104ms_v2": 118.980000, + "Standard_M416-104s_v2": 59.496000, + "Standard_M416-208ms_v2": 118.980000, + "Standard_M416-208s_v2": 59.496000, + "Standard_M416is_v2": 59.496000, + "Standard_M416ms_v2": 118.980000, + "Standard_M416s_8_v2": 79.750000, + "Standard_M416s_v2": 59.496000, + "Standard_M64": 9.670000, + "Standard_M64-16ms": 14.989000, + "Standard_M64-32ms": 14.989000, + "Standard_M64dms_v2": 14.993000, + "Standard_M64ds_v2": 9.670000, + "Standard_M64ls": 6.498000, + "Standard_M64m": 14.989000, + "Standard_M64ms": 14.989000, + "Standard_M64ms_v2": 14.688000, + "Standard_M64s": 9.670000, + "Standard_M64s_v2": 9.365000, + "Standard_M8-2ms": 2.227900, + "Standard_M8-4ms": 2.227900, + "Standard_M8ms": 2.227900, + "Standard_NC12s_v3": 8.388000, + "Standard_NC16as_T4_v3": 1.625000, + "Standard_NC24ads_A100_v4": 5.326000, + "Standard_NC24rs_v3": 18.454000, + "Standard_NC24s_v3": 16.776000, + "Standard_NC48ads_A100_v4": 10.652000, + "Standard_NC4as_T4_v3": 0.710000, + "Standard_NC64as_T4_v3": 5.875000, + "Standard_NC6s_v3": 4.194000, + "Standard_NC8as_T4_v3": 1.015000, + "Standard_NC96ads_A100_v4": 21.303000, + "Standard_ND96ams_A100_v4": 47.517000, + "Standard_ND96amsr_A100_v4": 47.517000, + "Standard_NP10s": 2.392000, + "Standard_NP20s": 4.785000, + "Standard_NP40s": 9.570000, + "Standard_NV12": 3.160000, + "Standard_NV12ads_A10_v5": 1.317000, + "Standard_NV12s_v3": 1.580000, + "Standard_NV16as_v4": 1.351000, + "Standard_NV18ads_A10_v5": 2.320000, + "Standard_NV24": 6.319000, + "Standard_NV24s_v3": 3.160000, + "Standard_NV32as_v4": 2.703000, + "Standard_NV36adms_A10_v5": 6.554000, + "Standard_NV36ads_A10_v5": 4.640000, + "Standard_NV48s_v3": 6.320000, + "Standard_NV4as_v4": 0.338000, + "Standard_NV6": 1.580000, + "Standard_NV6ads_A10_v5": 0.658000, + "Standard_NV72ads_A10_v5": 9.454000, + "Standard_NV8as_v4": 0.676000, + } + // japanwest + initialOnDemandPrices["japanwest"] = map[string]float64{ + "Basic_A0": 0.017100, + "Basic_A1": 0.032000, + "Basic_A2": 0.099000, + "Basic_A3": 0.219600, + "Basic_A4": 0.439200, + "DCdsv3 Type1": 7.709000, + "DCsv3 Type1": 6.547000, + "Dadsv5_Type1": 8.254000, + "Dasv4_Type1": 6.820000, + "Dasv4_Type2": 6.820000, + "Dasv5_Type1": 6.899000, + "Ddsv4_Type 1": 5.625000, + "Ddsv4_Type2": 6.680000, + "Ddsv5_Type1": 7.709000, + "Dsv3_Type3": 5.460000, + "Dsv3_Type4": 6.824000, + "Dsv4_Type1": 5.456000, + "Dsv4_Type2": 6.820000, + "Dsv5_Type1": 6.820000, + "Eadsv5_Type1": 8.395000, + "Easv4_Type1": 8.026000, + "Easv4_Type2": 8.026000, + "Easv5_Type1": 7.234000, + "Ebdsv5-Type1": 7.040000, + "Ebsv5-Type1": 6.266000, + "Edsv4_Type 1": 6.737000, + "Edsv4_Type2": 8.001000, + "Edsv5_Type1": 9.187000, + "Esv3_Type3": 5.192000, + "Esv3_Type4": 7.034000, + "Esv4_Type1": 5.351000, + "Esv4_Type2": 7.023000, + "Esv5_Type1": 8.026000, + "Fsv2 Type3": 5.465000, + "Fsv2_Type2": 4.277000, + "Fsv2_Type4": 5.940000, + "Lasv3_Type1": 8.052000, + "Lsv3_Type1": 8.998000, + "Mdmsv2MedMem _Type1": 42.571000, + "Mdsv2MedMem_Type1": 21.277000, + "Msmv2MedMem Type1": 41.899000, + "Msmv2_Type1": 169.050000, + "Msv2MedMem Type1": 20.606000, + "Msv2_Type1": 84.534000, + "Standard_A0": 0.021000, + "Standard_A1": 0.073000, + "Standard_A1_v2": 0.054000, + "Standard_A2": 0.146000, + "Standard_A2_v2": 0.113000, + "Standard_A2m_v2": 0.178000, + "Standard_A3": 0.292000, + "Standard_A4": 0.584000, + "Standard_A4_v2": 0.238000, + "Standard_A4m_v2": 0.374000, + "Standard_A5": 0.258000, + "Standard_A6": 0.516000, + "Standard_A7": 1.032000, + "Standard_A8_v2": 0.500000, + "Standard_A8m_v2": 0.786000, + "Standard_B12ms": 0.719000, + "Standard_B16als_v2": 0.694000, + "Standard_B16as_v2": 0.858000, + "Standard_B16ls_v2": 0.771000, + "Standard_B16ms": 0.958000, + "Standard_B16s_v2": 0.870000, + "Standard_B1ls": 0.007490, + "Standard_B1ms": 0.030000, + "Standard_B1s": 0.015000, + "Standard_B20ms": 1.198000, + "Standard_B2als_v2": 0.049000, + "Standard_B2as_v2": 0.107000, + "Standard_B2ats_v2": 0.012300, + "Standard_B2ls_v2": 0.063600, + "Standard_B2ms": 0.120000, + "Standard_B2s": 0.059900, + "Standard_B2s_v2": 0.109000, + "Standard_B2ts_v2": 0.022800, + "Standard_B32als_v2": 1.388000, + "Standard_B32as_v2": 1.568000, + "Standard_B32ls_v2": 1.689000, + "Standard_B32s_v2": 1.741000, + "Standard_B4als_v2": 0.192000, + "Standard_B4as_v2": 0.214000, + "Standard_B4ls_v2": 0.211000, + "Standard_B4ms": 0.240000, + "Standard_B4s_v2": 0.236000, + "Standard_B8als_v2": 0.384000, + "Standard_B8as_v2": 0.392000, + "Standard_B8ls_v2": 0.422000, + "Standard_B8ms": 0.479000, + "Standard_B8s_v2": 0.472000, + "Standard_D1": 0.096000, + "Standard_D11": 0.210000, + "Standard_D11_v2": 0.200000, + "Standard_D11_v2_Promo": 0.200000, + "Standard_D12": 0.420000, + "Standard_D12_v2": 0.399000, + "Standard_D12_v2_Promo": 0.399000, + "Standard_D13": 0.840000, + "Standard_D13_v2": 0.798000, + "Standard_D13_v2_Promo": 0.798000, + "Standard_D14": 1.680000, + "Standard_D14_v2": 1.596000, + "Standard_D14_v2_Promo": 1.596000, + "Standard_D15_v2": 1.995000, + "Standard_D15i_v2": 1.995000, + "Standard_D16_v3": 1.032000, + "Standard_D16_v4": 1.091000, + "Standard_D16_v5": 0.992000, + "Standard_D16a_v4": 0.992000, + "Standard_D16ads_v5": 1.808000, + "Standard_D16as_v4": 0.992000, + "Standard_D16as_v5": 1.632000, + "Standard_D16d_v4": 1.284000, + "Standard_D16d_v5": 1.168000, + "Standard_D16ds_v4": 1.284000, + "Standard_D16ds_v5": 1.168000, + "Standard_D16lds_v5": 0.976000, + "Standard_D16ls_v5": 0.856000, + "Standard_D16s_v3": 1.032000, + "Standard_D16s_v4": 1.091000, + "Standard_D16s_v5": 0.992000, + "Standard_D1_v2": 0.091000, + "Standard_D2": 0.192000, + "Standard_D2_v2": 0.182000, + "Standard_D2_v2_Promo": 0.182000, + "Standard_D2_v3": 0.129000, + "Standard_D2_v4": 0.136000, + "Standard_D2_v5": 0.124000, + "Standard_D2a_v4": 0.124000, + "Standard_D2ads_v5": 0.226000, + "Standard_D2as_v4": 0.124000, + "Standard_D2as_v5": 0.112000, + "Standard_D2d_v4": 0.161000, + "Standard_D2d_v5": 0.146000, + "Standard_D2ds_v4": 0.161000, + "Standard_D2ds_v5": 0.146000, + "Standard_D2lds_v5": 0.122000, + "Standard_D2ls_v5": 0.107000, + "Standard_D2s_v3": 0.129000, + "Standard_D2s_v4": 0.136000, + "Standard_D2s_v5": 0.124000, + "Standard_D3": 0.384000, + "Standard_D32-16s_v3": 2.064000, + "Standard_D32-8s_v3": 2.064000, + "Standard_D32_v3": 2.064000, + "Standard_D32_v4": 2.182000, + "Standard_D32_v5": 1.984000, + "Standard_D32a_v4": 1.984000, + "Standard_D32ads_v5": 2.144000, + "Standard_D32as_v4": 1.984000, + "Standard_D32as_v5": 3.264000, + "Standard_D32d_v4": 2.569000, + "Standard_D32d_v5": 2.336000, + "Standard_D32ds_v4": 2.569000, + "Standard_D32ds_v5": 2.336000, + "Standard_D32lds_v5": 1.952000, + "Standard_D32ls_v5": 1.712000, + "Standard_D32s_v3": 2.064000, + "Standard_D32s_v4": 2.182000, + "Standard_D32s_v5": 1.984000, + "Standard_D3_v2": 0.365000, + "Standard_D3_v2_Promo": 0.365000, + "Standard_D4": 0.768000, + "Standard_D48_v3": 3.096000, + "Standard_D48_v4": 3.274000, + "Standard_D48_v5": 2.976000, + "Standard_D48a_v4": 2.976000, + "Standard_D48ads_v5": 3.216000, + "Standard_D48as_v4": 2.976000, + "Standard_D48as_v5": 2.688000, + "Standard_D48d_v4": 3.853000, + "Standard_D48d_v5": 3.504000, + "Standard_D48ds_v4": 3.853000, + "Standard_D48ds_v5": 3.504000, + "Standard_D48lds_v5": 2.928000, + "Standard_D48ls_v5": 2.568000, + "Standard_D48s_v3": 3.096000, + "Standard_D48s_v4": 3.274000, + "Standard_D48s_v5": 2.976000, + "Standard_D4_v2": 0.729000, + "Standard_D4_v2_Promo": 0.729000, + "Standard_D4_v3": 0.258000, + "Standard_D4_v4": 0.273000, + "Standard_D4_v5": 0.248000, + "Standard_D4a_v4": 0.248000, + "Standard_D4ads_v5": 0.268000, + "Standard_D4as_v4": 0.248000, + "Standard_D4as_v5": 0.408000, + "Standard_D4d_v4": 0.321000, + "Standard_D4d_v5": 0.292000, + "Standard_D4ds_v4": 0.321000, + "Standard_D4ds_v5": 0.292000, + "Standard_D4lds_v5": 0.244000, + "Standard_D4ls_v5": 0.214000, + "Standard_D4s_v3": 0.258000, + "Standard_D4s_v4": 0.273000, + "Standard_D4s_v5": 0.248000, + "Standard_D5_v2": 1.459000, + "Standard_D5_v2_Promo": 1.459000, + "Standard_D64-16s_v3": 4.128000, + "Standard_D64-32s_v3": 4.128000, + "Standard_D64_v3": 4.128000, + "Standard_D64_v4": 4.365000, + "Standard_D64_v5": 3.968000, + "Standard_D64a_v4": 3.968000, + "Standard_D64ads_v5": 7.232000, + "Standard_D64as_v4": 3.968000, + "Standard_D64as_v5": 3.584000, + "Standard_D64d_v4": 5.138000, + "Standard_D64d_v5": 4.672000, + "Standard_D64ds_v4": 5.138000, + "Standard_D64ds_v5": 4.672000, + "Standard_D64lds_v5": 3.904000, + "Standard_D64ls_v5": 3.424000, + "Standard_D64s_v3": 4.128000, + "Standard_D64s_v4": 4.365000, + "Standard_D64s_v5": 3.968000, + "Standard_D8_v3": 0.516000, + "Standard_D8_v4": 0.546000, + "Standard_D8_v5": 0.496000, + "Standard_D8a_v4": 0.496000, + "Standard_D8ads_v5": 0.904000, + "Standard_D8as_v4": 0.496000, + "Standard_D8as_v5": 0.816000, + "Standard_D8d_v4": 0.642000, + "Standard_D8d_v5": 0.584000, + "Standard_D8ds_v4": 0.642000, + "Standard_D8ds_v5": 0.584000, + "Standard_D8lds_v5": 0.488000, + "Standard_D8ls_v5": 0.428000, + "Standard_D8s_v3": 0.516000, + "Standard_D8s_v4": 0.546000, + "Standard_D8s_v5": 0.496000, + "Standard_D96_v5": 5.952000, + "Standard_D96a_v4": 5.952000, + "Standard_D96ads_v5": 10.848000, + "Standard_D96as_v4": 5.952000, + "Standard_D96as_v5": 5.376000, + "Standard_D96d_v5": 7.008000, + "Standard_D96ds_v5": 7.008000, + "Standard_D96lds_v5": 5.856000, + "Standard_D96ls_v5": 5.136000, + "Standard_D96s_v5": 5.952000, + "Standard_DC16ds_v3": 2.336000, + "Standard_DC16s_v3": 1.984000, + "Standard_DC1ds_v3": 0.146000, + "Standard_DC1s_v3": 0.124000, + "Standard_DC24ds_v3": 3.504000, + "Standard_DC24s_v3": 2.976000, + "Standard_DC2ds_v3": 0.292000, + "Standard_DC2s_v3": 0.248000, + "Standard_DC32ds_v3": 4.672000, + "Standard_DC32s_v3": 3.968000, + "Standard_DC48ds_v3": 7.008000, + "Standard_DC48s_v3": 5.952000, + "Standard_DC4ds_v3": 0.584000, + "Standard_DC4s_v3": 0.496000, + "Standard_DC8ds_v3": 1.168000, + "Standard_DC8s_v3": 0.992000, + "Standard_DS1": 0.096000, + "Standard_DS11": 0.210000, + "Standard_DS11-1_v2": 0.200000, + "Standard_DS11_v2": 0.200000, + "Standard_DS11_v2_Promo": 0.200000, + "Standard_DS12": 0.420000, + "Standard_DS12-1_v2": 0.399000, + "Standard_DS12-2_v2": 0.399000, + "Standard_DS12_v2": 0.399000, + "Standard_DS12_v2_Promo": 0.399000, + "Standard_DS13": 0.840000, + "Standard_DS13-2_v2": 0.798000, + "Standard_DS13-4_v2": 0.798000, + "Standard_DS13_v2": 0.798000, + "Standard_DS13_v2_Promo": 0.798000, + "Standard_DS14": 1.680000, + "Standard_DS14-4_v2": 1.596000, + "Standard_DS14-8_v2": 1.596000, + "Standard_DS14_v2": 1.596000, + "Standard_DS14_v2_Promo": 1.596000, + "Standard_DS15_v2": 1.995000, + "Standard_DS15i_v2": 1.995000, + "Standard_DS1_v2": 0.091000, + "Standard_DS2": 0.192000, + "Standard_DS2_v2": 0.182000, + "Standard_DS2_v2_Promo": 0.182000, + "Standard_DS3": 0.384000, + "Standard_DS3_v2": 0.365000, + "Standard_DS3_v2_Promo": 0.365000, + "Standard_DS4": 0.768000, + "Standard_DS4_v2": 0.729000, + "Standard_DS4_v2_Promo": 0.729000, + "Standard_DS5_v2": 1.459000, + "Standard_DS5_v2_Promo": 1.459000, + "Standard_E104i_v5": 8.694000, + "Standard_E104id_v5": 9.953000, + "Standard_E104ids_v5": 9.953000, + "Standard_E104is_v5": 8.694000, + "Standard_E112iads_v5": 9.794000, + "Standard_E112ias_v5": 8.439000, + "Standard_E112ibds_v5": 12.320000, + "Standard_E112ibs_v5": 10.965000, + "Standard_E16-4ads_v5": 2.008000, + "Standard_E16-4as_v4": 1.216000, + "Standard_E16-4as_v5": 1.832000, + "Standard_E16-4ds_v4": 1.529000, + "Standard_E16-4ds_v5": 1.392000, + "Standard_E16-4s_v3": 1.280000, + "Standard_E16-4s_v4": 1.338000, + "Standard_E16-4s_v5": 1.216000, + "Standard_E16-8ads_v5": 2.008000, + "Standard_E16-8as_v4": 1.216000, + "Standard_E16-8as_v5": 1.832000, + "Standard_E16-8ds_v4": 1.529000, + "Standard_E16-8ds_v5": 1.392000, + "Standard_E16-8s_v3": 1.280000, + "Standard_E16-8s_v4": 1.338000, + "Standard_E16-8s_v5": 1.216000, + "Standard_E16_v3": 1.280000, + "Standard_E16_v4": 1.338000, + "Standard_E16_v5": 1.216000, + "Standard_E16a_v4": 1.216000, + "Standard_E16ads_v5": 1.272000, + "Standard_E16as_v4": 1.216000, + "Standard_E16as_v5": 1.142000, + "Standard_E16bds_v5": 1.600000, + "Standard_E16bs_v5": 1.424000, + "Standard_E16d_v4": 1.529000, + "Standard_E16d_v5": 1.392000, + "Standard_E16ds_v4": 1.529000, + "Standard_E16ds_v5": 1.392000, + "Standard_E16s_v3": 1.280000, + "Standard_E16s_v4": 1.338000, + "Standard_E16s_v5": 1.216000, + "Standard_E20_v3": 1.600000, + "Standard_E20_v4": 1.672000, + "Standard_E20_v5": 1.520000, + "Standard_E20a_v4": 1.520000, + "Standard_E20ads_v5": 1.636000, + "Standard_E20as_v4": 1.520000, + "Standard_E20as_v5": 1.370000, + "Standard_E20d_v4": 1.911000, + "Standard_E20d_v5": 1.740000, + "Standard_E20ds_v4": 1.911000, + "Standard_E20ds_v5": 1.740000, + "Standard_E20s_v3": 1.600000, + "Standard_E20s_v4": 1.672000, + "Standard_E20s_v5": 1.520000, + "Standard_E2_v3": 0.160000, + "Standard_E2_v4": 0.167000, + "Standard_E2_v5": 0.152000, + "Standard_E2a_v4": 0.152000, + "Standard_E2ads_v5": 0.205000, + "Standard_E2as_v4": 0.152000, + "Standard_E2as_v5": 0.183000, + "Standard_E2bds_v5": 0.200000, + "Standard_E2bs_v5": 0.178000, + "Standard_E2d_v4": 0.191000, + "Standard_E2d_v5": 0.174000, + "Standard_E2ds_v4": 0.191000, + "Standard_E2ds_v5": 0.174000, + "Standard_E2s_v3": 0.160000, + "Standard_E2s_v4": 0.167000, + "Standard_E2s_v5": 0.152000, + "Standard_E32-16ads_v5": 2.544000, + "Standard_E32-16as_v4": 2.432000, + "Standard_E32-16as_v5": 2.192000, + "Standard_E32-16ds_v4": 3.057000, + "Standard_E32-16ds_v5": 2.784000, + "Standard_E32-16s_v3": 2.560000, + "Standard_E32-16s_v4": 2.675000, + "Standard_E32-16s_v5": 2.432000, + "Standard_E32-8ads_v5": 4.016000, + "Standard_E32-8as_v4": 2.432000, + "Standard_E32-8as_v5": 2.192000, + "Standard_E32-8ds_v4": 3.057000, + "Standard_E32-8ds_v5": 2.784000, + "Standard_E32-8s_v3": 2.560000, + "Standard_E32-8s_v4": 2.675000, + "Standard_E32-8s_v5": 2.432000, + "Standard_E32_v3": 2.560000, + "Standard_E32_v4": 2.675000, + "Standard_E32_v5": 2.432000, + "Standard_E32a_v4": 2.432000, + "Standard_E32ads_v5": 2.544000, + "Standard_E32as_v4": 2.432000, + "Standard_E32as_v5": 2.238000, + "Standard_E32bds_v5": 3.200000, + "Standard_E32bs_v5": 2.848000, + "Standard_E32d_v4": 3.057000, + "Standard_E32d_v5": 2.784000, + "Standard_E32ds_v4": 3.057000, + "Standard_E32ds_v5": 2.784000, + "Standard_E32s_v3": 2.560000, + "Standard_E32s_v4": 2.675000, + "Standard_E32s_v5": 2.432000, + "Standard_E4-2ads_v5": 0.318000, + "Standard_E4-2as_v4": 0.304000, + "Standard_E4-2as_v5": 0.274000, + "Standard_E4-2ds_v4": 0.382000, + "Standard_E4-2ds_v5": 0.348000, + "Standard_E4-2s_v3": 0.320000, + "Standard_E4-2s_v4": 0.334000, + "Standard_E4-2s_v5": 0.304000, + "Standard_E48_v3": 3.840000, + "Standard_E48_v4": 4.013000, + "Standard_E48_v5": 3.648000, + "Standard_E48a_v4": 3.648000, + "Standard_E48ads_v5": 3.816000, + "Standard_E48as_v4": 3.648000, + "Standard_E48as_v5": 3.288000, + "Standard_E48bds_v5": 4.800000, + "Standard_E48bs_v5": 4.272000, + "Standard_E48d_v4": 4.586000, + "Standard_E48d_v5": 4.176000, + "Standard_E48ds_v4": 4.586000, + "Standard_E48ds_v5": 4.176000, + "Standard_E48s_v3": 3.840000, + "Standard_E48s_v4": 4.013000, + "Standard_E48s_v5": 3.648000, + "Standard_E4_v3": 0.320000, + "Standard_E4_v4": 0.334000, + "Standard_E4_v5": 0.304000, + "Standard_E4a_v4": 0.304000, + "Standard_E4ads_v5": 0.364000, + "Standard_E4as_v4": 0.304000, + "Standard_E4as_v5": 0.274000, + "Standard_E4bds_v5": 0.400000, + "Standard_E4bs_v5": 0.356000, + "Standard_E4d_v4": 0.382000, + "Standard_E4d_v5": 0.348000, + "Standard_E4ds_v4": 0.382000, + "Standard_E4ds_v5": 0.348000, + "Standard_E4s_v3": 0.320000, + "Standard_E4s_v4": 0.334000, + "Standard_E4s_v5": 0.304000, + "Standard_E64-16ads_v5": 8.032000, + "Standard_E64-16as_v4": 4.864000, + "Standard_E64-16as_v5": 4.384000, + "Standard_E64-16ds_v4": 6.115000, + "Standard_E64-16ds_v5": 5.568000, + "Standard_E64-16s_v3": 4.609000, + "Standard_E64-16s_v4": 5.350000, + "Standard_E64-16s_v5": 4.864000, + "Standard_E64-32ads_v5": 5.088000, + "Standard_E64-32as_v4": 4.864000, + "Standard_E64-32as_v5": 4.384000, + "Standard_E64-32ds_v4": 6.115000, + "Standard_E64-32ds_v5": 5.568000, + "Standard_E64-32s_v3": 4.609000, + "Standard_E64-32s_v4": 5.350000, + "Standard_E64-32s_v5": 4.864000, + "Standard_E64_v3": 4.609000, + "Standard_E64_v4": 5.350000, + "Standard_E64_v5": 4.864000, + "Standard_E64a_v4": 4.864000, + "Standard_E64ads_v5": 5.088000, + "Standard_E64as_v4": 4.864000, + "Standard_E64as_v5": 4.384000, + "Standard_E64bds_v5": 6.400000, + "Standard_E64bs_v5": 5.696000, + "Standard_E64d_v4": 6.115000, + "Standard_E64d_v5": 5.568000, + "Standard_E64ds_v4": 6.115000, + "Standard_E64ds_v5": 5.568000, + "Standard_E64i_v3": 4.609000, + "Standard_E64is_v3": 4.609000, + "Standard_E64s_v3": 4.609000, + "Standard_E64s_v4": 5.350000, + "Standard_E64s_v5": 4.864000, + "Standard_E8-2ads_v5": 1.004000, + "Standard_E8-2as_v4": 0.608000, + "Standard_E8-2as_v5": 0.548000, + "Standard_E8-2ds_v4": 0.764000, + "Standard_E8-2ds_v5": 0.696000, + "Standard_E8-2s_v3": 0.640000, + "Standard_E8-2s_v4": 0.669000, + "Standard_E8-2s_v5": 0.608000, + "Standard_E8-4ads_v5": 0.636000, + "Standard_E8-4as_v4": 0.608000, + "Standard_E8-4as_v5": 0.916000, + "Standard_E8-4ds_v4": 0.764000, + "Standard_E8-4ds_v5": 0.696000, + "Standard_E8-4s_v3": 0.640000, + "Standard_E8-4s_v4": 0.669000, + "Standard_E8-4s_v5": 0.608000, + "Standard_E80ids_v4": 7.644000, + "Standard_E80is_v4": 6.688000, + "Standard_E8_v3": 0.640000, + "Standard_E8_v4": 0.669000, + "Standard_E8_v5": 0.608000, + "Standard_E8a_v4": 0.608000, + "Standard_E8ads_v5": 0.636000, + "Standard_E8as_v4": 0.608000, + "Standard_E8as_v5": 0.594000, + "Standard_E8bds_v5": 0.800000, + "Standard_E8bs_v5": 0.712000, + "Standard_E8d_v4": 0.764000, + "Standard_E8d_v5": 0.696000, + "Standard_E8ds_v4": 0.764000, + "Standard_E8ds_v5": 0.696000, + "Standard_E8s_v3": 0.640000, + "Standard_E8s_v4": 0.669000, + "Standard_E8s_v5": 0.608000, + "Standard_E96-24ads_v5": 12.048000, + "Standard_E96-24as_v4": 7.296000, + "Standard_E96-24as_v5": 6.576000, + "Standard_E96-24ds_v5": 8.352000, + "Standard_E96-24s_v5": 7.296000, + "Standard_E96-48ads_v5": 12.048000, + "Standard_E96-48as_v4": 7.296000, + "Standard_E96-48as_v5": 6.576000, + "Standard_E96-48ds_v5": 8.352000, + "Standard_E96-48s_v5": 7.296000, + "Standard_E96_v5": 7.296000, + "Standard_E96a_v4": 7.296000, + "Standard_E96ads_v5": 7.678000, + "Standard_E96as_v4": 7.296000, + "Standard_E96as_v5": 6.576000, + "Standard_E96bds_v5": 9.600000, + "Standard_E96bs_v5": 8.544000, + "Standard_E96d_v5": 8.352000, + "Standard_E96ds_v5": 8.352000, + "Standard_E96iads_v5": 8.395000, + "Standard_E96ias_v4": 7.296000, + "Standard_E96ias_v5": 7.234000, + "Standard_E96s_v5": 7.296000, + "Standard_F1": 0.063000, + "Standard_F16": 1.008000, + "Standard_F16s": 1.008000, + "Standard_F16s_v2": 0.938000, + "Standard_F1s": 0.063000, + "Standard_F2": 0.126000, + "Standard_F2s": 0.126000, + "Standard_F2s_v2": 0.117000, + "Standard_F32s_v2": 1.877000, + "Standard_F4": 0.252000, + "Standard_F48s_v2": 2.815000, + "Standard_F4s": 0.252000, + "Standard_F4s_v2": 0.235000, + "Standard_F64s_v2": 3.754000, + "Standard_F72s_v2": 4.223000, + "Standard_F8": 0.504000, + "Standard_F8s": 0.504000, + "Standard_F8s_v2": 0.469000, + "Standard_L16as_v3": 1.464000, + "Standard_L16s_v3": 1.636000, + "Standard_L32as_v3": 2.928000, + "Standard_L32s_v3": 3.272000, + "Standard_L48as_v3": 4.392000, + "Standard_L48s_v3": 4.908000, + "Standard_L64as_v3": 5.856000, + "Standard_L64s_v3": 6.544000, + "Standard_L80as_v3": 7.320000, + "Standard_L80s_v3": 8.180000, + "Standard_L8as_v3": 0.732000, + "Standard_L8s_v3": 0.818000, + "Standard_M128": 19.340000, + "Standard_M128-32ms": 38.698000, + "Standard_M128-64ms": 38.698000, + "Standard_M128dms_v2": 38.701000, + "Standard_M128ds_v2": 19.343000, + "Standard_M128m": 38.698000, + "Standard_M128ms": 38.698000, + "Standard_M128ms_v2": 38.090000, + "Standard_M128s": 19.340000, + "Standard_M128s_v2": 18.733000, + "Standard_M16-4ms": 4.455900, + "Standard_M16-8ms": 4.455900, + "Standard_M16ms": 4.455900, + "Standard_M16s": 3.461000, + "Standard_M192idms_v2": 46.493000, + "Standard_M192ids_v2": 23.246000, + "Standard_M192ims_v2": 45.882000, + "Standard_M192is_v2": 22.636000, + "Standard_M208-104ms_v2": 69.161000, + "Standard_M208-104s_v2": 34.580000, + "Standard_M208-52ms_v2": 69.161000, + "Standard_M208-52s_v2": 34.580000, + "Standard_M208ms_v2": 69.161000, + "Standard_M208s_v2": 34.580000, + "Standard_M32-16ms": 8.911700, + "Standard_M32-8ms": 8.911700, + "Standard_M32dms_v2": 8.912000, + "Standard_M32ls": 3.447600, + "Standard_M32ms": 8.911700, + "Standard_M32ms_v2": 8.759000, + "Standard_M32s": 4.835800, + "Standard_M32ts": 3.248000, + "Standard_M416-104ms_v2": 153.680000, + "Standard_M416-104s_v2": 76.849000, + "Standard_M416-208ms_v2": 153.680000, + "Standard_M416-208s_v2": 76.849000, + "Standard_M416is_v2": 76.849000, + "Standard_M416ms_v2": 153.680000, + "Standard_M416s_8_v2": 102.470000, + "Standard_M416s_v2": 76.849000, + "Standard_M64": 9.670000, + "Standard_M64-16ms": 14.989000, + "Standard_M64-32ms": 14.989000, + "Standard_M64dms_v2": 14.993000, + "Standard_M64ds_v2": 9.670000, + "Standard_M64ls": 6.498000, + "Standard_M64m": 14.989000, + "Standard_M64ms": 14.989000, + "Standard_M64ms_v2": 14.688000, + "Standard_M64s": 9.670000, + "Standard_M64s_v2": 9.365000, + "Standard_M8-2ms": 2.227900, + "Standard_M8-4ms": 2.227900, + "Standard_M8ms": 2.227900, + "Standard_ND96ams_A100_v4": 50.794000, + "Standard_ND96amsr_A100_v4": 50.794000, + } + // jioindiacentral + initialOnDemandPrices["jioindiacentral"] = map[string]float64{ + "Dadsv5_Type1": 4.133000, + "Dasv4_Type1": 5.333000, + "Dasv4_Type2": 5.333000, + "Dasv5_Type1": 3.422000, + "Dsv3_Type3": 4.445000, + "Dsv5_Type1": 5.555000, + "Easv4_Type1": 6.864000, + "Easv4_Type2": 6.864000, + "Easv5_Type1": 3.775000, + "Ebdsv5-Type1": 6.195000, + "Ebsv5-Type1": 5.456000, + "Esv3_Type3": 4.506000, + "Esv3_Type4": 6.105000, + "Lasv3_Type1": 7.788000, + "Lsv3_Type1": 8.712000, + "Mdmsv2MedMem _Type1": 30.290000, + "Mdsv2MedMem_Type1": 15.139000, + "Ms_Type1": 15.134000, + "Msm_Type1": 30.294000, + "Standard_A1_v2": 0.047000, + "Standard_A2_v2": 0.098000, + "Standard_A2m_v2": 0.158000, + "Standard_A4_v2": 0.206000, + "Standard_A4m_v2": 0.333000, + "Standard_A8_v2": 0.433000, + "Standard_A8m_v2": 0.699000, + "Standard_B12ms": 0.538000, + "Standard_B16als_v2": 0.422000, + "Standard_B16as_v2": 0.467000, + "Standard_B16ls_v2": 0.635000, + "Standard_B16ms": 0.717000, + "Standard_B16s_v2": 0.790000, + "Standard_B1ls": 0.005600, + "Standard_B1ms": 0.022400, + "Standard_B1s": 0.011200, + "Standard_B20ms": 0.896000, + "Standard_B2als_v2": 0.024600, + "Standard_B2as_v2": 0.049200, + "Standard_B2ats_v2": 0.006150, + "Standard_B2ls_v2": 0.054000, + "Standard_B2ms": 0.089600, + "Standard_B2s": 0.044800, + "Standard_B2s_v2": 0.089600, + "Standard_B2ts_v2": 0.020400, + "Standard_B32als_v2": 0.844000, + "Standard_B32as_v2": 0.787000, + "Standard_B32ls_v2": 1.417000, + "Standard_B32s_v2": 1.581000, + "Standard_B4als_v2": 0.087200, + "Standard_B4as_v2": 0.098400, + "Standard_B4ls_v2": 0.177000, + "Standard_B4ms": 0.179000, + "Standard_B4s_v2": 0.179000, + "Standard_B8als_v2": 0.174000, + "Standard_B8as_v2": 0.197000, + "Standard_B8ls_v2": 0.354000, + "Standard_B8ms": 0.358000, + "Standard_B8s_v2": 0.358000, + "Standard_D11_v2": 0.189000, + "Standard_D12_v2": 0.379000, + "Standard_D13_v2": 0.758000, + "Standard_D14_v2": 1.516000, + "Standard_D15_v2": 1.895000, + "Standard_D15i_v2": 1.895000, + "Standard_D16_v3": 0.840000, + "Standard_D16a_v4": 0.493000, + "Standard_D16ads_v5": 0.537000, + "Standard_D16as_v4": 0.493000, + "Standard_D16as_v5": 0.444000, + "Standard_D16lds_v5": 0.792000, + "Standard_D16ls_v5": 0.680000, + "Standard_D16s_v3": 0.840000, + "Standard_D1_v2": 0.084000, + "Standard_D2_v2": 0.169000, + "Standard_D2_v3": 0.105000, + "Standard_D2a_v4": 0.062200, + "Standard_D2ads_v5": 0.067100, + "Standard_D2as_v4": 0.062200, + "Standard_D2as_v5": 0.055600, + "Standard_D2lds_v5": 0.099000, + "Standard_D2ls_v5": 0.085000, + "Standard_D2s_v3": 0.105000, + "Standard_D32-16s_v3": 1.680000, + "Standard_D32-8s_v3": 1.680000, + "Standard_D32_v3": 1.680000, + "Standard_D32a_v4": 0.987000, + "Standard_D32ads_v5": 2.546000, + "Standard_D32as_v4": 0.987000, + "Standard_D32as_v5": 2.361000, + "Standard_D32lds_v5": 1.584000, + "Standard_D32ls_v5": 1.360000, + "Standard_D32s_v3": 1.680000, + "Standard_D3_v2": 0.337000, + "Standard_D48_v3": 2.520000, + "Standard_D48a_v4": 1.481000, + "Standard_D48ads_v5": 1.610000, + "Standard_D48as_v4": 1.481000, + "Standard_D48as_v5": 3.541000, + "Standard_D48lds_v5": 2.376000, + "Standard_D48ls_v5": 2.040000, + "Standard_D48s_v3": 2.520000, + "Standard_D4_v2": 0.675000, + "Standard_D4_v3": 0.210000, + "Standard_D4a_v4": 0.123000, + "Standard_D4ads_v5": 0.318000, + "Standard_D4as_v4": 0.123000, + "Standard_D4as_v5": 0.111000, + "Standard_D4lds_v5": 0.198000, + "Standard_D4ls_v5": 0.170000, + "Standard_D4s_v3": 0.210000, + "Standard_D5_v2": 1.350000, + "Standard_D64-16s_v3": 3.360000, + "Standard_D64-32s_v3": 3.360000, + "Standard_D64_v3": 3.360000, + "Standard_D64a_v4": 1.975000, + "Standard_D64ads_v5": 5.091000, + "Standard_D64as_v4": 1.975000, + "Standard_D64as_v5": 4.722000, + "Standard_D64lds_v5": 3.168000, + "Standard_D64ls_v5": 2.720000, + "Standard_D64s_v3": 3.360000, + "Standard_D8_v3": 0.420000, + "Standard_D8a_v4": 0.246000, + "Standard_D8ads_v5": 0.636000, + "Standard_D8as_v4": 0.246000, + "Standard_D8as_v5": 0.222000, + "Standard_D8lds_v5": 0.396000, + "Standard_D8ls_v5": 0.340000, + "Standard_D8s_v3": 0.420000, + "Standard_D96a_v4": 2.962000, + "Standard_D96ads_v5": 3.221000, + "Standard_D96as_v4": 2.962000, + "Standard_D96as_v5": 2.666000, + "Standard_D96lds_v5": 4.752000, + "Standard_D96ls_v5": 4.080000, + "Standard_DS11-1_v2": 0.189000, + "Standard_DS11_v2": 0.189000, + "Standard_DS12-1_v2": 0.379000, + "Standard_DS12-2_v2": 0.379000, + "Standard_DS12_v2": 0.379000, + "Standard_DS13-2_v2": 0.758000, + "Standard_DS13-4_v2": 0.758000, + "Standard_DS13_v2": 0.758000, + "Standard_DS14-4_v2": 1.516000, + "Standard_DS14-8_v2": 1.516000, + "Standard_DS14_v2": 1.516000, + "Standard_DS15_v2": 1.895000, + "Standard_DS15i_v2": 1.895000, + "Standard_DS1_v2": 0.084000, + "Standard_DS2_v2": 0.169000, + "Standard_DS3_v2": 0.337000, + "Standard_DS4_v2": 0.675000, + "Standard_DS5_v2": 1.350000, + "Standard_E112iads_v5": 5.116000, + "Standard_E112ias_v5": 4.404000, + "Standard_E112ibds_v5": 10.842000, + "Standard_E112ibs_v5": 9.548000, + "Standard_E16-4ads_v5": 1.400000, + "Standard_E16-4as_v4": 1.040000, + "Standard_E16-4as_v5": 0.572000, + "Standard_E16-4s_v3": 1.096000, + "Standard_E16-8ads_v5": 1.400000, + "Standard_E16-8as_v4": 1.040000, + "Standard_E16-8as_v5": 1.308000, + "Standard_E16-8s_v3": 1.096000, + "Standard_E16_v3": 1.096000, + "Standard_E16a_v4": 0.635000, + "Standard_E16ads_v5": 0.664000, + "Standard_E16as_v4": 0.635000, + "Standard_E16as_v5": 0.572000, + "Standard_E16bds_v5": 1.408000, + "Standard_E16bs_v5": 1.240000, + "Standard_E16s_v3": 1.096000, + "Standard_E20_v3": 1.370000, + "Standard_E20a_v4": 0.794000, + "Standard_E20ads_v5": 0.830000, + "Standard_E20as_v4": 0.794000, + "Standard_E20as_v5": 0.761000, + "Standard_E20s_v3": 1.370000, + "Standard_E2_v3": 0.137000, + "Standard_E2a_v4": 0.080000, + "Standard_E2ads_v5": 0.083000, + "Standard_E2as_v4": 0.080000, + "Standard_E2as_v5": 0.071500, + "Standard_E2bds_v5": 0.176000, + "Standard_E2bs_v5": 0.155000, + "Standard_E2s_v3": 0.137000, + "Standard_E32-16ads_v5": 2.801000, + "Standard_E32-16as_v4": 2.080000, + "Standard_E32-16as_v5": 1.144000, + "Standard_E32-16s_v3": 2.192000, + "Standard_E32-8ads_v5": 2.801000, + "Standard_E32-8as_v4": 2.080000, + "Standard_E32-8as_v5": 2.616000, + "Standard_E32-8s_v3": 2.192000, + "Standard_E32_v3": 2.192000, + "Standard_E32a_v4": 1.271000, + "Standard_E32ads_v5": 1.329000, + "Standard_E32as_v4": 1.271000, + "Standard_E32as_v5": 1.144000, + "Standard_E32bds_v5": 2.816000, + "Standard_E32bs_v5": 2.480000, + "Standard_E32s_v3": 2.192000, + "Standard_E4-2ads_v5": 0.350000, + "Standard_E4-2as_v4": 0.260000, + "Standard_E4-2as_v5": 0.143000, + "Standard_E4-2s_v3": 0.274000, + "Standard_E48_v3": 3.281000, + "Standard_E48a_v4": 1.906000, + "Standard_E48ads_v5": 2.039000, + "Standard_E48as_v4": 1.906000, + "Standard_E48as_v5": 1.716000, + "Standard_E48bds_v5": 4.224000, + "Standard_E48bs_v5": 3.720000, + "Standard_E48s_v3": 3.281000, + "Standard_E4_v3": 0.274000, + "Standard_E4a_v4": 0.158000, + "Standard_E4ads_v5": 0.212000, + "Standard_E4as_v4": 0.158000, + "Standard_E4as_v5": 0.143000, + "Standard_E4bds_v5": 0.352000, + "Standard_E4bs_v5": 0.310000, + "Standard_E4s_v3": 0.274000, + "Standard_E64-16ads_v5": 2.658000, + "Standard_E64-16as_v4": 4.160000, + "Standard_E64-16as_v5": 5.232000, + "Standard_E64-16s_v3": 3.937000, + "Standard_E64-32ads_v5": 2.658000, + "Standard_E64-32as_v4": 4.160000, + "Standard_E64-32as_v5": 5.232000, + "Standard_E64-32s_v3": 3.937000, + "Standard_E64_v3": 3.937000, + "Standard_E64a_v4": 2.542000, + "Standard_E64ads_v5": 2.704000, + "Standard_E64as_v4": 2.542000, + "Standard_E64as_v5": 2.334000, + "Standard_E64bds_v5": 5.632000, + "Standard_E64bs_v5": 4.960000, + "Standard_E64s_v3": 3.937000, + "Standard_E8-2ads_v5": 0.700000, + "Standard_E8-2as_v4": 0.520000, + "Standard_E8-2as_v5": 0.286000, + "Standard_E8-2s_v3": 0.548000, + "Standard_E8-4ads_v5": 0.700000, + "Standard_E8-4as_v4": 0.520000, + "Standard_E8-4as_v5": 0.654000, + "Standard_E8-4s_v3": 0.548000, + "Standard_E8_v3": 0.548000, + "Standard_E8a_v4": 0.317000, + "Standard_E8ads_v5": 0.378000, + "Standard_E8as_v4": 0.317000, + "Standard_E8as_v5": 0.332000, + "Standard_E8bds_v5": 0.704000, + "Standard_E8bs_v5": 0.620000, + "Standard_E8s_v3": 0.548000, + "Standard_E96-24ads_v5": 3.986000, + "Standard_E96-24as_v4": 6.240000, + "Standard_E96-24as_v5": 7.848000, + "Standard_E96-48ads_v5": 3.986000, + "Standard_E96-48as_v4": 6.240000, + "Standard_E96-48as_v5": 3.432000, + "Standard_E96a_v4": 3.813000, + "Standard_E96ads_v5": 3.986000, + "Standard_E96as_v4": 3.813000, + "Standard_E96as_v5": 3.478000, + "Standard_E96bds_v5": 8.448000, + "Standard_E96bs_v5": 7.440000, + "Standard_E96iads_v5": 4.385000, + "Standard_E96ias_v4": 6.240000, + "Standard_E96ias_v5": 3.775000, + "Standard_F1": 0.049000, + "Standard_F16": 0.790000, + "Standard_F16s": 0.790000, + "Standard_F1s": 0.049000, + "Standard_F2": 0.098800, + "Standard_F2s": 0.098800, + "Standard_F4": 0.198000, + "Standard_F4s": 0.198000, + "Standard_F8": 0.395000, + "Standard_F8s": 0.395000, + "Standard_L16as_v3": 1.416000, + "Standard_L16s_v3": 1.584000, + "Standard_L32as_v3": 2.832000, + "Standard_L32s_v3": 3.168000, + "Standard_L48as_v3": 4.248000, + "Standard_L48s_v3": 4.752000, + "Standard_L64as_v3": 5.664000, + "Standard_L64s_v3": 6.336000, + "Standard_L80as_v3": 7.080000, + "Standard_L80s_v3": 7.920000, + "Standard_L8as_v3": 0.708000, + "Standard_L8s_v3": 0.792000, + "Standard_M128": 13.762000, + "Standard_M128-32ms": 27.524000, + "Standard_M128-64ms": 27.524000, + "Standard_M128dms_v2": 27.536000, + "Standard_M128ds_v2": 13.763000, + "Standard_M128m": 27.536000, + "Standard_M128ms": 27.524000, + "Standard_M128s": 13.762000, + "Standard_M16-4ms": 3.170000, + "Standard_M16-8ms": 3.170000, + "Standard_M16ms": 3.170000, + "Standard_M16s": 2.462000, + "Standard_M192idms_v2": 33.080000, + "Standard_M192ids_v2": 16.540000, + "Standard_M32-16ms": 6.341000, + "Standard_M32-8ms": 6.341000, + "Standard_M32dms_v2": 6.341000, + "Standard_M32ls": 2.964074, + "Standard_M32ms": 6.341000, + "Standard_M32s": 3.440000, + "Standard_M32ts": 2.792812, + "Standard_M64": 6.881000, + "Standard_M64-16ms": 10.666000, + "Standard_M64-32ms": 10.666000, + "Standard_M64dms_v2": 10.668000, + "Standard_M64ds_v2": 6.880000, + "Standard_M64ls": 5.586656, + "Standard_M64m": 10.666000, + "Standard_M64ms": 10.666000, + "Standard_M64s": 6.881000, + "Standard_M8-2ms": 1.585000, + "Standard_M8-4ms": 1.585000, + "Standard_M8ms": 1.585000, + } + // jioindiawest + initialOnDemandPrices["jioindiawest"] = map[string]float64{ + "Dadsv5_Type1": 4.133000, + "Dasv4_Type1": 5.333000, + "Dasv4_Type2": 5.333000, + "Dasv5_Type1": 3.422000, + "Dsv3_Type3": 4.445000, + "Dsv5_Type1": 5.555000, + "Eadsv5_Type1": 4.385000, + "Easv4_Type1": 6.864000, + "Easv4_Type2": 6.864000, + "Easv5_Type1": 3.775000, + "Ebdsv5-Type1": 6.195000, + "Ebsv5-Type1": 5.456000, + "Esv3_Type3": 4.506000, + "Esv3_Type4": 6.105000, + "Fsv2 Type3": 3.927000, + "Fsv2_Type2": 3.366000, + "Fsv2_Type4": 5.142000, + "Lasv3_Type1": 7.788000, + "Lsv3_Type1": 8.712000, + "Mdmsv2MedMem _Type1": 30.290000, + "Mdsv2MedMem_Type1": 15.139000, + "Ms_Type1": 15.134000, + "Msm_Type1": 30.294000, + "Standard_A1_v2": 0.047000, + "Standard_A2_v2": 0.098000, + "Standard_A2m_v2": 0.158000, + "Standard_A4_v2": 0.206000, + "Standard_A4m_v2": 0.333000, + "Standard_A8_v2": 0.433000, + "Standard_A8m_v2": 0.699000, + "Standard_B12ms": 0.538000, + "Standard_B16als_v2": 0.349000, + "Standard_B16as_v2": 0.394000, + "Standard_B16ls_v2": 0.708000, + "Standard_B16ms": 0.717000, + "Standard_B16s_v2": 0.790000, + "Standard_B1ls": 0.005600, + "Standard_B1ms": 0.022400, + "Standard_B1s": 0.011200, + "Standard_B20ms": 0.896000, + "Standard_B2als_v2": 0.024600, + "Standard_B2as_v2": 0.058400, + "Standard_B2ats_v2": 0.015400, + "Standard_B2ls_v2": 0.054000, + "Standard_B2ms": 0.089600, + "Standard_B2s": 0.044800, + "Standard_B2s_v2": 0.098800, + "Standard_B2ts_v2": 0.020400, + "Standard_B32als_v2": 0.844000, + "Standard_B32as_v2": 0.787000, + "Standard_B32ls_v2": 1.269000, + "Standard_B32s_v2": 1.434000, + "Standard_B4als_v2": 0.087200, + "Standard_B4as_v2": 0.098400, + "Standard_B4ls_v2": 0.159000, + "Standard_B4ms": 0.179000, + "Standard_B4s_v2": 0.198000, + "Standard_B8als_v2": 0.174000, + "Standard_B8as_v2": 0.197000, + "Standard_B8ls_v2": 0.354000, + "Standard_B8ms": 0.358000, + "Standard_B8s_v2": 0.358000, + "Standard_D11_v2": 0.189000, + "Standard_D12_v2": 0.379000, + "Standard_D13_v2": 0.758000, + "Standard_D14_v2": 1.516000, + "Standard_D14_v2_Promo": 1.516000, + "Standard_D15_v2": 1.895000, + "Standard_D15i_v2": 1.895000, + "Standard_D16_v3": 0.840000, + "Standard_D16a_v4": 0.493000, + "Standard_D16ads_v5": 1.273000, + "Standard_D16as_v4": 0.493000, + "Standard_D16as_v5": 1.180000, + "Standard_D16lds_v5": 0.792000, + "Standard_D16ls_v5": 0.680000, + "Standard_D16s_v3": 0.840000, + "Standard_D1_v2": 0.084000, + "Standard_D2_v2": 0.169000, + "Standard_D2_v3": 0.105000, + "Standard_D2a_v4": 0.062200, + "Standard_D2ads_v5": 0.067100, + "Standard_D2as_v4": 0.062200, + "Standard_D2as_v5": 0.148000, + "Standard_D2lds_v5": 0.099000, + "Standard_D2ls_v5": 0.085000, + "Standard_D2s_v3": 0.105000, + "Standard_D32-16s_v3": 1.680000, + "Standard_D32-8s_v3": 1.680000, + "Standard_D32_v3": 1.680000, + "Standard_D32a_v4": 0.987000, + "Standard_D32ads_v5": 1.074000, + "Standard_D32as_v4": 0.987000, + "Standard_D32as_v5": 2.361000, + "Standard_D32lds_v5": 1.584000, + "Standard_D32ls_v5": 1.360000, + "Standard_D32s_v3": 1.680000, + "Standard_D3_v2": 0.337000, + "Standard_D48_v3": 2.520000, + "Standard_D48a_v4": 1.481000, + "Standard_D48ads_v5": 1.610000, + "Standard_D48as_v4": 1.481000, + "Standard_D48as_v5": 1.333000, + "Standard_D48lds_v5": 2.376000, + "Standard_D48ls_v5": 2.040000, + "Standard_D48s_v3": 2.520000, + "Standard_D4_v2": 0.675000, + "Standard_D4_v3": 0.210000, + "Standard_D4a_v4": 0.123000, + "Standard_D4ads_v5": 0.134000, + "Standard_D4as_v4": 0.123000, + "Standard_D4as_v5": 0.295000, + "Standard_D4lds_v5": 0.198000, + "Standard_D4ls_v5": 0.170000, + "Standard_D4s_v3": 0.210000, + "Standard_D5_v2": 1.350000, + "Standard_D5_v2_Promo": 1.350000, + "Standard_D64-16s_v3": 3.360000, + "Standard_D64-32s_v3": 3.360000, + "Standard_D64_v3": 3.360000, + "Standard_D64a_v4": 1.975000, + "Standard_D64ads_v5": 5.091000, + "Standard_D64as_v4": 1.975000, + "Standard_D64as_v5": 4.722000, + "Standard_D64lds_v5": 3.168000, + "Standard_D64ls_v5": 2.720000, + "Standard_D64s_v3": 3.360000, + "Standard_D8_v3": 0.420000, + "Standard_D8a_v4": 0.246000, + "Standard_D8ads_v5": 0.636000, + "Standard_D8as_v4": 0.246000, + "Standard_D8as_v5": 0.222000, + "Standard_D8lds_v5": 0.396000, + "Standard_D8ls_v5": 0.340000, + "Standard_D8s_v3": 0.420000, + "Standard_D96a_v4": 2.962000, + "Standard_D96ads_v5": 7.637000, + "Standard_D96as_v4": 2.962000, + "Standard_D96as_v5": 7.082000, + "Standard_D96lds_v5": 4.752000, + "Standard_D96ls_v5": 4.080000, + "Standard_DS11-1_v2": 0.189000, + "Standard_DS11_v2": 0.189000, + "Standard_DS12-1_v2": 0.379000, + "Standard_DS12-2_v2": 0.379000, + "Standard_DS12_v2": 0.379000, + "Standard_DS13-2_v2": 0.758000, + "Standard_DS13-4_v2": 0.758000, + "Standard_DS13_v2": 0.758000, + "Standard_DS14-4_v2": 1.516000, + "Standard_DS14-8_v2": 1.516000, + "Standard_DS14_v2": 1.516000, + "Standard_DS14_v2_Promo": 1.516000, + "Standard_DS15_v2": 1.895000, + "Standard_DS15i_v2": 1.895000, + "Standard_DS1_v2": 0.084000, + "Standard_DS2_v2": 0.169000, + "Standard_DS3_v2": 0.337000, + "Standard_DS4_v2": 0.675000, + "Standard_DS5_v2": 1.350000, + "Standard_DS5_v2_Promo": 1.350000, + "Standard_E112iads_v5": 5.116000, + "Standard_E112ias_v5": 4.404000, + "Standard_E112ibds_v5": 10.842000, + "Standard_E112ibs_v5": 9.548000, + "Standard_E16-4ads_v5": 1.400000, + "Standard_E16-4as_v4": 1.040000, + "Standard_E16-4as_v5": 0.572000, + "Standard_E16-4s_v3": 1.096000, + "Standard_E16-8ads_v5": 0.664000, + "Standard_E16-8as_v4": 1.040000, + "Standard_E16-8as_v5": 1.308000, + "Standard_E16-8s_v3": 1.096000, + "Standard_E16_v3": 1.096000, + "Standard_E16a_v4": 0.635000, + "Standard_E16ads_v5": 0.710000, + "Standard_E16as_v4": 0.635000, + "Standard_E16as_v5": 0.618000, + "Standard_E16bds_v5": 1.408000, + "Standard_E16bs_v5": 1.240000, + "Standard_E16ds_v5": 0.604000, + "Standard_E16s_v3": 1.096000, + "Standard_E16s_v5": 0.520000, + "Standard_E20_v3": 1.370000, + "Standard_E20a_v4": 0.794000, + "Standard_E20ads_v5": 0.830000, + "Standard_E20as_v4": 0.794000, + "Standard_E20as_v5": 0.761000, + "Standard_E20ds_v5": 0.755000, + "Standard_E20s_v3": 1.370000, + "Standard_E20s_v5": 0.650000, + "Standard_E2_v3": 0.137000, + "Standard_E2a_v4": 0.080000, + "Standard_E2ads_v5": 0.083000, + "Standard_E2as_v4": 0.080000, + "Standard_E2as_v5": 0.071500, + "Standard_E2bds_v5": 0.176000, + "Standard_E2bs_v5": 0.155000, + "Standard_E2ds_v5": 0.075500, + "Standard_E2s_v3": 0.137000, + "Standard_E2s_v5": 0.065000, + "Standard_E32-16ads_v5": 2.801000, + "Standard_E32-16as_v4": 2.080000, + "Standard_E32-16as_v5": 2.616000, + "Standard_E32-16s_v3": 2.192000, + "Standard_E32-8ads_v5": 2.801000, + "Standard_E32-8as_v4": 2.080000, + "Standard_E32-8as_v5": 1.144000, + "Standard_E32-8s_v3": 2.192000, + "Standard_E32_v3": 2.192000, + "Standard_E32a_v4": 1.271000, + "Standard_E32ads_v5": 1.375000, + "Standard_E32as_v4": 1.271000, + "Standard_E32as_v5": 1.144000, + "Standard_E32bds_v5": 2.816000, + "Standard_E32bs_v5": 2.480000, + "Standard_E32ds_v5": 1.208000, + "Standard_E32s_v3": 2.192000, + "Standard_E32s_v5": 1.040000, + "Standard_E4-2ads_v5": 0.350000, + "Standard_E4-2as_v4": 0.260000, + "Standard_E4-2as_v5": 0.143000, + "Standard_E4-2s_v3": 0.274000, + "Standard_E48_v3": 3.281000, + "Standard_E48a_v4": 1.906000, + "Standard_E48ads_v5": 1.993000, + "Standard_E48as_v4": 1.906000, + "Standard_E48as_v5": 1.762000, + "Standard_E48bds_v5": 4.224000, + "Standard_E48bs_v5": 3.720000, + "Standard_E48ds_v5": 1.812000, + "Standard_E48s_v3": 3.281000, + "Standard_E48s_v5": 1.560000, + "Standard_E4_v3": 0.274000, + "Standard_E4a_v4": 0.158000, + "Standard_E4ads_v5": 0.166000, + "Standard_E4as_v4": 0.158000, + "Standard_E4as_v5": 0.143000, + "Standard_E4bds_v5": 0.352000, + "Standard_E4bs_v5": 0.310000, + "Standard_E4ds_v5": 0.151000, + "Standard_E4s_v3": 0.274000, + "Standard_E4s_v5": 0.130000, + "Standard_E64-16ads_v5": 2.658000, + "Standard_E64-16as_v4": 4.160000, + "Standard_E64-16as_v5": 5.232000, + "Standard_E64-16s_v3": 3.937000, + "Standard_E64-32ads_v5": 5.602000, + "Standard_E64-32as_v4": 4.160000, + "Standard_E64-32as_v5": 5.232000, + "Standard_E64-32s_v3": 3.937000, + "Standard_E64_v3": 3.937000, + "Standard_E64a_v4": 2.542000, + "Standard_E64ads_v5": 2.704000, + "Standard_E64as_v4": 2.542000, + "Standard_E64as_v5": 2.334000, + "Standard_E64bds_v5": 5.632000, + "Standard_E64bs_v5": 4.960000, + "Standard_E64ds_v5": 2.416000, + "Standard_E64s_v3": 3.937000, + "Standard_E64s_v5": 2.080000, + "Standard_E8-2ads_v5": 0.332000, + "Standard_E8-2as_v4": 0.520000, + "Standard_E8-2as_v5": 0.286000, + "Standard_E8-2s_v3": 0.548000, + "Standard_E8-4ads_v5": 0.332000, + "Standard_E8-4as_v4": 0.520000, + "Standard_E8-4as_v5": 0.286000, + "Standard_E8-4s_v3": 0.548000, + "Standard_E8_v3": 0.548000, + "Standard_E8a_v4": 0.317000, + "Standard_E8ads_v5": 0.378000, + "Standard_E8as_v4": 0.317000, + "Standard_E8as_v5": 0.332000, + "Standard_E8bds_v5": 0.704000, + "Standard_E8bs_v5": 0.620000, + "Standard_E8ds_v5": 0.302000, + "Standard_E8s_v3": 0.548000, + "Standard_E8s_v5": 0.260000, + "Standard_E96-24ads_v5": 3.986000, + "Standard_E96-24as_v4": 6.240000, + "Standard_E96-24as_v5": 7.848000, + "Standard_E96-48ads_v5": 8.402000, + "Standard_E96-48as_v4": 6.240000, + "Standard_E96-48as_v5": 3.432000, + "Standard_E96a_v4": 3.813000, + "Standard_E96ads_v5": 3.986000, + "Standard_E96as_v4": 3.813000, + "Standard_E96as_v5": 3.432000, + "Standard_E96bds_v5": 8.448000, + "Standard_E96bs_v5": 7.440000, + "Standard_E96ds_v5": 3.624000, + "Standard_E96iads_v5": 4.385000, + "Standard_E96ias_v4": 6.240000, + "Standard_E96ias_v5": 3.775000, + "Standard_E96s_v5": 3.120000, + "Standard_F1": 0.049000, + "Standard_F16": 0.790000, + "Standard_F16s": 0.790000, + "Standard_F16s_v2": 0.680000, + "Standard_F1s": 0.049000, + "Standard_F2": 0.098800, + "Standard_F2s": 0.098800, + "Standard_F2s_v2": 0.085000, + "Standard_F32s_v2": 1.360000, + "Standard_F4": 0.198000, + "Standard_F48s_v2": 2.040000, + "Standard_F4s": 0.198000, + "Standard_F4s_v2": 0.170000, + "Standard_F64s_v2": 2.720000, + "Standard_F72s_v2": 3.060000, + "Standard_F8": 0.395000, + "Standard_F8s": 0.395000, + "Standard_F8s_v2": 0.340000, + "Standard_L16as_v3": 1.416000, + "Standard_L16s_v3": 1.584000, + "Standard_L32as_v3": 2.832000, + "Standard_L32s_v3": 3.168000, + "Standard_L48as_v3": 4.248000, + "Standard_L48s_v3": 4.752000, + "Standard_L64as_v3": 5.664000, + "Standard_L64s_v3": 6.336000, + "Standard_L80as_v3": 7.080000, + "Standard_L80s_v3": 7.920000, + "Standard_L8as_v3": 0.708000, + "Standard_L8s_v3": 0.792000, + "Standard_M128": 13.762000, + "Standard_M128-32ms": 27.524000, + "Standard_M128-64ms": 27.524000, + "Standard_M128dms_v2": 27.536000, + "Standard_M128ds_v2": 13.763000, + "Standard_M128m": 27.536000, + "Standard_M128ms": 27.524000, + "Standard_M128s": 13.762000, + "Standard_M16-4ms": 3.170000, + "Standard_M16-8ms": 3.170000, + "Standard_M16ms": 3.170000, + "Standard_M16s": 2.462000, + "Standard_M192idms_v2": 33.080000, + "Standard_M192ids_v2": 16.540000, + "Standard_M32-16ms": 6.341000, + "Standard_M32-8ms": 6.341000, + "Standard_M32dms_v2": 6.341000, + "Standard_M32ls": 2.964074, + "Standard_M32ms": 6.341000, + "Standard_M32s": 3.440000, + "Standard_M32ts": 2.792812, + "Standard_M64": 6.881000, + "Standard_M64-16ms": 10.666000, + "Standard_M64-32ms": 10.666000, + "Standard_M64dms_v2": 10.668000, + "Standard_M64ds_v2": 6.880000, + "Standard_M64ls": 5.586656, + "Standard_M64m": 10.666000, + "Standard_M64ms": 10.666000, + "Standard_M64s": 6.881000, + "Standard_M8-2ms": 1.585000, + "Standard_M8-4ms": 1.585000, + "Standard_M8ms": 1.585000, + "Standard_NC12s_v3": 8.468000, + "Standard_NC24ads_A100_v4": 5.142000, + "Standard_NC24rs_v3": 18.630000, + "Standard_NC24s_v3": 16.936000, + "Standard_NC48ads_A100_v4": 10.284000, + "Standard_NC6s_v3": 4.234000, + "Standard_NC96ads_A100_v4": 20.569000, + } + // koreacentral + initialOnDemandPrices["koreacentral"] = map[string]float64{ + "Basic_A0": 0.018000, + "Basic_A1": 0.032000, + "Basic_A2": 0.108000, + "Basic_A3": 0.224000, + "Basic_A4": 0.449000, + "Dadsv5_Type1": 7.823000, + "Dasv4_Type1": 6.230000, + "Dasv4_Type2": 6.490000, + "Dasv5_Type1": 6.530000, + "Ddsv4_Type 1": 4.893000, + "Ddsv4_Type2": 5.810000, + "Ddsv5_Type1": 7.339000, + "Dsv3_Type3": 5.193000, + "Dsv3_Type4": 6.491000, + "Dsv4_Type1": 5.191000, + "Dsv4_Type2": 6.230000, + "Dsv5_Type1": 6.490000, + "Eadsv5_Type1": 8.342000, + "Easv4_Type1": 8.025000, + "Easv4_Type2": 8.026000, + "Easv5_Type1": 7.181000, + "Ebdsv5-Type1": 7.005000, + "Ebsv5-Type1": 6.266000, + "Edsv4_Type 1": 6.090000, + "Edsv4_Type2": 7.231000, + "Edsv5_Type1": 9.134000, + "Esv3_Type3": 5.162000, + "Esv3_Type4": 6.994000, + "Esv4_Type1": 5.350000, + "Esv4_Type2": 7.022000, + "Esv5_Type1": 8.026000, + "Fsv2 Type3": 4.435000, + "Fsv2_Type2": 3.802000, + "Fsv2_Type4": 5.069000, + "Lasv3_Type1": 8.052000, + "Lsv3_Type1": 8.998000, + "Mdmsv2MedMem _Type1": 42.571000, + "Mdsv2MedMem_Type1": 21.277000, + "Msmv2MedMem Type1": 41.899000, + "Msmv2_Type1": 130.880000, + "Msv2MedMem Type1": 20.606000, + "Msv2_Type1": 65.446000, + "NVasv4_Type1": 9.688000, + "Standard_A0": 0.020000, + "Standard_A1": 0.066000, + "Standard_A1_v2": 0.049000, + "Standard_A2": 0.133000, + "Standard_A2_v2": 0.102000, + "Standard_A2m_v2": 0.133000, + "Standard_A3": 0.264000, + "Standard_A4": 0.528000, + "Standard_A4_v2": 0.214000, + "Standard_A4m_v2": 0.278000, + "Standard_A5": 0.172000, + "Standard_A6": 0.343000, + "Standard_A7": 0.687000, + "Standard_A8_v2": 0.450000, + "Standard_A8m_v2": 0.585000, + "Standard_B12ms": 0.624000, + "Standard_B16als_v2": 0.663000, + "Standard_B16as_v2": 0.822000, + "Standard_B16ls_v2": 0.810000, + "Standard_B16ms": 0.832000, + "Standard_B16s_v2": 0.832000, + "Standard_B1ls": 0.006500, + "Standard_B1ms": 0.026000, + "Standard_B1s": 0.013000, + "Standard_B20ms": 1.040000, + "Standard_B2als_v2": 0.046800, + "Standard_B2as_v2": 0.093600, + "Standard_B2ats_v2": 0.020900, + "Standard_B2ls_v2": 0.061200, + "Standard_B2ms": 0.104000, + "Standard_B2s": 0.052000, + "Standard_B2s_v2": 0.104000, + "Standard_B2ts_v2": 0.022200, + "Standard_B32als_v2": 1.326000, + "Standard_B32as_v2": 1.645000, + "Standard_B32ls_v2": 1.621000, + "Standard_B32s_v2": 1.811000, + "Standard_B4als_v2": 0.184000, + "Standard_B4as_v2": 0.187000, + "Standard_B4ls_v2": 0.184000, + "Standard_B4ms": 0.208000, + "Standard_B4s_v2": 0.208000, + "Standard_B8als_v2": 0.368000, + "Standard_B8as_v2": 0.374000, + "Standard_B8ls_v2": 0.368000, + "Standard_B8ms": 0.416000, + "Standard_B8s_v2": 0.416000, + "Standard_D11_v2": 0.200000, + "Standard_D11_v2_Promo": 0.200000, + "Standard_D12_v2": 0.399000, + "Standard_D12_v2_Promo": 0.399000, + "Standard_D13_v2": 0.798000, + "Standard_D13_v2_Promo": 0.798000, + "Standard_D14_v2": 1.596000, + "Standard_D14_v2_Promo": 1.596000, + "Standard_D15_v2": 1.995000, + "Standard_D15i_v2": 1.995000, + "Standard_D16_v3": 0.984000, + "Standard_D16_v4": 0.944000, + "Standard_D16_v5": 0.944000, + "Standard_D16a_v4": 0.944000, + "Standard_D16ads_v5": 1.016000, + "Standard_D16as_v4": 0.944000, + "Standard_D16as_v5": 1.584000, + "Standard_D16as_v5_Promo": 0.752000, + "Standard_D16d_v4": 1.112000, + "Standard_D16d_v5": 1.112000, + "Standard_D16ds_v4": 1.112000, + "Standard_D16ds_v5": 1.112000, + "Standard_D16lds_v5": 0.880000, + "Standard_D16ls_v5": 0.768000, + "Standard_D16s_v3": 0.984000, + "Standard_D16s_v4": 0.944000, + "Standard_D16s_v5": 0.944000, + "Standard_D1_v2": 0.082500, + "Standard_D2_v2": 0.165000, + "Standard_D2_v2_Promo": 0.165000, + "Standard_D2_v3": 0.123000, + "Standard_D2_v4": 0.118000, + "Standard_D2_v5": 0.118000, + "Standard_D2a_v4": 0.118000, + "Standard_D2ads_v5": 0.219000, + "Standard_D2as_v4": 0.118000, + "Standard_D2as_v5": 0.106000, + "Standard_D2as_v5_Promo": 0.094000, + "Standard_D2d_v4": 0.139000, + "Standard_D2d_v5": 0.139000, + "Standard_D2ds_v4": 0.139000, + "Standard_D2ds_v5": 0.139000, + "Standard_D2lds_v5": 0.110000, + "Standard_D2ls_v5": 0.096000, + "Standard_D2s_v3": 0.123000, + "Standard_D2s_v4": 0.118000, + "Standard_D2s_v5": 0.118000, + "Standard_D32-16s_v3": 1.968000, + "Standard_D32-8s_v3": 1.968000, + "Standard_D32_v3": 1.968000, + "Standard_D32_v4": 1.888000, + "Standard_D32_v5": 1.888000, + "Standard_D32a_v4": 1.888000, + "Standard_D32ads_v5": 3.504000, + "Standard_D32as_v4": 1.888000, + "Standard_D32as_v5": 1.696000, + "Standard_D32as_v5_Promo": 1.504000, + "Standard_D32d_v4": 2.224000, + "Standard_D32d_v5": 2.224000, + "Standard_D32ds_v4": 2.224000, + "Standard_D32ds_v5": 2.224000, + "Standard_D32lds_v5": 1.760000, + "Standard_D32ls_v5": 1.536000, + "Standard_D32s_v3": 1.968000, + "Standard_D32s_v4": 1.888000, + "Standard_D32s_v5": 1.888000, + "Standard_D3_v2": 0.330000, + "Standard_D3_v2_Promo": 0.330000, + "Standard_D48_v3": 2.952000, + "Standard_D48_v4": 2.832000, + "Standard_D48_v5": 2.832000, + "Standard_D48a_v4": 2.832000, + "Standard_D48ads_v5": 5.256000, + "Standard_D48as_v4": 2.832000, + "Standard_D48as_v5": 2.544000, + "Standard_D48as_v5_Promo": 2.256000, + "Standard_D48d_v4": 3.336000, + "Standard_D48d_v5": 3.336000, + "Standard_D48ds_v4": 3.336000, + "Standard_D48ds_v5": 3.336000, + "Standard_D48lds_v5": 2.640000, + "Standard_D48ls_v5": 2.304000, + "Standard_D48s_v3": 2.952000, + "Standard_D48s_v4": 2.832000, + "Standard_D48s_v5": 2.832000, + "Standard_D4_v2": 0.660000, + "Standard_D4_v2_Promo": 0.660000, + "Standard_D4_v3": 0.246000, + "Standard_D4_v4": 0.236000, + "Standard_D4_v5": 0.236000, + "Standard_D4a_v4": 0.236000, + "Standard_D4ads_v5": 0.254000, + "Standard_D4as_v4": 0.236000, + "Standard_D4as_v5": 0.396000, + "Standard_D4as_v5_Promo": 0.188000, + "Standard_D4d_v4": 0.278000, + "Standard_D4d_v5": 0.278000, + "Standard_D4ds_v4": 0.278000, + "Standard_D4ds_v5": 0.278000, + "Standard_D4lds_v5": 0.220000, + "Standard_D4ls_v5": 0.192000, + "Standard_D4s_v3": 0.246000, + "Standard_D4s_v4": 0.236000, + "Standard_D4s_v5": 0.236000, + "Standard_D5_v2": 1.321000, + "Standard_D5_v2_Promo": 1.321000, + "Standard_D64-16s_v3": 3.936000, + "Standard_D64-32s_v3": 3.936000, + "Standard_D64_v3": 3.936000, + "Standard_D64_v4": 3.776000, + "Standard_D64_v5": 3.776000, + "Standard_D64a_v4": 3.776000, + "Standard_D64ads_v5": 4.064000, + "Standard_D64as_v4": 3.776000, + "Standard_D64as_v5": 6.336000, + "Standard_D64as_v5_Promo": 3.008000, + "Standard_D64d_v4": 4.448000, + "Standard_D64d_v5": 4.448000, + "Standard_D64ds_v4": 4.448000, + "Standard_D64ds_v5": 4.448000, + "Standard_D64lds_v5": 3.520000, + "Standard_D64ls_v5": 3.072000, + "Standard_D64s_v3": 3.936000, + "Standard_D64s_v4": 3.776000, + "Standard_D64s_v5": 3.776000, + "Standard_D8_v3": 0.492000, + "Standard_D8_v4": 0.472000, + "Standard_D8_v5": 0.472000, + "Standard_D8a_v4": 0.472000, + "Standard_D8ads_v5": 0.876000, + "Standard_D8as_v4": 0.472000, + "Standard_D8as_v5": 0.792000, + "Standard_D8as_v5_Promo": 0.376000, + "Standard_D8d_v4": 0.556000, + "Standard_D8d_v5": 0.556000, + "Standard_D8ds_v4": 0.556000, + "Standard_D8ds_v5": 0.556000, + "Standard_D8lds_v5": 0.440000, + "Standard_D8ls_v5": 0.384000, + "Standard_D8s_v3": 0.492000, + "Standard_D8s_v4": 0.472000, + "Standard_D8s_v5": 0.472000, + "Standard_D96_v5": 5.664000, + "Standard_D96a_v4": 5.664000, + "Standard_D96ads_v5": 6.096000, + "Standard_D96as_v4": 5.664000, + "Standard_D96as_v5": 5.088000, + "Standard_D96as_v5_Promo": 4.512000, + "Standard_D96d_v5": 6.672000, + "Standard_D96ds_v5": 6.672000, + "Standard_D96lds_v5": 5.280000, + "Standard_D96ls_v5": 4.608000, + "Standard_D96s_v5": 5.664000, + "Standard_DS11-1_v2": 0.200000, + "Standard_DS11_v2": 0.200000, + "Standard_DS11_v2_Promo": 0.200000, + "Standard_DS12-1_v2": 0.399000, + "Standard_DS12-2_v2": 0.399000, + "Standard_DS12_v2": 0.399000, + "Standard_DS12_v2_Promo": 0.399000, + "Standard_DS13-2_v2": 0.798000, + "Standard_DS13-4_v2": 0.798000, + "Standard_DS13_v2": 0.798000, + "Standard_DS13_v2_Promo": 0.798000, + "Standard_DS14-4_v2": 1.596000, + "Standard_DS14-8_v2": 1.596000, + "Standard_DS14_v2": 1.596000, + "Standard_DS14_v2_Promo": 1.596000, + "Standard_DS15_v2": 1.995000, + "Standard_DS15i_v2": 1.995000, + "Standard_DS1_v2": 0.082500, + "Standard_DS2_v2": 0.165000, + "Standard_DS2_v2_Promo": 0.165000, + "Standard_DS3_v2": 0.330000, + "Standard_DS3_v2_Promo": 0.330000, + "Standard_DS4_v2": 0.660000, + "Standard_DS4_v2_Promo": 0.660000, + "Standard_DS5_v2": 1.321000, + "Standard_DS5_v2_Promo": 1.321000, + "Standard_E104i_v5": 8.694000, + "Standard_E104id_v5": 9.896000, + "Standard_E104ids_v5": 9.896000, + "Standard_E104is_v5": 8.694000, + "Standard_E112iads_v5": 9.733000, + "Standard_E112ias_v5": 8.378000, + "Standard_E112ibds_v5": 12.258000, + "Standard_E112ibs_v5": 10.965000, + "Standard_E16-4ads_v5": 2.000000, + "Standard_E16-4as_v4": 1.216000, + "Standard_E16-4as_v5": 1.088000, + "Standard_E16-4ds_v4": 1.384000, + "Standard_E16-4ds_v5": 1.384000, + "Standard_E16-4s_v3": 1.280000, + "Standard_E16-4s_v4": 1.216000, + "Standard_E16-4s_v5": 1.216000, + "Standard_E16-8ads_v5": 2.000000, + "Standard_E16-8as_v4": 1.216000, + "Standard_E16-8as_v5": 1.824000, + "Standard_E16-8ds_v4": 1.384000, + "Standard_E16-8ds_v5": 1.384000, + "Standard_E16-8s_v3": 1.280000, + "Standard_E16-8s_v4": 1.216000, + "Standard_E16-8s_v5": 1.216000, + "Standard_E16_v3": 1.280000, + "Standard_E16_v4": 1.216000, + "Standard_E16_v5": 1.216000, + "Standard_E16a_v4": 1.216000, + "Standard_E16ads_v5": 1.310000, + "Standard_E16as_v4": 1.216000, + "Standard_E16as_v5": 1.134000, + "Standard_E16bds_v5": 1.592000, + "Standard_E16bs_v5": 1.424000, + "Standard_E16d_v4": 1.384000, + "Standard_E16d_v5": 1.384000, + "Standard_E16ds_v4": 1.384000, + "Standard_E16ds_v5": 1.384000, + "Standard_E16s_v3": 1.280000, + "Standard_E16s_v4": 1.216000, + "Standard_E16s_v5": 1.216000, + "Standard_E20_v3": 1.600000, + "Standard_E20_v4": 1.520000, + "Standard_E20_v5": 1.520000, + "Standard_E20a_v4": 1.520000, + "Standard_E20ads_v5": 1.580000, + "Standard_E20as_v4": 1.520000, + "Standard_E20as_v5": 1.406000, + "Standard_E20d_v4": 1.730000, + "Standard_E20d_v5": 1.730000, + "Standard_E20ds_v4": 1.730000, + "Standard_E20ds_v5": 1.730000, + "Standard_E20s_v3": 1.600000, + "Standard_E20s_v4": 1.520000, + "Standard_E20s_v5": 1.520000, + "Standard_E2_v3": 0.160000, + "Standard_E2_v4": 0.152000, + "Standard_E2_v5": 0.152000, + "Standard_E2a_v4": 0.152000, + "Standard_E2ads_v5": 0.158000, + "Standard_E2as_v4": 0.152000, + "Standard_E2as_v5": 0.182000, + "Standard_E2bds_v5": 0.199000, + "Standard_E2bs_v5": 0.178000, + "Standard_E2d_v4": 0.173000, + "Standard_E2d_v5": 0.173000, + "Standard_E2ds_v4": 0.173000, + "Standard_E2ds_v5": 0.173000, + "Standard_E2s_v3": 0.160000, + "Standard_E2s_v4": 0.152000, + "Standard_E2s_v5": 0.152000, + "Standard_E32-16ads_v5": 4.000000, + "Standard_E32-16as_v4": 2.432000, + "Standard_E32-16as_v5": 3.648000, + "Standard_E32-16ds_v4": 2.768000, + "Standard_E32-16ds_v5": 2.768000, + "Standard_E32-16s_v3": 2.560000, + "Standard_E32-16s_v4": 2.432000, + "Standard_E32-16s_v5": 2.432000, + "Standard_E32-8ads_v5": 2.528000, + "Standard_E32-8as_v4": 2.432000, + "Standard_E32-8as_v5": 2.176000, + "Standard_E32-8ds_v4": 2.768000, + "Standard_E32-8ds_v5": 2.768000, + "Standard_E32-8s_v3": 2.560000, + "Standard_E32-8s_v4": 2.432000, + "Standard_E32-8s_v5": 2.432000, + "Standard_E32_v3": 2.560000, + "Standard_E32_v4": 2.432000, + "Standard_E32_v5": 2.432000, + "Standard_E32a_v4": 2.432000, + "Standard_E32ads_v5": 2.528000, + "Standard_E32as_v4": 2.432000, + "Standard_E32as_v5": 2.176000, + "Standard_E32bds_v5": 3.184000, + "Standard_E32bs_v5": 2.848000, + "Standard_E32d_v4": 2.768000, + "Standard_E32d_v5": 2.768000, + "Standard_E32ds_v4": 2.768000, + "Standard_E32ds_v5": 2.768000, + "Standard_E32s_v3": 2.560000, + "Standard_E32s_v4": 2.432000, + "Standard_E32s_v5": 2.432000, + "Standard_E4-2ads_v5": 0.316000, + "Standard_E4-2as_v4": 0.304000, + "Standard_E4-2as_v5": 0.456000, + "Standard_E4-2ds_v4": 0.346000, + "Standard_E4-2ds_v5": 0.346000, + "Standard_E4-2s_v3": 0.320000, + "Standard_E4-2s_v4": 0.304000, + "Standard_E4-2s_v5": 0.304000, + "Standard_E48_v3": 3.647000, + "Standard_E48_v4": 3.648000, + "Standard_E48_v5": 3.648000, + "Standard_E48a_v4": 3.648000, + "Standard_E48ads_v5": 3.838000, + "Standard_E48as_v4": 3.648000, + "Standard_E48as_v5": 3.264000, + "Standard_E48bds_v5": 4.776000, + "Standard_E48bs_v5": 4.272000, + "Standard_E48d_v4": 4.152000, + "Standard_E48d_v5": 4.152000, + "Standard_E48ds_v4": 4.152000, + "Standard_E48ds_v5": 4.152000, + "Standard_E48s_v3": 3.647000, + "Standard_E48s_v4": 3.648000, + "Standard_E48s_v5": 3.648000, + "Standard_E4_v3": 0.320000, + "Standard_E4_v4": 0.304000, + "Standard_E4_v5": 0.304000, + "Standard_E4a_v4": 0.304000, + "Standard_E4ads_v5": 0.362000, + "Standard_E4as_v4": 0.304000, + "Standard_E4as_v5": 0.318000, + "Standard_E4bds_v5": 0.398000, + "Standard_E4bs_v5": 0.356000, + "Standard_E4d_v4": 0.346000, + "Standard_E4d_v5": 0.346000, + "Standard_E4ds_v4": 0.346000, + "Standard_E4ds_v5": 0.346000, + "Standard_E4s_v3": 0.320000, + "Standard_E4s_v4": 0.304000, + "Standard_E4s_v5": 0.304000, + "Standard_E64-16ads_v5": 8.000000, + "Standard_E64-16as_v4": 4.864000, + "Standard_E64-16as_v5": 7.296000, + "Standard_E64-16ds_v4": 5.536000, + "Standard_E64-16ds_v5": 5.536000, + "Standard_E64-16s_v3": 4.376000, + "Standard_E64-16s_v4": 4.864000, + "Standard_E64-16s_v5": 4.864000, + "Standard_E64-32ads_v5": 8.000000, + "Standard_E64-32as_v4": 4.864000, + "Standard_E64-32as_v5": 7.296000, + "Standard_E64-32ds_v4": 5.536000, + "Standard_E64-32ds_v5": 5.536000, + "Standard_E64-32s_v3": 4.376000, + "Standard_E64-32s_v4": 4.864000, + "Standard_E64-32s_v5": 4.864000, + "Standard_E64_v3": 4.376000, + "Standard_E64_v4": 4.864000, + "Standard_E64_v5": 4.864000, + "Standard_E64a_v4": 4.864000, + "Standard_E64ads_v5": 5.102000, + "Standard_E64as_v4": 4.864000, + "Standard_E64as_v5": 4.398000, + "Standard_E64bds_v5": 6.368000, + "Standard_E64bs_v5": 5.696000, + "Standard_E64d_v4": 5.536000, + "Standard_E64d_v5": 5.536000, + "Standard_E64ds_v4": 5.536000, + "Standard_E64ds_v5": 5.536000, + "Standard_E64i_v3": 4.376000, + "Standard_E64is_v3": 4.376000, + "Standard_E64s_v3": 4.376000, + "Standard_E64s_v4": 4.864000, + "Standard_E64s_v5": 4.864000, + "Standard_E8-2ads_v5": 1.000000, + "Standard_E8-2as_v4": 0.608000, + "Standard_E8-2as_v5": 0.544000, + "Standard_E8-2ds_v4": 0.692000, + "Standard_E8-2ds_v5": 0.692000, + "Standard_E8-2s_v3": 0.640000, + "Standard_E8-2s_v4": 0.608000, + "Standard_E8-2s_v5": 0.608000, + "Standard_E8-4ads_v5": 0.632000, + "Standard_E8-4as_v4": 0.608000, + "Standard_E8-4as_v5": 0.912000, + "Standard_E8-4ds_v4": 0.692000, + "Standard_E8-4ds_v5": 0.692000, + "Standard_E8-4s_v3": 0.640000, + "Standard_E8-4s_v4": 0.608000, + "Standard_E8-4s_v5": 0.608000, + "Standard_E80ids_v4": 6.920000, + "Standard_E80is_v4": 6.080000, + "Standard_E8_v3": 0.640000, + "Standard_E8_v4": 0.608000, + "Standard_E8_v5": 0.608000, + "Standard_E8a_v4": 0.608000, + "Standard_E8ads_v5": 0.632000, + "Standard_E8as_v4": 0.608000, + "Standard_E8as_v5": 0.544000, + "Standard_E8bds_v5": 0.796000, + "Standard_E8bs_v5": 0.712000, + "Standard_E8d_v4": 0.692000, + "Standard_E8d_v5": 0.692000, + "Standard_E8ds_v4": 0.692000, + "Standard_E8ds_v5": 0.692000, + "Standard_E8s_v3": 0.640000, + "Standard_E8s_v4": 0.608000, + "Standard_E8s_v5": 0.608000, + "Standard_E96-24ads_v5": 7.584000, + "Standard_E96-24as_v4": 7.296000, + "Standard_E96-24as_v5": 6.528000, + "Standard_E96-24ds_v5": 8.304000, + "Standard_E96-24s_v5": 7.296000, + "Standard_E96-48ads_v5": 7.584000, + "Standard_E96-48as_v4": 7.296000, + "Standard_E96-48as_v5": 6.528000, + "Standard_E96-48ds_v5": 8.304000, + "Standard_E96-48s_v5": 7.296000, + "Standard_E96_v5": 7.296000, + "Standard_E96a_v4": 7.296000, + "Standard_E96ads_v5": 7.584000, + "Standard_E96as_v4": 7.296000, + "Standard_E96as_v5": 6.528000, + "Standard_E96bds_v5": 9.552000, + "Standard_E96bs_v5": 8.544000, + "Standard_E96d_v5": 8.304000, + "Standard_E96ds_v5": 8.304000, + "Standard_E96iads_v5": 8.342000, + "Standard_E96ias_v5": 7.181000, + "Standard_E96s_v5": 7.296000, + "Standard_F1": 0.051000, + "Standard_F16": 0.816000, + "Standard_F16s": 0.816000, + "Standard_F16s_v2": 0.768000, + "Standard_F1s": 0.051000, + "Standard_F2": 0.102000, + "Standard_F2s": 0.102000, + "Standard_F2s_v2": 0.096000, + "Standard_F32s_v2": 1.536000, + "Standard_F4": 0.204000, + "Standard_F48s_v2": 2.304000, + "Standard_F4s": 0.204000, + "Standard_F4s_v2": 0.192000, + "Standard_F64s_v2": 3.072000, + "Standard_F72s_v2": 3.456000, + "Standard_F8": 0.408000, + "Standard_F8s": 0.408000, + "Standard_F8s_v2": 0.384000, + "Standard_FX12mds": 1.350000, + "Standard_FX24mds": 2.700000, + "Standard_FX36mds": 4.050000, + "Standard_FX48mds": 5.400000, + "Standard_FX4mds": 0.450000, + "Standard_HB176-144rs_v4": 9.720000, + "Standard_HB176-24rs_v4": 9.720000, + "Standard_HB176-48rs_v4": 9.720000, + "Standard_HB176-96rs_v4": 9.720000, + "Standard_HB176rs_v4": 9.720000, + "Standard_HC44-16rs": 4.277000, + "Standard_HC44-32rs": 4.277000, + "Standard_HC44rs": 4.277000, + "Standard_HX176-144rs": 11.664000, + "Standard_HX176-24rs": 11.664000, + "Standard_HX176-48rs": 11.664000, + "Standard_HX176-96rs": 11.664000, + "Standard_HX176rs": 11.664000, + "Standard_L16as_v3": 1.464000, + "Standard_L16s_v3": 1.636000, + "Standard_L32as_v3": 2.928000, + "Standard_L32s_v3": 3.272000, + "Standard_L48as_v3": 4.392000, + "Standard_L48s_v3": 4.908000, + "Standard_L64as_v3": 5.856000, + "Standard_L64s_v3": 6.544000, + "Standard_L80as_v3": 7.320000, + "Standard_L80s_v3": 8.180000, + "Standard_L8as_v3": 0.732000, + "Standard_L8s_v3": 0.818000, + "Standard_M128": 19.340000, + "Standard_M128-32ms": 38.698000, + "Standard_M128-64ms": 38.698000, + "Standard_M128dms_v2": 38.701000, + "Standard_M128ds_v2": 19.343000, + "Standard_M128m": 38.698000, + "Standard_M128ms": 38.698000, + "Standard_M128ms_v2": 38.090000, + "Standard_M128s": 19.340000, + "Standard_M128s_v2": 18.733000, + "Standard_M16-4ms": 4.456000, + "Standard_M16-8ms": 4.456000, + "Standard_M16ms": 4.456000, + "Standard_M16s": 3.461000, + "Standard_M192idms_v2": 46.493000, + "Standard_M192ids_v2": 23.246000, + "Standard_M192ims_v2": 45.882000, + "Standard_M192is_v2": 22.636000, + "Standard_M208-104ms_v2": 53.544000, + "Standard_M208-104s_v2": 26.772000, + "Standard_M208-52ms_v2": 53.544000, + "Standard_M208-52s_v2": 26.772000, + "Standard_M208ms_v2": 53.544000, + "Standard_M208s_v2": 26.772000, + "Standard_M32-16ms": 8.912000, + "Standard_M32-8ms": 8.912000, + "Standard_M32dms_v2": 8.912000, + "Standard_M32ls": 3.448000, + "Standard_M32ms": 8.912000, + "Standard_M32ms_v2": 8.759000, + "Standard_M32s": 4.836000, + "Standard_M32ts": 3.248000, + "Standard_M416-104ms_v2": 118.980000, + "Standard_M416-104s_v2": 59.496000, + "Standard_M416-208ms_v2": 118.980000, + "Standard_M416-208s_v2": 59.496000, + "Standard_M416is_v2": 59.496000, + "Standard_M416ms_v2": 118.980000, + "Standard_M416s_8_v2": 79.750000, + "Standard_M416s_v2": 59.496000, + "Standard_M64": 9.670000, + "Standard_M64-16ms": 14.989000, + "Standard_M64-32ms": 14.989000, + "Standard_M64dms_v2": 14.993000, + "Standard_M64ds_v2": 9.670000, + "Standard_M64ls": 6.498000, + "Standard_M64m": 14.989000, + "Standard_M64ms": 14.989000, + "Standard_M64ms_v2": 14.688000, + "Standard_M64s": 9.670000, + "Standard_M64s_v2": 9.365000, + "Standard_M8-2ms": 2.228000, + "Standard_M8-4ms": 2.228000, + "Standard_M8ms": 2.228000, + "Standard_NC12s_v3": 8.468000, + "Standard_NC16as_T4_v3": 1.481000, + "Standard_NC24rs_v3": 18.630000, + "Standard_NC24s_v3": 16.936000, + "Standard_NC4as_T4_v3": 0.647000, + "Standard_NC64as_T4_v3": 5.353000, + "Standard_NC6s_v3": 4.234000, + "Standard_NC8as_T4_v3": 0.925000, + "Standard_NV12ads_A10_v5": 1.226000, + "Standard_NV16as_v4": 1.258000, + "Standard_NV18ads_A10_v5": 2.160000, + "Standard_NV32as_v4": 2.516000, + "Standard_NV36adms_A10_v5": 6.102000, + "Standard_NV36ads_A10_v5": 4.320000, + "Standard_NV4as_v4": 0.315000, + "Standard_NV6ads_A10_v5": 0.613000, + "Standard_NV72ads_A10_v5": 8.802000, + "Standard_NV8as_v4": 0.629000, + } + // koreasouth + initialOnDemandPrices["koreasouth"] = map[string]float64{ + "Basic_A0": 0.016000, + "Basic_A1": 0.029000, + "Basic_A2": 0.097000, + "Basic_A3": 0.202000, + "Basic_A4": 0.404000, + "Dadsv5_Type1": 7.297000, + "Dasv4_Type1": 5.847000, + "Dasv4_Type2": 5.847000, + "Dasv5_Type1": 6.092000, + "Ddsv4_Type 1": 4.589000, + "Ddsv4_Type2": 5.449000, + "Ddsv5_Type1": 6.861000, + "Dsv3_Type3": 4.859000, + "Dsv3_Type4": 6.073000, + "Dsv4_Type1": 4.873000, + "Dsv4_Type2": 6.091000, + "Dsv5_Type1": 6.072000, + "Eadsv5_Type1": 7.954000, + "Easv4_Type1": 7.603000, + "Easv4_Type2": 7.603000, + "Easv5_Type1": 6.861000, + "Ebdsv5-Type1": 6.760000, + "Ebsv5-Type1": 6.032000, + "Edsv4_Type 1": 5.793000, + "Edsv4_Type2": 6.879000, + "Edsv5_Type1": 8.744000, + "Esv3_Type3": 4.596000, + "Esv3_Type4": 6.227000, + "Esv4_Type1": 5.069000, + "Esv4_Type2": 6.653000, + "Esv5_Type1": 7.651000, + "Fsv2 Type3": 4.858000, + "Fsv2_Type2": 3.802000, + "Fsv2_Type4": 5.280000, + "Lasv3_Type1": 7.894000, + "Lsv3_Type1": 8.830000, + "Msmv2_Type1": 136.330000, + "Msv2_Type1": 68.173000, + "Standard_A0": 0.018000, + "Standard_A1": 0.060000, + "Standard_A1_v2": 0.044000, + "Standard_A2": 0.119000, + "Standard_A2_v2": 0.092000, + "Standard_A2m_v2": 0.119000, + "Standard_A3": 0.238000, + "Standard_A4": 0.475000, + "Standard_A4_v2": 0.193000, + "Standard_A4m_v2": 0.251000, + "Standard_A5": 0.155000, + "Standard_A6": 0.309000, + "Standard_A7": 0.618000, + "Standard_A8_v2": 0.405000, + "Standard_A8m_v2": 0.526000, + "Standard_B12ms": 0.622000, + "Standard_B16als_v2": 0.613000, + "Standard_B16as_v2": 0.692000, + "Standard_B16ls_v2": 0.751000, + "Standard_B16ms": 0.830000, + "Standard_B16s_v2": 0.765000, + "Standard_B1ls": 0.007020, + "Standard_B1ms": 0.026000, + "Standard_B1s": 0.013000, + "Standard_B20ms": 1.037000, + "Standard_B2als_v2": 0.043200, + "Standard_B2as_v2": 0.095700, + "Standard_B2ats_v2": 0.020000, + "Standard_B2ls_v2": 0.057000, + "Standard_B2ms": 0.104000, + "Standard_B2s": 0.052000, + "Standard_B2s_v2": 0.095700, + "Standard_B2ts_v2": 0.012000, + "Standard_B32als_v2": 1.225000, + "Standard_B32as_v2": 1.384000, + "Standard_B32ls_v2": 1.503000, + "Standard_B32s_v2": 1.678000, + "Standard_B4als_v2": 0.153000, + "Standard_B4as_v2": 0.173000, + "Standard_B4ls_v2": 0.188000, + "Standard_B4ms": 0.207000, + "Standard_B4s_v2": 0.210000, + "Standard_B8als_v2": 0.306000, + "Standard_B8as_v2": 0.346000, + "Standard_B8ls_v2": 0.376000, + "Standard_B8ms": 0.415000, + "Standard_B8s_v2": 0.420000, + "Standard_D11_v2": 0.179000, + "Standard_D11_v2_Promo": 0.179000, + "Standard_D12_v2": 0.359000, + "Standard_D12_v2_Promo": 0.359000, + "Standard_D13_v2": 0.718000, + "Standard_D13_v2_Promo": 0.718000, + "Standard_D14_v2": 1.435000, + "Standard_D14_v2_Promo": 1.435000, + "Standard_D15_v2": 1.794000, + "Standard_D15i_v2": 1.794000, + "Standard_D16_v3": 0.886000, + "Standard_D16_v4": 0.886000, + "Standard_D16_v5": 0.883000, + "Standard_D16a_v4": 0.886000, + "Standard_D16ads_v5": 1.684000, + "Standard_D16as_v4": 0.886000, + "Standard_D16as_v5": 0.791000, + "Standard_D16d_v4": 1.043000, + "Standard_D16d_v5": 1.040000, + "Standard_D16ds_v4": 1.043000, + "Standard_D16ds_v5": 1.040000, + "Standard_D16lds_v5": 0.883000, + "Standard_D16ls_v5": 0.782000, + "Standard_D16s_v3": 0.886000, + "Standard_D16s_v4": 0.886000, + "Standard_D16s_v5": 0.883000, + "Standard_D1_v2": 0.074000, + "Standard_D2_v2": 0.149000, + "Standard_D2_v2_Promo": 0.149000, + "Standard_D2_v3": 0.111000, + "Standard_D2_v4": 0.111000, + "Standard_D2_v5": 0.110000, + "Standard_D2a_v4": 0.111000, + "Standard_D2ads_v5": 0.118000, + "Standard_D2as_v4": 0.111000, + "Standard_D2as_v5": 0.098900, + "Standard_D2d_v4": 0.130000, + "Standard_D2d_v5": 0.130000, + "Standard_D2ds_v4": 0.130000, + "Standard_D2ds_v5": 0.130000, + "Standard_D2lds_v5": 0.110000, + "Standard_D2ls_v5": 0.097800, + "Standard_D2s_v3": 0.111000, + "Standard_D2s_v4": 0.111000, + "Standard_D2s_v5": 0.110000, + "Standard_D32-16s_v3": 1.772000, + "Standard_D32-8s_v3": 1.772000, + "Standard_D32_v3": 1.772000, + "Standard_D32_v4": 1.772000, + "Standard_D32_v5": 1.766000, + "Standard_D32a_v4": 1.772000, + "Standard_D32ads_v5": 1.895000, + "Standard_D32as_v4": 1.772000, + "Standard_D32as_v5": 1.582000, + "Standard_D32d_v4": 2.086000, + "Standard_D32d_v5": 2.079000, + "Standard_D32ds_v4": 2.086000, + "Standard_D32ds_v5": 2.079000, + "Standard_D32lds_v5": 1.766000, + "Standard_D32ls_v5": 1.564000, + "Standard_D32s_v3": 1.772000, + "Standard_D32s_v4": 1.772000, + "Standard_D32s_v5": 1.766000, + "Standard_D3_v2": 0.297000, + "Standard_D3_v2_Promo": 0.297000, + "Standard_D48_v3": 2.658000, + "Standard_D48_v4": 2.658000, + "Standard_D48_v5": 2.650000, + "Standard_D48a_v4": 2.658000, + "Standard_D48ads_v5": 5.051000, + "Standard_D48as_v4": 2.658000, + "Standard_D48as_v5": 2.374000, + "Standard_D48d_v4": 3.129000, + "Standard_D48d_v5": 3.119000, + "Standard_D48ds_v4": 3.129000, + "Standard_D48ds_v5": 3.119000, + "Standard_D48lds_v5": 2.650000, + "Standard_D48ls_v5": 2.346000, + "Standard_D48s_v3": 2.658000, + "Standard_D48s_v4": 2.658000, + "Standard_D48s_v5": 2.650000, + "Standard_D4_v2": 0.594000, + "Standard_D4_v2_Promo": 0.594000, + "Standard_D4_v3": 0.221000, + "Standard_D4_v4": 0.221000, + "Standard_D4_v5": 0.221000, + "Standard_D4a_v4": 0.221000, + "Standard_D4ads_v5": 0.421000, + "Standard_D4as_v4": 0.221000, + "Standard_D4as_v5": 0.198000, + "Standard_D4d_v4": 0.261000, + "Standard_D4d_v5": 0.260000, + "Standard_D4ds_v4": 0.261000, + "Standard_D4ds_v5": 0.260000, + "Standard_D4lds_v5": 0.221000, + "Standard_D4ls_v5": 0.196000, + "Standard_D4s_v3": 0.221000, + "Standard_D4s_v4": 0.221000, + "Standard_D4s_v5": 0.221000, + "Standard_D5_v2": 1.189000, + "Standard_D5_v2_Promo": 1.189000, + "Standard_D64-16s_v3": 3.544000, + "Standard_D64-32s_v3": 3.544000, + "Standard_D64_v3": 3.544000, + "Standard_D64_v4": 3.544000, + "Standard_D64_v5": 3.533000, + "Standard_D64a_v4": 3.544000, + "Standard_D64ads_v5": 6.734000, + "Standard_D64as_v4": 3.544000, + "Standard_D64as_v5": 6.109000, + "Standard_D64d_v4": 4.171000, + "Standard_D64d_v5": 4.158000, + "Standard_D64ds_v4": 4.171000, + "Standard_D64ds_v5": 4.158000, + "Standard_D64lds_v5": 3.533000, + "Standard_D64ls_v5": 3.128000, + "Standard_D64s_v3": 3.544000, + "Standard_D64s_v4": 3.544000, + "Standard_D64s_v5": 3.533000, + "Standard_D8_v3": 0.443000, + "Standard_D8_v4": 0.443000, + "Standard_D8_v5": 0.442000, + "Standard_D8a_v4": 0.443000, + "Standard_D8ads_v5": 0.474000, + "Standard_D8as_v4": 0.443000, + "Standard_D8as_v5": 0.396000, + "Standard_D8d_v4": 0.521000, + "Standard_D8d_v5": 0.520000, + "Standard_D8ds_v4": 0.521000, + "Standard_D8ds_v5": 0.520000, + "Standard_D8lds_v5": 0.442000, + "Standard_D8ls_v5": 0.391000, + "Standard_D8s_v3": 0.443000, + "Standard_D8s_v4": 0.443000, + "Standard_D8s_v5": 0.442000, + "Standard_D96_v5": 5.299000, + "Standard_D96a_v4": 5.316000, + "Standard_D96ads_v5": 10.102000, + "Standard_D96as_v4": 5.316000, + "Standard_D96as_v5": 9.163000, + "Standard_D96d_v5": 6.238000, + "Standard_D96ds_v5": 6.238000, + "Standard_D96lds_v5": 5.299000, + "Standard_D96ls_v5": 4.692000, + "Standard_D96s_v5": 5.299000, + "Standard_DS11-1_v2": 0.179000, + "Standard_DS11_v2": 0.179000, + "Standard_DS11_v2_Promo": 0.179000, + "Standard_DS12-1_v2": 0.359000, + "Standard_DS12-2_v2": 0.359000, + "Standard_DS12_v2": 0.359000, + "Standard_DS12_v2_Promo": 0.359000, + "Standard_DS13-2_v2": 0.718000, + "Standard_DS13-4_v2": 0.718000, + "Standard_DS13_v2": 0.718000, + "Standard_DS13_v2_Promo": 0.718000, + "Standard_DS14-4_v2": 1.435000, + "Standard_DS14-8_v2": 1.435000, + "Standard_DS14_v2": 1.435000, + "Standard_DS14_v2_Promo": 1.435000, + "Standard_DS15_v2": 1.794000, + "Standard_DS15i_v2": 1.794000, + "Standard_DS1_v2": 0.074000, + "Standard_DS2_v2": 0.149000, + "Standard_DS2_v2_Promo": 0.149000, + "Standard_DS3_v2": 0.297000, + "Standard_DS3_v2_Promo": 0.297000, + "Standard_DS4_v2": 0.594000, + "Standard_DS4_v2_Promo": 0.594000, + "Standard_DS5_v2": 1.189000, + "Standard_DS5_v2_Promo": 1.189000, + "Standard_E104i_v5": 8.288000, + "Standard_E104id_v5": 9.472000, + "Standard_E104ids_v5": 9.472000, + "Standard_E104is_v5": 8.288000, + "Standard_E112iads_v5": 9.280000, + "Standard_E112ias_v5": 8.005000, + "Standard_E112ibds_v5": 11.830000, + "Standard_E112ibs_v5": 10.555000, + "Standard_E16-4ads_v5": 1.205000, + "Standard_E16-4as_v4": 1.152000, + "Standard_E16-4as_v5": 1.040000, + "Standard_E16-4ds_v4": 1.317000, + "Standard_E16-4ds_v5": 1.325000, + "Standard_E16-4s_v3": 1.152000, + "Standard_E16-4s_v4": 1.152000, + "Standard_E16-4s_v5": 1.159000, + "Standard_E16-8ads_v5": 1.205000, + "Standard_E16-8as_v4": 1.152000, + "Standard_E16-8as_v5": 1.040000, + "Standard_E16-8ds_v4": 1.317000, + "Standard_E16-8ds_v5": 1.325000, + "Standard_E16-8s_v3": 1.152000, + "Standard_E16-8s_v4": 1.152000, + "Standard_E16-8s_v5": 1.159000, + "Standard_E16_v3": 1.152000, + "Standard_E16_v4": 1.152000, + "Standard_E16_v5": 1.159000, + "Standard_E16a_v4": 1.152000, + "Standard_E16ads_v5": 1.205000, + "Standard_E16as_v4": 1.152000, + "Standard_E16as_v5": 1.040000, + "Standard_E16bds_v5": 1.536000, + "Standard_E16bs_v5": 1.371000, + "Standard_E16d_v4": 1.317000, + "Standard_E16d_v5": 1.325000, + "Standard_E16ds_v4": 1.317000, + "Standard_E16ds_v5": 1.325000, + "Standard_E16s_v3": 1.152000, + "Standard_E16s_v4": 1.152000, + "Standard_E16s_v5": 1.159000, + "Standard_E20_v3": 1.440000, + "Standard_E20_v4": 1.440000, + "Standard_E20_v5": 1.449000, + "Standard_E20a_v4": 1.440000, + "Standard_E20ads_v5": 1.506000, + "Standard_E20as_v4": 1.440000, + "Standard_E20as_v5": 1.300000, + "Standard_E20d_v4": 1.646000, + "Standard_E20d_v5": 1.656000, + "Standard_E20ds_v4": 1.646000, + "Standard_E20ds_v5": 1.656000, + "Standard_E20s_v3": 1.440000, + "Standard_E20s_v4": 1.440000, + "Standard_E20s_v5": 1.449000, + "Standard_E2_v3": 0.144000, + "Standard_E2_v4": 0.144000, + "Standard_E2_v5": 0.145000, + "Standard_E2a_v4": 0.144000, + "Standard_E2ads_v5": 0.197000, + "Standard_E2as_v4": 0.144000, + "Standard_E2as_v5": 0.176000, + "Standard_E2bds_v5": 0.192000, + "Standard_E2bs_v5": 0.171000, + "Standard_E2d_v4": 0.165000, + "Standard_E2d_v5": 0.166000, + "Standard_E2ds_v4": 0.165000, + "Standard_E2ds_v5": 0.166000, + "Standard_E2s_v3": 0.144000, + "Standard_E2s_v4": 0.144000, + "Standard_E2s_v5": 0.145000, + "Standard_E32-16ads_v5": 3.882000, + "Standard_E32-16as_v4": 2.304000, + "Standard_E32-16as_v5": 3.551000, + "Standard_E32-16ds_v4": 2.633000, + "Standard_E32-16ds_v5": 2.650000, + "Standard_E32-16s_v3": 2.304000, + "Standard_E32-16s_v4": 2.304000, + "Standard_E32-16s_v5": 2.318000, + "Standard_E32-8ads_v5": 3.882000, + "Standard_E32-8as_v4": 2.304000, + "Standard_E32-8as_v5": 3.551000, + "Standard_E32-8ds_v4": 2.633000, + "Standard_E32-8ds_v5": 2.650000, + "Standard_E32-8s_v3": 2.304000, + "Standard_E32-8s_v4": 2.304000, + "Standard_E32-8s_v5": 2.318000, + "Standard_E32_v3": 2.304000, + "Standard_E32_v4": 2.304000, + "Standard_E32_v5": 2.318000, + "Standard_E32a_v4": 2.304000, + "Standard_E32ads_v5": 2.456000, + "Standard_E32as_v4": 2.304000, + "Standard_E32as_v5": 2.125000, + "Standard_E32bds_v5": 3.073000, + "Standard_E32bs_v5": 2.742000, + "Standard_E32d_v4": 2.633000, + "Standard_E32d_v5": 2.650000, + "Standard_E32ds_v4": 2.633000, + "Standard_E32ds_v5": 2.650000, + "Standard_E32s_v3": 2.304000, + "Standard_E32s_v4": 2.304000, + "Standard_E32s_v5": 2.318000, + "Standard_E4-2ads_v5": 0.485000, + "Standard_E4-2as_v4": 0.288000, + "Standard_E4-2as_v5": 0.444000, + "Standard_E4-2ds_v4": 0.329000, + "Standard_E4-2ds_v5": 0.331000, + "Standard_E4-2s_v3": 0.288000, + "Standard_E4-2s_v4": 0.288000, + "Standard_E4-2s_v5": 0.290000, + "Standard_E48_v3": 3.234000, + "Standard_E48_v4": 3.456000, + "Standard_E48_v5": 3.478000, + "Standard_E48a_v4": 3.456000, + "Standard_E48ads_v5": 3.616000, + "Standard_E48as_v4": 3.456000, + "Standard_E48as_v5": 3.165000, + "Standard_E48bds_v5": 4.609000, + "Standard_E48bs_v5": 4.112000, + "Standard_E48d_v4": 3.950000, + "Standard_E48d_v5": 3.974000, + "Standard_E48ds_v4": 3.950000, + "Standard_E48ds_v5": 3.974000, + "Standard_E48s_v3": 3.234000, + "Standard_E48s_v4": 3.456000, + "Standard_E48s_v5": 3.478000, + "Standard_E4_v3": 0.288000, + "Standard_E4_v4": 0.288000, + "Standard_E4_v5": 0.290000, + "Standard_E4a_v4": 0.288000, + "Standard_E4ads_v5": 0.347000, + "Standard_E4as_v4": 0.288000, + "Standard_E4as_v5": 0.306000, + "Standard_E4bds_v5": 0.384000, + "Standard_E4bs_v5": 0.343000, + "Standard_E4d_v4": 0.329000, + "Standard_E4d_v5": 0.331000, + "Standard_E4ds_v4": 0.329000, + "Standard_E4ds_v5": 0.331000, + "Standard_E4s_v3": 0.288000, + "Standard_E4s_v4": 0.288000, + "Standard_E4s_v5": 0.290000, + "Standard_E64-16ads_v5": 7.765000, + "Standard_E64-16as_v4": 4.608000, + "Standard_E64-16as_v5": 4.158000, + "Standard_E64-16ds_v4": 5.266000, + "Standard_E64-16ds_v5": 5.299000, + "Standard_E64-16s_v3": 3.881000, + "Standard_E64-16s_v4": 4.608000, + "Standard_E64-16s_v5": 4.637000, + "Standard_E64-32ads_v5": 7.765000, + "Standard_E64-32as_v4": 4.608000, + "Standard_E64-32as_v5": 4.158000, + "Standard_E64-32ds_v4": 5.266000, + "Standard_E64-32ds_v5": 5.299000, + "Standard_E64-32s_v3": 3.881000, + "Standard_E64-32s_v4": 4.608000, + "Standard_E64-32s_v5": 4.637000, + "Standard_E64_v3": 3.881000, + "Standard_E64_v4": 4.608000, + "Standard_E64_v5": 4.637000, + "Standard_E64a_v4": 4.608000, + "Standard_E64ads_v5": 4.821000, + "Standard_E64as_v4": 4.608000, + "Standard_E64as_v5": 4.204000, + "Standard_E64bds_v5": 6.146000, + "Standard_E64bs_v5": 5.483000, + "Standard_E64d_v4": 5.266000, + "Standard_E64d_v5": 5.299000, + "Standard_E64ds_v4": 5.266000, + "Standard_E64ds_v5": 5.299000, + "Standard_E64i_v3": 3.778000, + "Standard_E64is_v3": 3.778000, + "Standard_E64s_v3": 3.881000, + "Standard_E64s_v4": 4.608000, + "Standard_E64s_v5": 4.637000, + "Standard_E8-2ads_v5": 0.971000, + "Standard_E8-2as_v4": 0.576000, + "Standard_E8-2as_v5": 0.520000, + "Standard_E8-2ds_v4": 0.658000, + "Standard_E8-2ds_v5": 0.662000, + "Standard_E8-2s_v3": 0.576000, + "Standard_E8-2s_v4": 0.576000, + "Standard_E8-2s_v5": 0.580000, + "Standard_E8-4ads_v5": 0.603000, + "Standard_E8-4as_v4": 0.576000, + "Standard_E8-4as_v5": 0.520000, + "Standard_E8-4ds_v4": 0.658000, + "Standard_E8-4ds_v5": 0.662000, + "Standard_E8-4s_v3": 0.576000, + "Standard_E8-4s_v4": 0.576000, + "Standard_E8-4s_v5": 0.580000, + "Standard_E80ids_v4": 7.203720, + "Standard_E80is_v4": 6.329280, + "Standard_E8_v3": 0.576000, + "Standard_E8_v4": 0.576000, + "Standard_E8_v5": 0.580000, + "Standard_E8a_v4": 0.576000, + "Standard_E8ads_v5": 0.603000, + "Standard_E8as_v4": 0.576000, + "Standard_E8as_v5": 0.566000, + "Standard_E8bds_v5": 0.768000, + "Standard_E8bs_v5": 0.685000, + "Standard_E8d_v4": 0.658000, + "Standard_E8d_v5": 0.662000, + "Standard_E8ds_v4": 0.658000, + "Standard_E8ds_v5": 0.662000, + "Standard_E8s_v3": 0.576000, + "Standard_E8s_v4": 0.576000, + "Standard_E8s_v5": 0.580000, + "Standard_E96-24ads_v5": 11.647000, + "Standard_E96-24as_v4": 6.912000, + "Standard_E96-24as_v5": 6.238000, + "Standard_E96-24ds_v5": 7.949000, + "Standard_E96-24s_v5": 6.955000, + "Standard_E96-48ads_v5": 7.231000, + "Standard_E96-48as_v4": 6.912000, + "Standard_E96-48as_v5": 10.654000, + "Standard_E96-48ds_v5": 7.949000, + "Standard_E96-48s_v5": 6.955000, + "Standard_E96_v5": 6.955000, + "Standard_E96a_v4": 6.912000, + "Standard_E96ads_v5": 7.231000, + "Standard_E96as_v4": 6.912000, + "Standard_E96as_v5": 6.238000, + "Standard_E96bds_v5": 9.218000, + "Standard_E96bs_v5": 8.225000, + "Standard_E96d_v5": 7.949000, + "Standard_E96ds_v5": 7.949000, + "Standard_E96iads_v5": 7.954000, + "Standard_E96ias_v5": 6.861000, + "Standard_E96s_v5": 6.955000, + "Standard_F1": 0.051000, + "Standard_F16": 0.816000, + "Standard_F16s": 0.816000, + "Standard_F16s_v2": 0.768000, + "Standard_F1s": 0.051000, + "Standard_F2": 0.102000, + "Standard_F2s": 0.102000, + "Standard_F2s_v2": 0.096100, + "Standard_F32s_v2": 1.537000, + "Standard_F4": 0.204000, + "Standard_F48s_v2": 2.305000, + "Standard_F4s": 0.204000, + "Standard_F4s_v2": 0.192000, + "Standard_F64s_v2": 3.074000, + "Standard_F72s_v2": 3.458000, + "Standard_F8": 0.408000, + "Standard_F8s": 0.408000, + "Standard_F8s_v2": 0.384000, + "Standard_L16as_v3": 1.435000, + "Standard_L16s_v3": 1.605000, + "Standard_L32as_v3": 2.870000, + "Standard_L32s_v3": 3.211000, + "Standard_L48as_v3": 4.306000, + "Standard_L48s_v3": 4.816000, + "Standard_L64as_v3": 5.741000, + "Standard_L64s_v3": 6.422000, + "Standard_L80as_v3": 7.176000, + "Standard_L80s_v3": 8.027000, + "Standard_L8as_v3": 0.718000, + "Standard_L8s_v3": 0.803000, + "Standard_M128": 19.340000, + "Standard_M128-32ms": 38.698000, + "Standard_M128-64ms": 38.698000, + "Standard_M128m": 38.698000, + "Standard_M128ms": 38.698000, + "Standard_M128s": 19.340000, + "Standard_M16-4ms": 4.456000, + "Standard_M16-8ms": 4.456000, + "Standard_M16ms": 4.456000, + "Standard_M16s": 3.461000, + "Standard_M208-104ms_v2": 55.775000, + "Standard_M208-104s_v2": 27.888000, + "Standard_M208-52ms_v2": 55.775000, + "Standard_M208-52s_v2": 27.888000, + "Standard_M208ms_v2": 55.775000, + "Standard_M208s_v2": 27.888000, + "Standard_M32-16ms": 8.912000, + "Standard_M32-8ms": 8.912000, + "Standard_M32ls": 3.448000, + "Standard_M32ms": 8.912000, + "Standard_M32s": 4.836000, + "Standard_M32ts": 3.248000, + "Standard_M416-104ms_v2": 123.940000, + "Standard_M416-104s_v2": 61.975000, + "Standard_M416-208ms_v2": 123.940000, + "Standard_M416-208s_v2": 61.975000, + "Standard_M416is_v2": 61.975000, + "Standard_M416ms_v2": 123.940000, + "Standard_M416s_8_v2": 82.640000, + "Standard_M416s_v2": 61.975000, + "Standard_M64": 9.670000, + "Standard_M64-16ms": 14.989000, + "Standard_M64-32ms": 14.989000, + "Standard_M64ls": 6.498000, + "Standard_M64m": 14.989000, + "Standard_M64ms": 14.989000, + "Standard_M64s": 9.670000, + "Standard_M8-2ms": 2.228000, + "Standard_M8-4ms": 2.228000, + "Standard_M8ms": 2.228000, + "Standard_NV12ads_A10_v5": 1.135000, + "Standard_NV18ads_A10_v5": 2.000000, + "Standard_NV36adms_A10_v5": 5.650000, + "Standard_NV36ads_A10_v5": 4.000000, + "Standard_NV6ads_A10_v5": 0.567000, + "Standard_NV72ads_A10_v5": 8.150000, + } + // northcentralus + initialOnDemandPrices["northcentralus"] = map[string]float64{ + "Basic_A0": 0.018000, + "Basic_A1": 0.023000, + "Basic_A2": 0.085000, + "Basic_A3": 0.188000, + "Basic_A4": 0.376000, + "DCsv2 Type 1": 0.845000, + "Dadsv5_Type1": 6.345000, + "Dasv4_Type1": 5.069000, + "Dasv4_Type2": 5.280000, + "Dasv5_Type1": 5.298000, + "Ddsv4_Type 1": 3.978000, + "Ddsv4_Type2": 4.723000, + "Ddsv5_Type1": 5.966000, + "Dsv3_Type3": 4.225000, + "Dsv3_Type4": 5.281000, + "Dsv4_Type1": 4.224000, + "Dsv4_Type2": 5.069000, + "Dsv5_Type1": 5.280000, + "Eadsv5_Type1": 6.917000, + "Easv4_Type1": 6.653000, + "Easv4_Type2": 6.653000, + "Easv5_Type1": 5.966000, + "Ebdsv5-Type1": 5.878000, + "Ebsv5-Type1": 5.245000, + "Edsv4_Type 1": 5.069000, + "Edsv4_Type2": 6.019000, + "Edsv5_Type1": 7.603000, + "Esv3_Type3": 4.297000, + "Esv3_Type4": 5.822000, + "Esv4_Type1": 4.435000, + "Esv4_Type2": 5.821000, + "Esv5_Type1": 6.653000, + "Fsv2 Type3": 3.927000, + "Fsv2_Type2": 3.366000, + "Fsv2_Type4": 4.488000, + "Lsv3_Type1": 7.678000, + "Mdmsv2MedMem _Type1": 29.359000, + "Mdsv2MedMem_Type1": 14.674000, + "Ms_Type1": 14.669000, + "Msm_Type1": 29.363000, + "Msmv2MedMem Type1": 28.896000, + "Msmv2_Type1": 109.060000, + "Msv2MedMem Type1": 14.211000, + "Msv2_Type1": 54.538000, + "NVasv4_Type1": 8.612000, + "Standard_A0": 0.020000, + "Standard_A1": 0.060000, + "Standard_A1_v2": 0.043000, + "Standard_A2": 0.120000, + "Standard_A2_v2": 0.091000, + "Standard_A2m_v2": 0.129000, + "Standard_A3": 0.240000, + "Standard_A4": 0.480000, + "Standard_A4_v2": 0.191000, + "Standard_A4m_v2": 0.270000, + "Standard_A5": 0.250000, + "Standard_A6": 0.500000, + "Standard_A7": 1.000000, + "Standard_A8_v2": 0.400000, + "Standard_A8m_v2": 0.568000, + "Standard_B12ms": 0.499000, + "Standard_B16als_v2": 0.606000, + "Standard_B16as_v2": 0.602000, + "Standard_B16ls_v2": 0.589000, + "Standard_B16ms": 0.666000, + "Standard_B16pls_v2": 0.550000, + "Standard_B16ps_v2": 0.611000, + "Standard_B16s_v2": 0.666000, + "Standard_B1ls": 0.005200, + "Standard_B1ms": 0.020800, + "Standard_B1s": 0.010400, + "Standard_B20ms": 0.832000, + "Standard_B2als_v2": 0.037600, + "Standard_B2as_v2": 0.075200, + "Standard_B2ats_v2": 0.018600, + "Standard_B2ls_v2": 0.041600, + "Standard_B2ms": 0.083200, + "Standard_B2pls_v2": 0.033600, + "Standard_B2ps_v2": 0.067200, + "Standard_B2pts_v2": 0.008400, + "Standard_B2s": 0.041600, + "Standard_B2s_v2": 0.092400, + "Standard_B2ts_v2": 0.019600, + "Standard_B32als_v2": 1.065000, + "Standard_B32as_v2": 1.350000, + "Standard_B32ls_v2": 1.179000, + "Standard_B32s_v2": 1.478000, + "Standard_B4als_v2": 0.152000, + "Standard_B4as_v2": 0.169000, + "Standard_B4ls_v2": 0.166000, + "Standard_B4ms": 0.166000, + "Standard_B4pls_v2": 0.137000, + "Standard_B4ps_v2": 0.134000, + "Standard_B4s_v2": 0.166000, + "Standard_B8als_v2": 0.303000, + "Standard_B8as_v2": 0.301000, + "Standard_B8ls_v2": 0.295000, + "Standard_B8ms": 0.333000, + "Standard_B8pls_v2": 0.238000, + "Standard_B8ps_v2": 0.269000, + "Standard_B8s_v2": 0.370000, + "Standard_D1": 0.077000, + "Standard_D11": 0.193000, + "Standard_D11_v2": 0.185000, + "Standard_D11_v2_Promo": 0.185000, + "Standard_D12": 0.386000, + "Standard_D12_v2": 0.370000, + "Standard_D12_v2_Promo": 0.370000, + "Standard_D13": 0.771000, + "Standard_D13_v2": 0.741000, + "Standard_D13_v2_Promo": 0.741000, + "Standard_D14": 1.542000, + "Standard_D14_v2": 1.482000, + "Standard_D14_v2_Promo": 1.482000, + "Standard_D15_v2": 1.852000, + "Standard_D15i_v2": 1.852000, + "Standard_D16_v3": 0.800000, + "Standard_D16_v4": 0.768000, + "Standard_D16_v5": 0.768000, + "Standard_D16a_v4": 0.768000, + "Standard_D16ads_v5": 0.824000, + "Standard_D16as_v4": 0.768000, + "Standard_D16as_v5": 0.688000, + "Standard_D16d_v4": 0.904000, + "Standard_D16d_v5": 0.904000, + "Standard_D16ds_v4": 0.904000, + "Standard_D16ds_v5": 0.904000, + "Standard_D16lds_v5": 0.768000, + "Standard_D16ls_v5": 0.680000, + "Standard_D16pds_v5": 0.723000, + "Standard_D16plds_v5": 0.614000, + "Standard_D16pls_v5": 0.544000, + "Standard_D16ps_v5": 0.616000, + "Standard_D16s_v3": 0.800000, + "Standard_D16s_v4": 0.768000, + "Standard_D16s_v5": 0.768000, + "Standard_D1_v2": 0.073000, + "Standard_D2": 0.154000, + "Standard_D2_v2": 0.146000, + "Standard_D2_v2_Promo": 0.146000, + "Standard_D2_v3": 0.100000, + "Standard_D2_v4": 0.096000, + "Standard_D2_v5": 0.096000, + "Standard_D2a_v4": 0.096000, + "Standard_D2ads_v5": 0.103000, + "Standard_D2as_v4": 0.096000, + "Standard_D2as_v5": 0.086000, + "Standard_D2d_v4": 0.113000, + "Standard_D2d_v5": 0.113000, + "Standard_D2ds_v4": 0.113000, + "Standard_D2ds_v5": 0.113000, + "Standard_D2lds_v5": 0.096000, + "Standard_D2ls_v5": 0.085000, + "Standard_D2pds_v5": 0.090400, + "Standard_D2plds_v5": 0.076800, + "Standard_D2pls_v5": 0.068000, + "Standard_D2ps_v5": 0.077000, + "Standard_D2s_v3": 0.100000, + "Standard_D2s_v4": 0.096000, + "Standard_D2s_v5": 0.096000, + "Standard_D3": 0.308000, + "Standard_D32-16s_v3": 1.600000, + "Standard_D32-8s_v3": 1.600000, + "Standard_D32_v3": 1.600000, + "Standard_D32_v4": 1.536000, + "Standard_D32_v5": 1.536000, + "Standard_D32a_v4": 1.536000, + "Standard_D32ads_v5": 1.648000, + "Standard_D32as_v4": 1.536000, + "Standard_D32as_v5": 2.848000, + "Standard_D32d_v4": 1.808000, + "Standard_D32d_v5": 1.808000, + "Standard_D32ds_v4": 1.808000, + "Standard_D32ds_v5": 1.808000, + "Standard_D32lds_v5": 1.536000, + "Standard_D32ls_v5": 1.360000, + "Standard_D32pds_v5": 1.446000, + "Standard_D32plds_v5": 1.229000, + "Standard_D32pls_v5": 1.088000, + "Standard_D32ps_v5": 1.232000, + "Standard_D32s_v3": 1.600000, + "Standard_D32s_v4": 1.536000, + "Standard_D32s_v5": 1.536000, + "Standard_D3_v2": 0.293000, + "Standard_D3_v2_Promo": 0.293000, + "Standard_D4": 0.616000, + "Standard_D48_v3": 2.400000, + "Standard_D48_v4": 2.304000, + "Standard_D48_v5": 2.304000, + "Standard_D48a_v4": 2.304000, + "Standard_D48ads_v5": 4.680000, + "Standard_D48as_v4": 2.304000, + "Standard_D48as_v5": 2.064000, + "Standard_D48d_v4": 2.712000, + "Standard_D48d_v5": 2.712000, + "Standard_D48ds_v4": 2.712000, + "Standard_D48ds_v5": 2.712000, + "Standard_D48lds_v5": 2.304000, + "Standard_D48ls_v5": 2.040000, + "Standard_D48pds_v5": 2.170000, + "Standard_D48plds_v5": 1.843000, + "Standard_D48pls_v5": 1.632000, + "Standard_D48ps_v5": 1.848000, + "Standard_D48s_v3": 2.400000, + "Standard_D48s_v4": 2.304000, + "Standard_D48s_v5": 2.304000, + "Standard_D4_v2": 0.585000, + "Standard_D4_v2_Promo": 0.585000, + "Standard_D4_v3": 0.200000, + "Standard_D4_v4": 0.192000, + "Standard_D4_v5": 0.192000, + "Standard_D4a_v4": 0.192000, + "Standard_D4ads_v5": 0.206000, + "Standard_D4as_v4": 0.192000, + "Standard_D4as_v5": 0.172000, + "Standard_D4d_v4": 0.226000, + "Standard_D4d_v5": 0.226000, + "Standard_D4ds_v4": 0.226000, + "Standard_D4ds_v5": 0.226000, + "Standard_D4lds_v5": 0.192000, + "Standard_D4ls_v5": 0.170000, + "Standard_D4pds_v5": 0.181000, + "Standard_D4plds_v5": 0.154000, + "Standard_D4pls_v5": 0.136000, + "Standard_D4ps_v5": 0.154000, + "Standard_D4s_v3": 0.200000, + "Standard_D4s_v4": 0.192000, + "Standard_D4s_v5": 0.192000, + "Standard_D5_v2": 1.170000, + "Standard_D5_v2_Promo": 1.170000, + "Standard_D64-16s_v3": 3.200000, + "Standard_D64-32s_v3": 3.200000, + "Standard_D64_v3": 3.200000, + "Standard_D64_v4": 3.072000, + "Standard_D64_v5": 3.072000, + "Standard_D64a_v4": 3.072000, + "Standard_D64ads_v5": 6.240000, + "Standard_D64as_v4": 3.072000, + "Standard_D64as_v5": 2.752000, + "Standard_D64d_v4": 3.616000, + "Standard_D64d_v5": 3.616000, + "Standard_D64ds_v4": 3.616000, + "Standard_D64ds_v5": 3.616000, + "Standard_D64lds_v5": 3.072000, + "Standard_D64ls_v5": 2.720000, + "Standard_D64pds_v5": 2.893000, + "Standard_D64plds_v5": 2.458000, + "Standard_D64pls_v5": 2.176000, + "Standard_D64ps_v5": 2.464000, + "Standard_D64s_v3": 3.200000, + "Standard_D64s_v4": 3.072000, + "Standard_D64s_v5": 3.072000, + "Standard_D8_v3": 0.400000, + "Standard_D8_v4": 0.384000, + "Standard_D8_v5": 0.384000, + "Standard_D8a_v4": 0.384000, + "Standard_D8ads_v5": 0.780000, + "Standard_D8as_v4": 0.384000, + "Standard_D8as_v5": 0.712000, + "Standard_D8d_v4": 0.452000, + "Standard_D8d_v5": 0.452000, + "Standard_D8ds_v4": 0.452000, + "Standard_D8ds_v5": 0.452000, + "Standard_D8lds_v5": 0.384000, + "Standard_D8ls_v5": 0.340000, + "Standard_D8pds_v5": 0.362000, + "Standard_D8plds_v5": 0.307000, + "Standard_D8pls_v5": 0.272000, + "Standard_D8ps_v5": 0.308000, + "Standard_D8s_v3": 0.400000, + "Standard_D8s_v4": 0.384000, + "Standard_D8s_v5": 0.384000, + "Standard_D96_v5": 4.608000, + "Standard_D96a_v4": 4.608000, + "Standard_D96ads_v5": 4.944000, + "Standard_D96as_v4": 4.608000, + "Standard_D96as_v5": 8.544000, + "Standard_D96d_v5": 5.424000, + "Standard_D96ds_v5": 5.424000, + "Standard_D96lds_v5": 4.608000, + "Standard_D96ls_v5": 4.080000, + "Standard_D96s_v5": 4.608000, + "Standard_DC1s_v2": 0.096000, + "Standard_DC2s_v2": 0.192000, + "Standard_DC4s_v2": 0.384000, + "Standard_DC8_v2": 0.768000, + "Standard_DS1": 0.077000, + "Standard_DS11": 0.193000, + "Standard_DS11-1_v2": 0.185000, + "Standard_DS11_v2": 0.185000, + "Standard_DS11_v2_Promo": 0.185000, + "Standard_DS12": 0.386000, + "Standard_DS12-1_v2": 0.370000, + "Standard_DS12-2_v2": 0.370000, + "Standard_DS12_v2": 0.370000, + "Standard_DS12_v2_Promo": 0.370000, + "Standard_DS13": 0.771000, + "Standard_DS13-2_v2": 0.741000, + "Standard_DS13-4_v2": 0.741000, + "Standard_DS13_v2": 0.741000, + "Standard_DS13_v2_Promo": 0.741000, + "Standard_DS14": 1.542000, + "Standard_DS14-4_v2": 1.482000, + "Standard_DS14-8_v2": 1.482000, + "Standard_DS14_v2": 1.482000, + "Standard_DS14_v2_Promo": 1.482000, + "Standard_DS15_v2": 1.852000, + "Standard_DS15i_v2": 1.852000, + "Standard_DS1_v2": 0.073000, + "Standard_DS2": 0.154000, + "Standard_DS2_v2": 0.146000, + "Standard_DS2_v2_Promo": 0.146000, + "Standard_DS3": 0.308000, + "Standard_DS3_v2": 0.293000, + "Standard_DS3_v2_Promo": 0.293000, + "Standard_DS4": 0.616000, + "Standard_DS4_v2": 0.585000, + "Standard_DS4_v2_Promo": 0.585000, + "Standard_DS5_v2": 1.170000, + "Standard_DS5_v2_Promo": 1.170000, + "Standard_E104i_v5": 7.207000, + "Standard_E104id_v5": 8.237000, + "Standard_E104ids_v5": 8.237000, + "Standard_E104is_v5": 7.207000, + "Standard_E112iads_v5": 8.070000, + "Standard_E112ias_v5": 6.961000, + "Standard_E112ibds_v5": 10.287000, + "Standard_E112ibs_v5": 9.178000, + "Standard_E16-4ads_v5": 1.048000, + "Standard_E16-4as_v4": 1.008000, + "Standard_E16-4as_v5": 0.904000, + "Standard_E16-4ds_v4": 1.152000, + "Standard_E16-4ds_v5": 1.152000, + "Standard_E16-4s_v3": 1.008000, + "Standard_E16-4s_v4": 1.008000, + "Standard_E16-4s_v5": 1.008000, + "Standard_E16-8ads_v5": 1.784000, + "Standard_E16-8as_v4": 1.008000, + "Standard_E16-8as_v5": 0.904000, + "Standard_E16-8ds_v4": 1.152000, + "Standard_E16-8ds_v5": 1.152000, + "Standard_E16-8s_v3": 1.008000, + "Standard_E16-8s_v4": 1.008000, + "Standard_E16-8s_v5": 1.008000, + "Standard_E16_v3": 1.008000, + "Standard_E16_v4": 1.008000, + "Standard_E16_v5": 1.008000, + "Standard_E16a_v4": 1.008000, + "Standard_E16ads_v5": 1.094000, + "Standard_E16as_v4": 1.008000, + "Standard_E16as_v5": 0.904000, + "Standard_E16bds_v5": 1.336000, + "Standard_E16bs_v5": 1.192000, + "Standard_E16d_v4": 1.152000, + "Standard_E16d_v5": 1.152000, + "Standard_E16ds_v4": 1.152000, + "Standard_E16ds_v5": 1.152000, + "Standard_E16pds_v5": 0.922000, + "Standard_E16ps_v5": 0.806000, + "Standard_E16s_v3": 1.008000, + "Standard_E16s_v4": 1.008000, + "Standard_E16s_v5": 1.008000, + "Standard_E20_v3": 1.260000, + "Standard_E20_v4": 1.260000, + "Standard_E20_v5": 1.260000, + "Standard_E20a_v4": 1.260000, + "Standard_E20ads_v5": 1.310000, + "Standard_E20as_v4": 1.260000, + "Standard_E20as_v5": 1.176000, + "Standard_E20d_v4": 1.440000, + "Standard_E20d_v5": 1.440000, + "Standard_E20ds_v4": 1.440000, + "Standard_E20ds_v5": 1.440000, + "Standard_E20pds_v5": 1.152000, + "Standard_E20ps_v5": 1.008000, + "Standard_E20s_v3": 1.260000, + "Standard_E20s_v4": 1.260000, + "Standard_E20s_v5": 1.260000, + "Standard_E2_v3": 0.126000, + "Standard_E2_v4": 0.126000, + "Standard_E2_v5": 0.126000, + "Standard_E2a_v4": 0.126000, + "Standard_E2ads_v5": 0.131000, + "Standard_E2as_v4": 0.126000, + "Standard_E2as_v5": 0.159000, + "Standard_E2bds_v5": 0.167000, + "Standard_E2bs_v5": 0.149000, + "Standard_E2d_v4": 0.144000, + "Standard_E2d_v5": 0.144000, + "Standard_E2ds_v4": 0.144000, + "Standard_E2ds_v5": 0.144000, + "Standard_E2pds_v5": 0.115000, + "Standard_E2ps_v5": 0.101000, + "Standard_E2s_v3": 0.126000, + "Standard_E2s_v4": 0.126000, + "Standard_E2s_v5": 0.126000, + "Standard_E32-16ads_v5": 3.568000, + "Standard_E32-16as_v4": 2.016000, + "Standard_E32-16as_v5": 3.280000, + "Standard_E32-16ds_v4": 2.304000, + "Standard_E32-16ds_v5": 2.304000, + "Standard_E32-16s_v3": 2.016000, + "Standard_E32-16s_v4": 2.016000, + "Standard_E32-16s_v5": 2.016000, + "Standard_E32-8ads_v5": 3.568000, + "Standard_E32-8as_v4": 2.016000, + "Standard_E32-8as_v5": 1.808000, + "Standard_E32-8ds_v4": 2.304000, + "Standard_E32-8ds_v5": 2.304000, + "Standard_E32-8s_v3": 2.016000, + "Standard_E32-8s_v4": 2.016000, + "Standard_E32-8s_v5": 2.016000, + "Standard_E32_v3": 2.016000, + "Standard_E32_v4": 2.016000, + "Standard_E32_v5": 2.016000, + "Standard_E32a_v4": 2.016000, + "Standard_E32ads_v5": 2.142000, + "Standard_E32as_v4": 2.016000, + "Standard_E32as_v5": 1.808000, + "Standard_E32bds_v5": 2.672000, + "Standard_E32bs_v5": 2.384000, + "Standard_E32d_v4": 2.304000, + "Standard_E32d_v5": 2.304000, + "Standard_E32ds_v4": 2.304000, + "Standard_E32ds_v5": 2.304000, + "Standard_E32pds_v5": 1.843000, + "Standard_E32ps_v5": 1.613000, + "Standard_E32s_v3": 2.016000, + "Standard_E32s_v4": 2.016000, + "Standard_E32s_v5": 2.016000, + "Standard_E4-2ads_v5": 0.446000, + "Standard_E4-2as_v4": 0.252000, + "Standard_E4-2as_v5": 0.226000, + "Standard_E4-2ds_v4": 0.288000, + "Standard_E4-2ds_v5": 0.288000, + "Standard_E4-2s_v3": 0.252000, + "Standard_E4-2s_v4": 0.252000, + "Standard_E4-2s_v5": 0.252000, + "Standard_E48_v3": 3.024000, + "Standard_E48_v4": 3.024000, + "Standard_E48_v5": 3.024000, + "Standard_E48a_v4": 3.024000, + "Standard_E48ads_v5": 3.144000, + "Standard_E48as_v4": 3.024000, + "Standard_E48as_v5": 2.758000, + "Standard_E48bds_v5": 4.008000, + "Standard_E48bs_v5": 3.576000, + "Standard_E48d_v4": 3.456000, + "Standard_E48d_v5": 3.456000, + "Standard_E48ds_v4": 3.456000, + "Standard_E48ds_v5": 3.456000, + "Standard_E48s_v3": 3.024000, + "Standard_E48s_v4": 3.024000, + "Standard_E48s_v5": 3.024000, + "Standard_E4_v3": 0.252000, + "Standard_E4_v4": 0.252000, + "Standard_E4_v5": 0.252000, + "Standard_E4a_v4": 0.252000, + "Standard_E4ads_v5": 0.308000, + "Standard_E4as_v4": 0.252000, + "Standard_E4as_v5": 0.226000, + "Standard_E4bds_v5": 0.334000, + "Standard_E4bs_v5": 0.298000, + "Standard_E4d_v4": 0.288000, + "Standard_E4d_v5": 0.288000, + "Standard_E4ds_v4": 0.288000, + "Standard_E4ds_v5": 0.288000, + "Standard_E4pds_v5": 0.230000, + "Standard_E4ps_v5": 0.202000, + "Standard_E4s_v3": 0.252000, + "Standard_E4s_v4": 0.252000, + "Standard_E4s_v5": 0.252000, + "Standard_E64-16ads_v5": 7.136000, + "Standard_E64-16as_v4": 4.032000, + "Standard_E64-16as_v5": 3.616000, + "Standard_E64-16ds_v4": 4.608000, + "Standard_E64-16ds_v5": 4.608000, + "Standard_E64-16s_v3": 3.629000, + "Standard_E64-16s_v4": 4.032000, + "Standard_E64-16s_v5": 4.032000, + "Standard_E64-32ads_v5": 7.136000, + "Standard_E64-32as_v4": 4.032000, + "Standard_E64-32as_v5": 6.560000, + "Standard_E64-32ds_v4": 4.608000, + "Standard_E64-32ds_v5": 4.608000, + "Standard_E64-32s_v3": 3.629000, + "Standard_E64-32s_v4": 4.032000, + "Standard_E64-32s_v5": 4.032000, + "Standard_E64_v3": 3.629000, + "Standard_E64_v4": 4.032000, + "Standard_E64_v5": 4.032000, + "Standard_E64a_v4": 4.032000, + "Standard_E64ads_v5": 4.192000, + "Standard_E64as_v4": 4.032000, + "Standard_E64as_v5": 3.662000, + "Standard_E64bds_v5": 5.344000, + "Standard_E64bs_v5": 4.768000, + "Standard_E64d_v4": 4.608000, + "Standard_E64d_v5": 4.608000, + "Standard_E64ds_v4": 4.608000, + "Standard_E64ds_v5": 4.608000, + "Standard_E64i_v3": 3.629000, + "Standard_E64is_v3": 3.629000, + "Standard_E64s_v3": 3.629000, + "Standard_E64s_v4": 4.032000, + "Standard_E64s_v5": 4.032000, + "Standard_E8-2ads_v5": 0.892000, + "Standard_E8-2as_v4": 0.504000, + "Standard_E8-2as_v5": 0.452000, + "Standard_E8-2ds_v4": 0.576000, + "Standard_E8-2ds_v5": 0.576000, + "Standard_E8-2s_v3": 0.504000, + "Standard_E8-2s_v4": 0.504000, + "Standard_E8-2s_v5": 0.504000, + "Standard_E8-4ads_v5": 0.892000, + "Standard_E8-4as_v4": 0.504000, + "Standard_E8-4as_v5": 0.820000, + "Standard_E8-4ds_v4": 0.576000, + "Standard_E8-4ds_v5": 0.576000, + "Standard_E8-4s_v3": 0.504000, + "Standard_E8-4s_v4": 0.504000, + "Standard_E8-4s_v5": 0.504000, + "Standard_E80ids_v4": 5.760000, + "Standard_E80is_v4": 5.040000, + "Standard_E8_v3": 0.504000, + "Standard_E8_v4": 0.504000, + "Standard_E8_v5": 0.504000, + "Standard_E8a_v4": 0.504000, + "Standard_E8ads_v5": 0.524000, + "Standard_E8as_v4": 0.504000, + "Standard_E8as_v5": 0.452000, + "Standard_E8bds_v5": 0.668000, + "Standard_E8bs_v5": 0.596000, + "Standard_E8d_v4": 0.576000, + "Standard_E8d_v5": 0.576000, + "Standard_E8ds_v4": 0.576000, + "Standard_E8ds_v5": 0.576000, + "Standard_E8pds_v5": 0.461000, + "Standard_E8ps_v5": 0.403000, + "Standard_E8s_v3": 0.504000, + "Standard_E8s_v4": 0.504000, + "Standard_E8s_v5": 0.504000, + "Standard_E96-24ads_v5": 10.704000, + "Standard_E96-24as_v4": 6.048000, + "Standard_E96-24as_v5": 5.424000, + "Standard_E96-24ds_v5": 6.912000, + "Standard_E96-24s_v5": 6.048000, + "Standard_E96-48ads_v5": 10.704000, + "Standard_E96-48as_v4": 6.048000, + "Standard_E96-48as_v5": 9.840000, + "Standard_E96-48ds_v5": 6.912000, + "Standard_E96-48s_v5": 6.048000, + "Standard_E96_v5": 6.048000, + "Standard_E96a_v4": 6.048000, + "Standard_E96ads_v5": 6.334000, + "Standard_E96as_v4": 6.048000, + "Standard_E96as_v5": 5.470000, + "Standard_E96bds_v5": 8.016000, + "Standard_E96bs_v5": 7.152000, + "Standard_E96d_v5": 6.912000, + "Standard_E96ds_v5": 6.912000, + "Standard_E96iads_v5": 6.917000, + "Standard_E96ias_v5": 5.966000, + "Standard_E96s_v5": 6.048000, + "Standard_F1": 0.049700, + "Standard_F16": 0.796000, + "Standard_F16s": 0.796000, + "Standard_F16s_v2": 0.680000, + "Standard_F1s": 0.049700, + "Standard_F2": 0.099000, + "Standard_F2s": 0.099000, + "Standard_F2s_v2": 0.085000, + "Standard_F32s_v2": 1.360000, + "Standard_F4": 0.199000, + "Standard_F48s_v2": 2.040000, + "Standard_F4s": 0.199000, + "Standard_F4s_v2": 0.170000, + "Standard_F64s_v2": 2.720000, + "Standard_F72s_v2": 3.060000, + "Standard_F8": 0.398000, + "Standard_F8s": 0.398000, + "Standard_F8s_v2": 0.340000, + "Standard_H16": 1.808000, + "Standard_H16m": 2.423000, + "Standard_H16mr": 2.665000, + "Standard_H16r": 1.989000, + "Standard_H8": 0.904000, + "Standard_H8m": 1.211000, + "Standard_HB120-16rs_v2": 4.320000, + "Standard_HB120-32rs_v2": 4.320000, + "Standard_HB120-64rs_v2": 4.320000, + "Standard_HB120-96rs_v2": 4.320000, + "Standard_HB120rs_v2": 4.320000, + "Standard_L112ias_v3": 9.610000, + "Standard_L16as_v3": 1.248000, + "Standard_L16s_v3": 1.396000, + "Standard_L32as_v3": 2.496000, + "Standard_L32s_v3": 2.792000, + "Standard_L48as_v3": 3.744000, + "Standard_L48s_v3": 4.188000, + "Standard_L64as_v3": 4.992000, + "Standard_L64s_v3": 5.584000, + "Standard_L80as_v3": 6.240000, + "Standard_L80s_v3": 6.980000, + "Standard_L8as_v3": 0.624000, + "Standard_L8s_v3": 0.698000, + "Standard_L96ias_v3": 8.237000, + "Standard_M128": 13.338000, + "Standard_M128-32ms": 26.688000, + "Standard_M128-64ms": 26.688000, + "Standard_M128dms_v2": 26.690000, + "Standard_M128ds_v2": 13.340000, + "Standard_M128m": 26.688000, + "Standard_M128ms": 26.688000, + "Standard_M128ms_v2": 26.269000, + "Standard_M128s": 13.338000, + "Standard_M128s_v2": 12.919000, + "Standard_M16-4ms": 3.073000, + "Standard_M16-8ms": 3.073000, + "Standard_M16ms": 3.073000, + "Standard_M16s": 2.387000, + "Standard_M192idms_v2": 32.064000, + "Standard_M192ids_v2": 16.032000, + "Standard_M192ims_v2": 31.643000, + "Standard_M192is_v2": 15.611000, + "Standard_M208-104ms_v2": 44.620000, + "Standard_M208-104s_v2": 22.310000, + "Standard_M208-52ms_v2": 44.620000, + "Standard_M208-52s_v2": 22.310000, + "Standard_M208ms_v2": 44.620000, + "Standard_M208s_v2": 22.310000, + "Standard_M32-16ms": 6.146000, + "Standard_M32-8ms": 6.146000, + "Standard_M32dms_v2": 6.146000, + "Standard_M32ls": 2.873000, + "Standard_M32ms": 6.146000, + "Standard_M32ms_v2": 6.041000, + "Standard_M32s": 3.335000, + "Standard_M32ts": 2.707000, + "Standard_M416-104ms_v2": 99.150000, + "Standard_M416-104s_v2": 49.580000, + "Standard_M416-208ms_v2": 99.150000, + "Standard_M416-208s_v2": 49.580000, + "Standard_M416is_v2": 49.580000, + "Standard_M416ms_v2": 99.150000, + "Standard_M416s_8_v2": 66.110000, + "Standard_M416s_v2": 49.580000, + "Standard_M64": 6.669000, + "Standard_M64-16ms": 10.337000, + "Standard_M64-32ms": 10.337000, + "Standard_M64dms_v2": 10.340000, + "Standard_M64ds_v2": 6.669000, + "Standard_M64ls": 5.415000, + "Standard_M64m": 10.337000, + "Standard_M64ms": 10.337000, + "Standard_M64ms_v2": 10.130000, + "Standard_M64s": 6.669000, + "Standard_M64s_v2": 6.459000, + "Standard_M8-2ms": 1.536500, + "Standard_M8-4ms": 1.536500, + "Standard_M8ms": 1.536500, + "Standard_NC12": 1.800000, + "Standard_NC16as_T4_v3": 1.204000, + "Standard_NC24": 3.600000, + "Standard_NC24r": 3.960000, + "Standard_NC4as_T4_v3": 0.526000, + "Standard_NC6": 0.900000, + "Standard_NC64as_T4_v3": 4.352000, + "Standard_NC8as_T4_v3": 0.752000, + "Standard_ND96ams_A100_v4": 39.324000, + "Standard_ND96amsr_A100_v4": 39.324000, + "Standard_NV12": 2.280000, + "Standard_NV12ads_A10_v5": 1.090000, + "Standard_NV16as_v4": 1.118000, + "Standard_NV18ads_A10_v5": 1.920000, + "Standard_NV24": 4.560000, + "Standard_NV32as_v4": 2.237000, + "Standard_NV36adms_A10_v5": 5.424000, + "Standard_NV36ads_A10_v5": 3.840000, + "Standard_NV4as_v4": 0.280000, + "Standard_NV6": 1.140000, + "Standard_NV6ads_A10_v5": 0.545000, + "Standard_NV72ads_A10_v5": 7.824000, + "Standard_NV8as_v4": 0.559000, + } + // northeurope + initialOnDemandPrices["northeurope"] = map[string]float64{ + "Basic_A0": 0.018000, + "Basic_A1": 0.025000, + "Basic_A2": 0.075000, + "Basic_A3": 0.188000, + "Basic_A4": 0.376000, + "DCadsv5 Type 1": 7.084000, + "DCasv5 Type 1": 5.914000, + "DCdsv3 Type1": 6.653000, + "DCsv2 Type 1": 0.942000, + "DCsv3 Type1": 5.650000, + "Dadsv5_Type1": 7.084000, + "Dasv4_Type1": 5.650000, + "Dasv4_Type2": 5.885000, + "Dasv5_Type1": 5.914000, + "Ddsv4_Type 1": 4.435000, + "Ddsv4_Type2": 5.267000, + "Ddsv5_Type1": 6.653000, + "Dsv3_Type3": 4.709000, + "Dsv3_Type4": 5.886000, + "Dsv4_Type1": 4.708000, + "Dsv4_Type2": 5.650000, + "Dsv5_Type1": 5.885000, + "ECadsv5 Type 1": 8.480000, + "ECasv5 Type 1": 7.376000, + "Eadsv5_Type1": 7.709000, + "Easv4_Type1": 7.392000, + "Easv4_Type2": 7.445000, + "Easv5_Type1": 6.706000, + "Ebdsv5-Type1": 6.582000, + "Ebsv5-Type1": 5.914000, + "Edsv4_Type 1": 5.632000, + "Edsv4_Type2": 6.688000, + "Edsv5_Type1": 8.448000, + "Esv3_Type3": 4.774000, + "Esv3_Type4": 6.469000, + "Esv4_Type1": 4.963000, + "Esv4_Type2": 6.514000, + "Esv5_Type1": 7.445000, + "Fsv2 Type3": 4.435000, + "Fsv2_Type2": 3.802000, + "Fsv2_Type4": 5.069000, + "Lsv2_Type1": 7.568000, + "Lsv3_Type1": 8.448000, + "Mdmsv2MedMem _Type1": 35.231000, + "Mdsv2MedMem_Type1": 17.609000, + "Ms_Type1": 17.603000, + "Msm_Type1": 35.236000, + "Msmv2MedMem Type1": 34.675000, + "Msmv2_Type1": 130.880000, + "Msv2MedMem Type1": 17.053000, + "Msv2_Type1": 65.446000, + "NVasv4_Type1": 8.612000, + "NVsv3_Type1": 5.317000, + "Standard_A0": 0.020000, + "Standard_A1": 0.060000, + "Standard_A1_v2": 0.041000, + "Standard_A2": 0.120000, + "Standard_A2_v2": 0.087000, + "Standard_A2m_v2": 0.139000, + "Standard_A3": 0.240000, + "Standard_A4": 0.480000, + "Standard_A4_v2": 0.183000, + "Standard_A4m_v2": 0.291000, + "Standard_A5": 0.248000, + "Standard_A6": 0.496000, + "Standard_A7": 0.992000, + "Standard_A8_v2": 0.383000, + "Standard_A8m_v2": 0.611000, + "Standard_B12ms": 0.546000, + "Standard_B16als_v2": 0.578000, + "Standard_B16as_v2": 0.726000, + "Standard_B16ls_v2": 0.646000, + "Standard_B16ms": 0.728000, + "Standard_B16pls_v2": 0.595000, + "Standard_B16ps_v2": 0.589000, + "Standard_B16s_v2": 0.730000, + "Standard_B1ls": 0.005700, + "Standard_B1ms": 0.022700, + "Standard_B1s": 0.011300, + "Standard_B20ms": 0.909000, + "Standard_B2als_v2": 0.050000, + "Standard_B2as_v2": 0.090800, + "Standard_B2ats_v2": 0.019400, + "Standard_B2ls_v2": 0.045600, + "Standard_B2ms": 0.091000, + "Standard_B2pls_v2": 0.046000, + "Standard_B2ps_v2": 0.082800, + "Standard_B2pts_v2": 0.009200, + "Standard_B2s": 0.045000, + "Standard_B2s_v2": 0.091200, + "Standard_B2ts_v2": 0.020600, + "Standard_B32als_v2": 1.156000, + "Standard_B32as_v2": 1.453000, + "Standard_B32ls_v2": 1.292000, + "Standard_B32s_v2": 1.459000, + "Standard_B4als_v2": 0.163000, + "Standard_B4as_v2": 0.163000, + "Standard_B4ls_v2": 0.180000, + "Standard_B4ms": 0.182000, + "Standard_B4pls_v2": 0.149000, + "Standard_B4ps_v2": 0.147000, + "Standard_B4s_v2": 0.201000, + "Standard_B8als_v2": 0.326000, + "Standard_B8as_v2": 0.326000, + "Standard_B8ls_v2": 0.323000, + "Standard_B8ms": 0.364000, + "Standard_B8pls_v2": 0.261000, + "Standard_B8ps_v2": 0.294000, + "Standard_B8s_v2": 0.402000, + "Standard_D1": 0.073000, + "Standard_D11": 0.193000, + "Standard_D11_v2": 0.185000, + "Standard_D11_v2_Promo": 0.185000, + "Standard_D12": 0.386000, + "Standard_D12_v2": 0.371000, + "Standard_D12_v2_Promo": 0.371000, + "Standard_D13": 0.771000, + "Standard_D13_v2": 0.741000, + "Standard_D13_v2_Promo": 0.741000, + "Standard_D14": 1.542000, + "Standard_D14_v2": 1.482000, + "Standard_D14_v2_Promo": 1.482000, + "Standard_D15_v2": 1.853000, + "Standard_D15i_v2": 1.853000, + "Standard_D16_v3": 0.856000, + "Standard_D16_v4": 0.856000, + "Standard_D16_v5": 0.856000, + "Standard_D16a_v4": 0.856000, + "Standard_D16ads_v5": 0.920000, + "Standard_D16as_v4": 0.856000, + "Standard_D16as_v5": 1.504000, + "Standard_D16d_v4": 1.008000, + "Standard_D16d_v5": 1.008000, + "Standard_D16ds_v4": 1.008000, + "Standard_D16ds_v5": 1.008000, + "Standard_D16lds_v5": 0.872000, + "Standard_D16ls_v5": 0.730000, + "Standard_D16pds_v5": 0.806000, + "Standard_D16plds_v5": 0.698000, + "Standard_D16pls_v5": 0.584000, + "Standard_D16ps_v5": 0.688000, + "Standard_D16s_v3": 0.856000, + "Standard_D16s_v4": 0.856000, + "Standard_D16s_v5": 0.856000, + "Standard_D1_v2": 0.065800, + "Standard_D2": 0.146000, + "Standard_D2_v2": 0.132000, + "Standard_D2_v2_Promo": 0.132000, + "Standard_D2_v3": 0.107000, + "Standard_D2_v4": 0.107000, + "Standard_D2_v5": 0.107000, + "Standard_D2a_v4": 0.107000, + "Standard_D2ads_v5": 0.207000, + "Standard_D2as_v4": 0.107000, + "Standard_D2as_v5": 0.188000, + "Standard_D2d_v4": 0.126000, + "Standard_D2d_v5": 0.126000, + "Standard_D2ds_v4": 0.126000, + "Standard_D2ds_v5": 0.126000, + "Standard_D2lds_v5": 0.109000, + "Standard_D2ls_v5": 0.091200, + "Standard_D2pds_v5": 0.101000, + "Standard_D2plds_v5": 0.087200, + "Standard_D2pls_v5": 0.073000, + "Standard_D2ps_v5": 0.086000, + "Standard_D2s_v3": 0.107000, + "Standard_D2s_v4": 0.107000, + "Standard_D2s_v5": 0.107000, + "Standard_D3": 0.292000, + "Standard_D32-16s_v3": 1.712000, + "Standard_D32-8s_v3": 1.712000, + "Standard_D32_v3": 1.712000, + "Standard_D32_v4": 1.712000, + "Standard_D32_v5": 1.712000, + "Standard_D32a_v4": 1.712000, + "Standard_D32ads_v5": 3.312000, + "Standard_D32as_v4": 1.712000, + "Standard_D32as_v5": 1.536000, + "Standard_D32d_v4": 2.016000, + "Standard_D32d_v5": 2.016000, + "Standard_D32ds_v4": 2.016000, + "Standard_D32ds_v5": 2.016000, + "Standard_D32lds_v5": 1.744000, + "Standard_D32ls_v5": 1.459000, + "Standard_D32pds_v5": 1.613000, + "Standard_D32plds_v5": 1.395000, + "Standard_D32pls_v5": 1.167000, + "Standard_D32ps_v5": 1.376000, + "Standard_D32s_v3": 1.712000, + "Standard_D32s_v4": 1.712000, + "Standard_D32s_v5": 1.712000, + "Standard_D3_v2": 0.263000, + "Standard_D3_v2_Promo": 0.263000, + "Standard_D4": 0.584000, + "Standard_D48_v3": 2.568000, + "Standard_D48_v4": 2.568000, + "Standard_D48_v5": 2.568000, + "Standard_D48a_v4": 2.568000, + "Standard_D48ads_v5": 4.968000, + "Standard_D48as_v4": 2.568000, + "Standard_D48as_v5": 2.304000, + "Standard_D48d_v4": 3.024000, + "Standard_D48d_v5": 3.024000, + "Standard_D48ds_v4": 3.024000, + "Standard_D48ds_v5": 3.024000, + "Standard_D48lds_v5": 2.616000, + "Standard_D48ls_v5": 2.189000, + "Standard_D48pds_v5": 2.419000, + "Standard_D48plds_v5": 2.093000, + "Standard_D48pls_v5": 1.751000, + "Standard_D48ps_v5": 2.064000, + "Standard_D48s_v3": 2.568000, + "Standard_D48s_v4": 2.568000, + "Standard_D48s_v5": 2.568000, + "Standard_D4_v2": 0.527000, + "Standard_D4_v2_Promo": 0.527000, + "Standard_D4_v3": 0.214000, + "Standard_D4_v4": 0.214000, + "Standard_D4_v5": 0.214000, + "Standard_D4a_v4": 0.214000, + "Standard_D4ads_v5": 0.414000, + "Standard_D4as_v4": 0.214000, + "Standard_D4as_v5": 0.192000, + "Standard_D4d_v4": 0.252000, + "Standard_D4d_v5": 0.252000, + "Standard_D4ds_v4": 0.252000, + "Standard_D4ds_v5": 0.252000, + "Standard_D4lds_v5": 0.218000, + "Standard_D4ls_v5": 0.182000, + "Standard_D4pds_v5": 0.202000, + "Standard_D4plds_v5": 0.174000, + "Standard_D4pls_v5": 0.146000, + "Standard_D4ps_v5": 0.172000, + "Standard_D4s_v3": 0.214000, + "Standard_D4s_v4": 0.214000, + "Standard_D4s_v5": 0.214000, + "Standard_D5_v2": 1.053000, + "Standard_D5_v2_Promo": 1.053000, + "Standard_D64-16s_v3": 3.424000, + "Standard_D64-32s_v3": 3.424000, + "Standard_D64_v3": 3.424000, + "Standard_D64_v4": 3.424000, + "Standard_D64_v5": 3.424000, + "Standard_D64a_v4": 3.424000, + "Standard_D64ads_v5": 3.680000, + "Standard_D64as_v4": 3.424000, + "Standard_D64as_v5": 6.016000, + "Standard_D64d_v4": 4.032000, + "Standard_D64d_v5": 4.032000, + "Standard_D64ds_v4": 4.032000, + "Standard_D64ds_v5": 4.032000, + "Standard_D64lds_v5": 3.488000, + "Standard_D64ls_v5": 2.918000, + "Standard_D64pds_v5": 3.226000, + "Standard_D64plds_v5": 2.790000, + "Standard_D64pls_v5": 2.334000, + "Standard_D64ps_v5": 2.752000, + "Standard_D64s_v3": 3.424000, + "Standard_D64s_v4": 3.424000, + "Standard_D64s_v5": 3.424000, + "Standard_D8_v3": 0.428000, + "Standard_D8_v4": 0.428000, + "Standard_D8_v5": 0.428000, + "Standard_D8a_v4": 0.428000, + "Standard_D8ads_v5": 0.460000, + "Standard_D8as_v4": 0.428000, + "Standard_D8as_v5": 0.384000, + "Standard_D8d_v4": 0.504000, + "Standard_D8d_v5": 0.504000, + "Standard_D8ds_v4": 0.504000, + "Standard_D8ds_v5": 0.504000, + "Standard_D8lds_v5": 0.436000, + "Standard_D8ls_v5": 0.365000, + "Standard_D8pds_v5": 0.403000, + "Standard_D8plds_v5": 0.349000, + "Standard_D8pls_v5": 0.292000, + "Standard_D8ps_v5": 0.344000, + "Standard_D8s_v3": 0.428000, + "Standard_D8s_v4": 0.428000, + "Standard_D8s_v5": 0.428000, + "Standard_D96_v5": 5.136000, + "Standard_D96a_v4": 5.136000, + "Standard_D96ads_v5": 9.936000, + "Standard_D96as_v4": 5.136000, + "Standard_D96as_v5": 4.608000, + "Standard_D96d_v5": 6.048000, + "Standard_D96ds_v5": 6.048000, + "Standard_D96lds_v5": 5.232000, + "Standard_D96ls_v5": 4.378000, + "Standard_D96s_v5": 5.136000, + "Standard_DC16ads_cc_v5": 0.920000, + "Standard_DC16ads_v5": 0.920000, + "Standard_DC16as_cc_v5": 0.768000, + "Standard_DC16as_v5": 0.768000, + "Standard_DC16ds_v3": 2.016000, + "Standard_DC16eds_v5": 1.008000, + "Standard_DC16es_v5": 0.856000, + "Standard_DC16s_v3": 1.712000, + "Standard_DC1ds_v3": 0.126000, + "Standard_DC1s_v2": 0.107000, + "Standard_DC1s_v3": 0.107000, + "Standard_DC24ds_v3": 3.024000, + "Standard_DC24s_v3": 2.568000, + "Standard_DC2ads_v5": 0.115000, + "Standard_DC2as_v5": 0.096000, + "Standard_DC2ds_v3": 0.252000, + "Standard_DC2eds_v5": 0.126000, + "Standard_DC2es_v5": 0.107000, + "Standard_DC2s_v2": 0.214000, + "Standard_DC2s_v3": 0.214000, + "Standard_DC32ads_cc_v5": 1.840000, + "Standard_DC32ads_v5": 1.840000, + "Standard_DC32as_cc_v5": 1.536000, + "Standard_DC32as_v5": 1.536000, + "Standard_DC32ds_v3": 4.032000, + "Standard_DC32eds_v5": 2.016000, + "Standard_DC32es_v5": 1.712000, + "Standard_DC32s_v3": 3.424000, + "Standard_DC48ads_cc_v5": 2.760000, + "Standard_DC48ads_v5": 2.760000, + "Standard_DC48as_cc_v5": 2.304000, + "Standard_DC48as_v5": 2.304000, + "Standard_DC48ds_v3": 6.048000, + "Standard_DC48eds_v5": 3.024000, + "Standard_DC48es_v5": 2.568000, + "Standard_DC48s_v3": 5.136000, + "Standard_DC4ads_cc_v5": 0.230000, + "Standard_DC4ads_v5": 0.230000, + "Standard_DC4as_cc_v5": 0.192000, + "Standard_DC4as_v5": 0.192000, + "Standard_DC4ds_v3": 0.504000, + "Standard_DC4eds_v5": 0.252000, + "Standard_DC4es_v5": 0.214000, + "Standard_DC4s_v2": 0.428000, + "Standard_DC4s_v3": 0.428000, + "Standard_DC64ads_cc_v5": 3.680000, + "Standard_DC64ads_v5": 3.680000, + "Standard_DC64as_cc_v5": 3.072000, + "Standard_DC64as_v5": 3.072000, + "Standard_DC64eds_v5": 4.032000, + "Standard_DC64es_v5": 3.424000, + "Standard_DC8_v2": 0.856000, + "Standard_DC8ads_cc_v5": 0.460000, + "Standard_DC8ads_v5": 0.460000, + "Standard_DC8as_cc_v5": 0.384000, + "Standard_DC8as_v5": 0.384000, + "Standard_DC8ds_v3": 1.008000, + "Standard_DC8eds_v5": 0.504000, + "Standard_DC8es_v5": 0.428000, + "Standard_DC8s_v3": 0.856000, + "Standard_DC96ads_cc_v5": 5.520000, + "Standard_DC96ads_v5": 5.520000, + "Standard_DC96as_cc_v5": 4.608000, + "Standard_DC96as_v5": 4.608000, + "Standard_DC96eds_v5": 6.048000, + "Standard_DC96es_v5": 5.136000, + "Standard_DS1": 0.073000, + "Standard_DS11": 0.193000, + "Standard_DS11-1_v2": 0.185000, + "Standard_DS11_v2": 0.185000, + "Standard_DS11_v2_Promo": 0.185000, + "Standard_DS12": 0.386000, + "Standard_DS12-1_v2": 0.371000, + "Standard_DS12-2_v2": 0.371000, + "Standard_DS12_v2": 0.371000, + "Standard_DS12_v2_Promo": 0.371000, + "Standard_DS13": 0.771000, + "Standard_DS13-2_v2": 0.741000, + "Standard_DS13-4_v2": 0.741000, + "Standard_DS13_v2": 0.741000, + "Standard_DS13_v2_Promo": 0.741000, + "Standard_DS14": 1.542000, + "Standard_DS14-4_v2": 1.482000, + "Standard_DS14-8_v2": 1.482000, + "Standard_DS14_v2": 1.482000, + "Standard_DS14_v2_Promo": 1.482000, + "Standard_DS15_v2": 1.853000, + "Standard_DS15i_v2": 1.853000, + "Standard_DS1_v2": 0.065800, + "Standard_DS2": 0.146000, + "Standard_DS2_v2": 0.132000, + "Standard_DS2_v2_Promo": 0.132000, + "Standard_DS3": 0.292000, + "Standard_DS3_v2": 0.263000, + "Standard_DS3_v2_Promo": 0.263000, + "Standard_DS4": 0.584000, + "Standard_DS4_v2": 0.527000, + "Standard_DS4_v2_Promo": 0.527000, + "Standard_DS5_v2": 1.053000, + "Standard_DS5_v2_Promo": 1.053000, + "Standard_E104i_v5": 8.065000, + "Standard_E104id_v5": 9.152000, + "Standard_E104ids_v5": 9.152000, + "Standard_E104is_v5": 8.065000, + "Standard_E112iads_v5": 8.994000, + "Standard_E112ias_v5": 7.823000, + "Standard_E112ibds_v5": 11.519000, + "Standard_E112ibs_v5": 10.349000, + "Standard_E16-4ads_v5": 1.168000, + "Standard_E16-4as_v4": 1.128000, + "Standard_E16-4as_v5": 1.752000, + "Standard_E16-4ds_v4": 1.280000, + "Standard_E16-4ds_v5": 1.280000, + "Standard_E16-4s_v3": 1.128000, + "Standard_E16-4s_v4": 1.128000, + "Standard_E16-4s_v5": 1.128000, + "Standard_E16-8ads_v5": 1.904000, + "Standard_E16-8as_v4": 1.128000, + "Standard_E16-8as_v5": 1.752000, + "Standard_E16-8ds_v4": 1.280000, + "Standard_E16-8ds_v5": 1.280000, + "Standard_E16-8s_v3": 1.128000, + "Standard_E16-8s_v4": 1.128000, + "Standard_E16-8s_v5": 1.128000, + "Standard_E16_v3": 1.128000, + "Standard_E16_v4": 1.128000, + "Standard_E16_v5": 1.128000, + "Standard_E16a_v4": 1.128000, + "Standard_E16ads_v5": 1.168000, + "Standard_E16as_v4": 1.128000, + "Standard_E16as_v5": 1.016000, + "Standard_E16bds_v5": 1.496000, + "Standard_E16bs_v5": 1.344000, + "Standard_E16d_v4": 1.280000, + "Standard_E16d_v5": 1.280000, + "Standard_E16ds_v4": 1.280000, + "Standard_E16ds_v5": 1.280000, + "Standard_E16pds_v5": 1.024000, + "Standard_E16ps_v5": 0.902000, + "Standard_E16s_v3": 1.128000, + "Standard_E16s_v4": 1.128000, + "Standard_E16s_v5": 1.128000, + "Standard_E20_v3": 1.410000, + "Standard_E20_v4": 1.410000, + "Standard_E20_v5": 1.410000, + "Standard_E20a_v4": 1.410000, + "Standard_E20ads_v5": 1.506000, + "Standard_E20as_v4": 1.410000, + "Standard_E20as_v5": 1.270000, + "Standard_E20d_v4": 1.600000, + "Standard_E20d_v5": 1.600000, + "Standard_E20ds_v4": 1.600000, + "Standard_E20ds_v5": 1.600000, + "Standard_E20pds_v5": 1.280000, + "Standard_E20ps_v5": 1.128000, + "Standard_E20s_v3": 1.410000, + "Standard_E20s_v4": 1.410000, + "Standard_E20s_v5": 1.410000, + "Standard_E2_v3": 0.141000, + "Standard_E2_v4": 0.141000, + "Standard_E2_v5": 0.141000, + "Standard_E2a_v4": 0.141000, + "Standard_E2ads_v5": 0.192000, + "Standard_E2as_v4": 0.141000, + "Standard_E2as_v5": 0.173000, + "Standard_E2bds_v5": 0.187000, + "Standard_E2bs_v5": 0.168000, + "Standard_E2d_v4": 0.160000, + "Standard_E2d_v5": 0.160000, + "Standard_E2ds_v4": 0.160000, + "Standard_E2ds_v5": 0.160000, + "Standard_E2pds_v5": 0.128000, + "Standard_E2ps_v5": 0.113000, + "Standard_E2s_v3": 0.141000, + "Standard_E2s_v4": 0.141000, + "Standard_E2s_v5": 0.141000, + "Standard_E32-16ads_v5": 2.336000, + "Standard_E32-16as_v4": 2.256000, + "Standard_E32-16as_v5": 2.032000, + "Standard_E32-16ds_v4": 2.560000, + "Standard_E32-16ds_v5": 2.560000, + "Standard_E32-16s_v3": 2.256000, + "Standard_E32-16s_v4": 2.256000, + "Standard_E32-16s_v5": 2.256000, + "Standard_E32-8ads_v5": 3.808000, + "Standard_E32-8as_v4": 2.256000, + "Standard_E32-8as_v5": 3.504000, + "Standard_E32-8ds_v4": 2.560000, + "Standard_E32-8ds_v5": 2.560000, + "Standard_E32-8s_v3": 2.256000, + "Standard_E32-8s_v4": 2.256000, + "Standard_E32-8s_v5": 2.256000, + "Standard_E32_v3": 2.256000, + "Standard_E32_v4": 2.256000, + "Standard_E32_v5": 2.256000, + "Standard_E32a_v4": 2.256000, + "Standard_E32ads_v5": 2.382000, + "Standard_E32as_v4": 2.256000, + "Standard_E32as_v5": 2.078000, + "Standard_E32bds_v5": 2.992000, + "Standard_E32bs_v5": 2.688000, + "Standard_E32d_v4": 2.560000, + "Standard_E32d_v5": 2.560000, + "Standard_E32ds_v4": 2.560000, + "Standard_E32ds_v5": 2.560000, + "Standard_E32pds_v5": 2.048000, + "Standard_E32ps_v5": 1.805000, + "Standard_E32s_v3": 2.256000, + "Standard_E32s_v4": 2.256000, + "Standard_E32s_v5": 2.256000, + "Standard_E4-2ads_v5": 0.476000, + "Standard_E4-2as_v4": 0.282000, + "Standard_E4-2as_v5": 0.254000, + "Standard_E4-2ds_v4": 0.320000, + "Standard_E4-2ds_v5": 0.320000, + "Standard_E4-2s_v3": 0.282000, + "Standard_E4-2s_v4": 0.282000, + "Standard_E4-2s_v5": 0.282000, + "Standard_E48_v3": 3.384000, + "Standard_E48_v4": 3.384000, + "Standard_E48_v5": 3.384000, + "Standard_E48a_v4": 3.384000, + "Standard_E48ads_v5": 3.504000, + "Standard_E48as_v4": 3.384000, + "Standard_E48as_v5": 3.094000, + "Standard_E48bds_v5": 4.488000, + "Standard_E48bs_v5": 4.032000, + "Standard_E48d_v4": 3.840000, + "Standard_E48d_v5": 3.840000, + "Standard_E48ds_v4": 3.840000, + "Standard_E48ds_v5": 3.840000, + "Standard_E48s_v3": 3.384000, + "Standard_E48s_v4": 3.384000, + "Standard_E48s_v5": 3.384000, + "Standard_E4_v3": 0.282000, + "Standard_E4_v4": 0.282000, + "Standard_E4_v5": 0.282000, + "Standard_E4a_v4": 0.282000, + "Standard_E4ads_v5": 0.338000, + "Standard_E4as_v4": 0.282000, + "Standard_E4as_v5": 0.254000, + "Standard_E4bds_v5": 0.374000, + "Standard_E4bs_v5": 0.336000, + "Standard_E4d_v4": 0.320000, + "Standard_E4d_v5": 0.320000, + "Standard_E4ds_v4": 0.320000, + "Standard_E4ds_v5": 0.320000, + "Standard_E4pds_v5": 0.256000, + "Standard_E4ps_v5": 0.226000, + "Standard_E4s_v3": 0.282000, + "Standard_E4s_v4": 0.282000, + "Standard_E4s_v5": 0.282000, + "Standard_E64-16ads_v5": 4.672000, + "Standard_E64-16as_v4": 4.512000, + "Standard_E64-16as_v5": 7.008000, + "Standard_E64-16ds_v4": 5.120000, + "Standard_E64-16ds_v5": 5.120000, + "Standard_E64-16s_v3": 4.061000, + "Standard_E64-16s_v4": 4.512000, + "Standard_E64-16s_v5": 4.512000, + "Standard_E64-32ads_v5": 4.672000, + "Standard_E64-32as_v4": 4.512000, + "Standard_E64-32as_v5": 4.064000, + "Standard_E64-32ds_v4": 5.120000, + "Standard_E64-32ds_v5": 5.120000, + "Standard_E64-32s_v3": 4.061000, + "Standard_E64-32s_v4": 4.512000, + "Standard_E64-32s_v5": 4.512000, + "Standard_E64_v3": 4.061000, + "Standard_E64_v4": 4.512000, + "Standard_E64_v5": 4.512000, + "Standard_E64a_v4": 4.512000, + "Standard_E64ads_v5": 4.672000, + "Standard_E64as_v4": 4.512000, + "Standard_E64as_v5": 4.110000, + "Standard_E64bds_v5": 5.984000, + "Standard_E64bs_v5": 5.376000, + "Standard_E64d_v4": 5.120000, + "Standard_E64d_v5": 5.120000, + "Standard_E64ds_v4": 5.120000, + "Standard_E64ds_v5": 5.120000, + "Standard_E64i_v3": 4.061000, + "Standard_E64is_v3": 4.061000, + "Standard_E64s_v3": 4.061000, + "Standard_E64s_v4": 4.512000, + "Standard_E64s_v5": 4.512000, + "Standard_E8-2ads_v5": 0.952000, + "Standard_E8-2as_v4": 0.564000, + "Standard_E8-2as_v5": 0.876000, + "Standard_E8-2ds_v4": 0.640000, + "Standard_E8-2ds_v5": 0.640000, + "Standard_E8-2s_v3": 0.564000, + "Standard_E8-2s_v4": 0.564000, + "Standard_E8-2s_v5": 0.564000, + "Standard_E8-4ads_v5": 0.584000, + "Standard_E8-4as_v4": 0.564000, + "Standard_E8-4as_v5": 0.508000, + "Standard_E8-4ds_v4": 0.640000, + "Standard_E8-4ds_v5": 0.640000, + "Standard_E8-4s_v3": 0.564000, + "Standard_E8-4s_v4": 0.564000, + "Standard_E8-4s_v5": 0.564000, + "Standard_E80ids_v4": 6.400000, + "Standard_E80is_v4": 5.640000, + "Standard_E8_v3": 0.564000, + "Standard_E8_v4": 0.564000, + "Standard_E8_v5": 0.564000, + "Standard_E8a_v4": 0.564000, + "Standard_E8ads_v5": 0.584000, + "Standard_E8as_v4": 0.564000, + "Standard_E8as_v5": 0.554000, + "Standard_E8bds_v5": 0.748000, + "Standard_E8bs_v5": 0.672000, + "Standard_E8d_v4": 0.640000, + "Standard_E8d_v5": 0.640000, + "Standard_E8ds_v4": 0.640000, + "Standard_E8ds_v5": 0.640000, + "Standard_E8pds_v5": 0.512000, + "Standard_E8ps_v5": 0.451000, + "Standard_E8s_v3": 0.564000, + "Standard_E8s_v4": 0.564000, + "Standard_E8s_v5": 0.564000, + "Standard_E96-24ads_v5": 11.424000, + "Standard_E96-24as_v4": 6.768000, + "Standard_E96-24as_v5": 6.096000, + "Standard_E96-24ds_v5": 7.680000, + "Standard_E96-24s_v5": 6.768000, + "Standard_E96-48ads_v5": 7.008000, + "Standard_E96-48as_v4": 6.768000, + "Standard_E96-48as_v5": 10.512000, + "Standard_E96-48ds_v5": 7.680000, + "Standard_E96-48s_v5": 6.768000, + "Standard_E96_v5": 6.768000, + "Standard_E96a_v4": 6.768000, + "Standard_E96ads_v5": 7.008000, + "Standard_E96as_v4": 6.768000, + "Standard_E96as_v5": 6.096000, + "Standard_E96bds_v5": 8.976000, + "Standard_E96bs_v5": 8.064000, + "Standard_E96d_v5": 7.680000, + "Standard_E96ds_v5": 7.680000, + "Standard_E96iads_v5": 7.709000, + "Standard_E96ias_v5": 6.706000, + "Standard_E96s_v5": 6.768000, + "Standard_EC16ads_cc_v5": 1.168000, + "Standard_EC16ads_v5": 1.168000, + "Standard_EC16as_cc_v5": 1.016000, + "Standard_EC16as_v5": 1.016000, + "Standard_EC16eds_v5": 1.280000, + "Standard_EC16es_v5": 1.128000, + "Standard_EC20ads_cc_v5": 1.460000, + "Standard_EC20ads_v5": 1.460000, + "Standard_EC20as_cc_v5": 1.270000, + "Standard_EC20as_v5": 1.270000, + "Standard_EC2ads_v5": 0.146000, + "Standard_EC2as_v5": 0.127000, + "Standard_EC2eds_v5": 0.160000, + "Standard_EC2es_v5": 0.141000, + "Standard_EC32ads_cc_v5": 2.336000, + "Standard_EC32ads_v5": 2.336000, + "Standard_EC32as_cc_v5": 2.032000, + "Standard_EC32as_v5": 2.032000, + "Standard_EC32eds_v5": 2.560000, + "Standard_EC32es_v5": 2.256000, + "Standard_EC48ads_cc_v5": 3.504000, + "Standard_EC48ads_v5": 3.504000, + "Standard_EC48as_cc_v5": 3.048000, + "Standard_EC48as_v5": 3.048000, + "Standard_EC48eds_v5": 3.840000, + "Standard_EC48es_v5": 3.384000, + "Standard_EC4ads_cc_v5": 0.292000, + "Standard_EC4ads_v5": 0.292000, + "Standard_EC4as_cc_v5": 0.254000, + "Standard_EC4as_v5": 0.254000, + "Standard_EC4eds_v5": 0.320000, + "Standard_EC4es_v5": 0.282000, + "Standard_EC64ads_cc_v5": 4.672000, + "Standard_EC64ads_v5": 4.672000, + "Standard_EC64as_cc_v5": 4.064000, + "Standard_EC64as_v5": 4.064000, + "Standard_EC64eds_v5": 5.120000, + "Standard_EC64es_v5": 4.512000, + "Standard_EC8ads_cc_v5": 0.584000, + "Standard_EC8ads_v5": 0.584000, + "Standard_EC8as_cc_v5": 0.508000, + "Standard_EC8as_v5": 0.508000, + "Standard_EC8eds_v5": 0.640000, + "Standard_EC8es_v5": 0.564000, + "Standard_EC96ads_cc_v5": 7.008000, + "Standard_EC96ads_v5": 7.008000, + "Standard_EC96as_cc_v5": 6.096000, + "Standard_EC96as_v5": 6.096000, + "Standard_EC96eds_v5": 7.680000, + "Standard_EC96es_v5": 6.768000, + "Standard_EC96iads_v5": 7.709000, + "Standard_EC96ias_v5": 6.706000, + "Standard_F1": 0.056600, + "Standard_F16": 0.905000, + "Standard_F16s": 0.905000, + "Standard_F16s_v2": 0.768000, + "Standard_F1s": 0.056600, + "Standard_F2": 0.113000, + "Standard_F2s": 0.113000, + "Standard_F2s_v2": 0.096000, + "Standard_F32s_v2": 1.536000, + "Standard_F4": 0.226000, + "Standard_F48s_v2": 2.304000, + "Standard_F4s": 0.226000, + "Standard_F4s_v2": 0.192000, + "Standard_F64s_v2": 3.072000, + "Standard_F72s_v2": 3.456000, + "Standard_F8": 0.452000, + "Standard_F8s": 0.452000, + "Standard_F8s_v2": 0.384000, + "Standard_FX12mds": 1.248000, + "Standard_FX24mds": 2.496000, + "Standard_FX36mds": 3.744000, + "Standard_FX48mds": 4.992000, + "Standard_FX4mds": 0.416000, + "Standard_H16": 1.941000, + "Standard_H16m": 2.601000, + "Standard_H16mr": 2.861000, + "Standard_H16r": 2.136000, + "Standard_H8": 0.971000, + "Standard_H8m": 1.301000, + "Standard_HC44-16rs": 3.802000, + "Standard_HC44-32rs": 3.802000, + "Standard_HC44rs": 3.802000, + "Standard_L112ias_v3": 10.595000, + "Standard_L16as_v3": 1.376000, + "Standard_L16s_v2": 1.376000, + "Standard_L16s_v3": 1.536000, + "Standard_L32as_v3": 2.752000, + "Standard_L32s_v2": 2.752000, + "Standard_L32s_v3": 3.072000, + "Standard_L48as_v3": 4.128000, + "Standard_L48s_v2": 4.128000, + "Standard_L48s_v3": 4.608000, + "Standard_L64as_v3": 5.504000, + "Standard_L64s_v2": 5.504000, + "Standard_L64s_v3": 6.144000, + "Standard_L80as_v3": 6.880000, + "Standard_L80s_v2": 6.880000, + "Standard_L80s_v3": 7.680000, + "Standard_L88is_v2": 7.568000, + "Standard_L8as_v3": 0.688000, + "Standard_L8s_v2": 0.688000, + "Standard_L8s_v3": 0.768000, + "Standard_L96ias_v3": 9.082000, + "Standard_M128": 16.006000, + "Standard_M128-32ms": 32.025600, + "Standard_M128-64ms": 32.025600, + "Standard_M128dms_v2": 32.028000, + "Standard_M128ds_v2": 16.008000, + "Standard_M128m": 32.025600, + "Standard_M128ms": 32.025600, + "Standard_M128ms_v2": 31.523000, + "Standard_M128s": 16.006000, + "Standard_M128s_v2": 15.503000, + "Standard_M16-4ms": 3.687600, + "Standard_M16-8ms": 3.687600, + "Standard_M16ms": 3.687600, + "Standard_M16s": 2.864000, + "Standard_M192idms_v2": 38.477000, + "Standard_M192ids_v2": 19.238000, + "Standard_M192ims_v2": 37.972000, + "Standard_M192is_v2": 18.733000, + "Standard_M208-104ms_v2": 49.974000, + "Standard_M208-104s_v2": 24.987000, + "Standard_M208-52ms_v2": 49.974000, + "Standard_M208-52s_v2": 24.987000, + "Standard_M208ms_v2": 49.974000, + "Standard_M208s_v2": 24.987000, + "Standard_M32-16ms": 7.375200, + "Standard_M32-8ms": 7.375200, + "Standard_M32dms_v2": 7.375000, + "Standard_M32ls": 3.189000, + "Standard_M32ms": 7.375200, + "Standard_M32ms_v2": 7.249000, + "Standard_M32s": 4.002000, + "Standard_M32ts": 3.004770, + "Standard_M416-104ms_v2": 111.050000, + "Standard_M416-104s_v2": 55.530000, + "Standard_M416-208ms_v2": 111.050000, + "Standard_M416-208s_v2": 55.530000, + "Standard_M416is_v2": 55.530000, + "Standard_M416ms_v2": 111.050000, + "Standard_M416s_8_v2": 73.980000, + "Standard_M416s_v2": 55.530000, + "Standard_M64": 8.003000, + "Standard_M64-16ms": 12.405000, + "Standard_M64-32ms": 12.405000, + "Standard_M64dms_v2": 12.408000, + "Standard_M64ds_v2": 8.003000, + "Standard_M64ls": 6.010650, + "Standard_M64m": 12.405000, + "Standard_M64ms": 12.405000, + "Standard_M64ms_v2": 12.156000, + "Standard_M64s": 8.003000, + "Standard_M64s_v2": 7.750000, + "Standard_M8-2ms": 1.843800, + "Standard_M8-4ms": 1.843800, + "Standard_M8ms": 1.843800, + "Standard_NC12": 1.944000, + "Standard_NC12s_v3": 6.610000, + "Standard_NC16as_T4_v3": 1.342000, + "Standard_NC24": 3.888000, + "Standard_NC24ads_A100_v4": 4.408000, + "Standard_NC24r": 4.277000, + "Standard_NC24rs_v3": 14.541000, + "Standard_NC24s_v3": 13.219000, + "Standard_NC48ads_A100_v4": 8.815000, + "Standard_NC4as_T4_v3": 0.587000, + "Standard_NC6": 0.972000, + "Standard_NC64as_T4_v3": 4.853000, + "Standard_NC6s_v3": 3.305000, + "Standard_NC8as_T4_v3": 0.838000, + "Standard_NC96ads_A100_v4": 17.630000, + "Standard_NV12": 2.420000, + "Standard_NV12ads_A10_v5": 1.090000, + "Standard_NV12s_v3": 1.210000, + "Standard_NV16as_v4": 1.118000, + "Standard_NV18ads_A10_v5": 1.920000, + "Standard_NV24": 4.840000, + "Standard_NV24s_v3": 2.420000, + "Standard_NV32as_v4": 2.237000, + "Standard_NV36adms_A10_v5": 5.424000, + "Standard_NV36ads_A10_v5": 3.840000, + "Standard_NV48s_v3": 4.838000, + "Standard_NV4as_v4": 0.280000, + "Standard_NV6": 1.210000, + "Standard_NV6ads_A10_v5": 0.545000, + "Standard_NV72ads_A10_v5": 7.824000, + "Standard_NV8as_v4": 0.559000, + } + // norwayeast + initialOnDemandPrices["norwayeast"] = map[string]float64{ + "Dadsv5_Type1": 9.073000, + "Dasv4_Type1": 7.550000, + "Dasv4_Type2": 7.550000, + "Dasv5_Type1": 7.576000, + "Ddsv5_Type1": 7.899000, + "Dsv3_Type3": 6.042000, + "Dsv3_Type4": 7.552000, + "Dsv5_Type1": 6.958000, + "Eadsv5_Type1": 9.891000, + "Easv4_Type1": 9.514000, + "Easv4_Type2": 9.514000, + "Easv5_Type1": 8.532000, + "Ebdsv5-Type1": 8.406000, + "Ebsv5-Type1": 7.500000, + "Edsv5_Type1": 10.048000, + "Esv3_Type3": 6.145000, + "Esv3_Type4": 8.325000, + "Esv5_Type1": 8.828000, + "Fsv2 Type3": 5.398800, + "Fsv2_Type2": 4.225100, + "Fsv2_Type4": 5.867400, + "Lasv3_Type1": 9.816000, + "Lsv3_Type1": 10.980000, + "Mdmsv2MedMem _Type1": 41.983000, + "Mdsv2MedMem_Type1": 20.984000, + "Ms_Type1": 20.976000, + "Msm_Type1": 41.989000, + "Msmv2MedMem Type1": 41.321000, + "Msv2MedMem Type1": 20.322000, + "NVsv3_Type1": 9.718000, + "Standard_A1_v2": 0.045100, + "Standard_A2_v2": 0.095700, + "Standard_A2m_v2": 0.136000, + "Standard_A4_v2": 0.201000, + "Standard_A4m_v2": 0.286000, + "Standard_A8_v2": 0.421000, + "Standard_A8m_v2": 0.601000, + "Standard_B12ms": 0.634000, + "Standard_B16als_v2": 0.747000, + "Standard_B16as_v2": 0.834000, + "Standard_B16ls_v2": 0.822000, + "Standard_B16ms": 0.845000, + "Standard_B16s_v2": 0.845000, + "Standard_B1ls": 0.006600, + "Standard_B1ms": 0.026400, + "Standard_B1s": 0.013200, + "Standard_B20ms": 1.056000, + "Standard_B2als_v2": 0.047500, + "Standard_B2as_v2": 0.095000, + "Standard_B2ats_v2": 0.021100, + "Standard_B2ls_v2": 0.062000, + "Standard_B2ms": 0.106000, + "Standard_B2s": 0.052800, + "Standard_B2s_v2": 0.106000, + "Standard_B2ts_v2": 0.013200, + "Standard_B32als_v2": 1.494000, + "Standard_B32as_v2": 1.521000, + "Standard_B32ls_v2": 1.643000, + "Standard_B32s_v2": 1.837000, + "Standard_B4als_v2": 0.187000, + "Standard_B4as_v2": 0.208000, + "Standard_B4ls_v2": 0.187000, + "Standard_B4ms": 0.211000, + "Standard_B4s_v2": 0.230000, + "Standard_B8als_v2": 0.374000, + "Standard_B8as_v2": 0.380000, + "Standard_B8ls_v2": 0.411000, + "Standard_B8ms": 0.422000, + "Standard_B8s_v2": 0.459000, + "Standard_D11_v2": 0.209000, + "Standard_D12_v2": 0.417000, + "Standard_D13_v2": 0.835000, + "Standard_D14_v2": 1.670000, + "Standard_D15_v2": 2.087000, + "Standard_D15i_v2": 2.087000, + "Standard_D16_v3": 1.056000, + "Standard_D16_v4": 1.012000, + "Standard_D16_v5": 1.012000, + "Standard_D16a_v4": 1.098000, + "Standard_D16ads_v5": 1.914000, + "Standard_D16as_v4": 1.098000, + "Standard_D16as_v5": 1.720000, + "Standard_D16d_v4": 1.197000, + "Standard_D16d_v5": 1.197000, + "Standard_D16ds_v4": 1.197000, + "Standard_D16ds_v5": 1.197000, + "Standard_D16lds_v5": 1.098000, + "Standard_D16ls_v5": 0.972000, + "Standard_D16s_v3": 1.056000, + "Standard_D16s_v4": 1.012000, + "Standard_D16s_v5": 1.012000, + "Standard_D1_v2": 0.074700, + "Standard_D2_v2": 0.150000, + "Standard_D2_v3": 0.132000, + "Standard_D2_v4": 0.126000, + "Standard_D2_v5": 0.127000, + "Standard_D2a_v4": 0.137000, + "Standard_D2ads_v5": 0.239000, + "Standard_D2as_v4": 0.137000, + "Standard_D2as_v5": 0.215000, + "Standard_D2d_v4": 0.150000, + "Standard_D2d_v5": 0.150000, + "Standard_D2ds_v4": 0.150000, + "Standard_D2ds_v5": 0.150000, + "Standard_D2lds_v5": 0.137000, + "Standard_D2ls_v5": 0.122000, + "Standard_D2s_v3": 0.132000, + "Standard_D2s_v4": 0.126000, + "Standard_D2s_v5": 0.127000, + "Standard_D32-16s_v3": 2.112000, + "Standard_D32-8s_v3": 2.112000, + "Standard_D32_v3": 2.112000, + "Standard_D32_v4": 2.024000, + "Standard_D32_v5": 2.024000, + "Standard_D32a_v4": 2.196000, + "Standard_D32ads_v5": 3.829000, + "Standard_D32as_v4": 2.196000, + "Standard_D32as_v5": 3.440000, + "Standard_D32d_v4": 2.394000, + "Standard_D32d_v5": 2.394000, + "Standard_D32ds_v4": 2.394000, + "Standard_D32ds_v5": 2.394000, + "Standard_D32lds_v5": 2.196000, + "Standard_D32ls_v5": 1.945000, + "Standard_D32s_v3": 2.112000, + "Standard_D32s_v4": 2.024000, + "Standard_D32s_v5": 2.024000, + "Standard_D3_v2": 0.299000, + "Standard_D48_v3": 3.168000, + "Standard_D48_v4": 3.036000, + "Standard_D48_v5": 3.036000, + "Standard_D48a_v4": 3.295000, + "Standard_D48ads_v5": 3.535000, + "Standard_D48as_v4": 3.295000, + "Standard_D48as_v5": 2.952000, + "Standard_D48d_v4": 3.590000, + "Standard_D48d_v5": 3.590000, + "Standard_D48ds_v4": 3.590000, + "Standard_D48ds_v5": 3.590000, + "Standard_D48lds_v5": 3.295000, + "Standard_D48ls_v5": 2.917000, + "Standard_D48s_v3": 3.168000, + "Standard_D48s_v4": 3.036000, + "Standard_D48s_v5": 3.036000, + "Standard_D4_v2": 0.598000, + "Standard_D4_v3": 0.264000, + "Standard_D4_v4": 0.253000, + "Standard_D4_v5": 0.253000, + "Standard_D4a_v4": 0.275000, + "Standard_D4ads_v5": 0.479000, + "Standard_D4as_v4": 0.275000, + "Standard_D4as_v5": 0.246000, + "Standard_D4d_v4": 0.299000, + "Standard_D4d_v5": 0.299000, + "Standard_D4ds_v4": 0.299000, + "Standard_D4ds_v5": 0.299000, + "Standard_D4lds_v5": 0.275000, + "Standard_D4ls_v5": 0.243000, + "Standard_D4s_v3": 0.264000, + "Standard_D4s_v4": 0.253000, + "Standard_D4s_v5": 0.253000, + "Standard_D5_v2": 1.196000, + "Standard_D64-16s_v3": 4.224000, + "Standard_D64-32s_v3": 4.224000, + "Standard_D64_v3": 4.224000, + "Standard_D64_v4": 4.048000, + "Standard_D64_v5": 4.048000, + "Standard_D64a_v4": 4.393000, + "Standard_D64ads_v5": 4.713000, + "Standard_D64as_v4": 4.393000, + "Standard_D64as_v5": 6.879000, + "Standard_D64d_v4": 4.787000, + "Standard_D64d_v5": 4.787000, + "Standard_D64ds_v4": 4.787000, + "Standard_D64ds_v5": 4.787000, + "Standard_D64lds_v5": 4.393000, + "Standard_D64ls_v5": 3.890000, + "Standard_D64s_v3": 4.224000, + "Standard_D64s_v4": 4.048000, + "Standard_D64s_v5": 4.048000, + "Standard_D8_v3": 0.528000, + "Standard_D8_v4": 0.506000, + "Standard_D8_v5": 0.506000, + "Standard_D8a_v4": 0.549000, + "Standard_D8ads_v5": 0.589000, + "Standard_D8as_v4": 0.549000, + "Standard_D8as_v5": 0.492000, + "Standard_D8d_v4": 0.598000, + "Standard_D8d_v5": 0.598000, + "Standard_D8ds_v4": 0.598000, + "Standard_D8ds_v5": 0.598000, + "Standard_D8lds_v5": 0.549000, + "Standard_D8ls_v5": 0.486000, + "Standard_D8s_v3": 0.528000, + "Standard_D8s_v4": 0.506000, + "Standard_D8s_v5": 0.506000, + "Standard_D96_v5": 6.072000, + "Standard_D96a_v4": 6.589000, + "Standard_D96ads_v5": 7.070000, + "Standard_D96as_v4": 6.589000, + "Standard_D96as_v5": 5.903000, + "Standard_D96d_v5": 7.181000, + "Standard_D96ds_v5": 7.181000, + "Standard_D96lds_v5": 6.589000, + "Standard_D96ls_v5": 5.834000, + "Standard_D96s_v5": 6.072000, + "Standard_DS11-1_v2": 0.209000, + "Standard_DS11_v2": 0.209000, + "Standard_DS12-1_v2": 0.417000, + "Standard_DS12-2_v2": 0.417000, + "Standard_DS12_v2": 0.417000, + "Standard_DS13-2_v2": 0.835000, + "Standard_DS13-4_v2": 0.835000, + "Standard_DS13_v2": 0.835000, + "Standard_DS14-4_v2": 1.670000, + "Standard_DS14-8_v2": 1.670000, + "Standard_DS14_v2": 1.670000, + "Standard_DS15_v2": 2.087000, + "Standard_DS15i_v2": 2.087000, + "Standard_DS1_v2": 0.074700, + "Standard_DS2_v2": 0.150000, + "Standard_DS3_v2": 0.299000, + "Standard_DS4_v2": 0.598000, + "Standard_DS5_v2": 1.196000, + "Standard_E104i_v5": 9.564000, + "Standard_E104id_v5": 10.885000, + "Standard_E104ids_v5": 10.885000, + "Standard_E104is_v5": 9.564000, + "Standard_E112iads_v5": 11.540000, + "Standard_E112ias_v5": 9.954000, + "Standard_E112ibds_v5": 14.711000, + "Standard_E112ibs_v5": 13.125000, + "Standard_E16-4ads_v5": 2.235000, + "Standard_E16-4as_v4": 1.441000, + "Standard_E16-4as_v5": 2.029000, + "Standard_E16-4ds_v4": 1.522000, + "Standard_E16-4ds_v5": 1.522000, + "Standard_E16-4s_v3": 1.408000, + "Standard_E16-4s_v4": 1.338000, + "Standard_E16-4s_v5": 1.338000, + "Standard_E16-8ads_v5": 2.235000, + "Standard_E16-8as_v4": 1.441000, + "Standard_E16-8as_v5": 2.029000, + "Standard_E16-8ds_v4": 1.522000, + "Standard_E16-8ds_v5": 1.522000, + "Standard_E16-8s_v3": 1.408000, + "Standard_E16-8s_v4": 1.338000, + "Standard_E16-8s_v5": 1.338000, + "Standard_E16_v3": 1.408000, + "Standard_E16_v4": 1.338000, + "Standard_E16_v5": 1.338000, + "Standard_E16a_v4": 1.441000, + "Standard_E16ads_v5": 1.499000, + "Standard_E16as_v4": 1.441000, + "Standard_E16as_v5": 1.293000, + "Standard_E16bds_v5": 1.910000, + "Standard_E16bs_v5": 1.705000, + "Standard_E16d_v4": 1.522000, + "Standard_E16d_v5": 1.522000, + "Standard_E16ds_v4": 1.522000, + "Standard_E16ds_v5": 1.522000, + "Standard_E16s_v3": 1.408000, + "Standard_E16s_v4": 1.338000, + "Standard_E16s_v5": 1.338000, + "Standard_E20_v3": 1.760000, + "Standard_E20_v4": 1.672000, + "Standard_E20_v5": 1.672000, + "Standard_E20a_v4": 1.802000, + "Standard_E20ads_v5": 1.873000, + "Standard_E20as_v4": 1.802000, + "Standard_E20as_v5": 1.662000, + "Standard_E20d_v4": 1.903000, + "Standard_E20d_v5": 1.903000, + "Standard_E20ds_v4": 1.903000, + "Standard_E20ds_v5": 1.903000, + "Standard_E20s_v3": 1.760000, + "Standard_E20s_v4": 1.672000, + "Standard_E20s_v5": 1.672000, + "Standard_E2_v3": 0.176000, + "Standard_E2_v4": 0.167000, + "Standard_E2_v5": 0.167000, + "Standard_E2a_v4": 0.180000, + "Standard_E2ads_v5": 0.187000, + "Standard_E2as_v4": 0.180000, + "Standard_E2as_v5": 0.208000, + "Standard_E2bds_v5": 0.239000, + "Standard_E2bs_v5": 0.213000, + "Standard_E2d_v4": 0.190000, + "Standard_E2d_v5": 0.190000, + "Standard_E2ds_v4": 0.190000, + "Standard_E2ds_v5": 0.190000, + "Standard_E2s_v3": 0.176000, + "Standard_E2s_v4": 0.167000, + "Standard_E2s_v5": 0.167000, + "Standard_E32-16ads_v5": 2.997000, + "Standard_E32-16as_v4": 2.883000, + "Standard_E32-16as_v5": 4.057000, + "Standard_E32-16ds_v4": 3.045000, + "Standard_E32-16ds_v5": 3.045000, + "Standard_E32-16s_v3": 2.816000, + "Standard_E32-16s_v4": 2.675000, + "Standard_E32-16s_v5": 2.675000, + "Standard_E32-8ads_v5": 2.997000, + "Standard_E32-8as_v4": 2.883000, + "Standard_E32-8as_v5": 2.585000, + "Standard_E32-8ds_v4": 3.045000, + "Standard_E32-8ds_v5": 3.045000, + "Standard_E32-8s_v3": 2.816000, + "Standard_E32-8s_v4": 2.675000, + "Standard_E32-8s_v5": 2.675000, + "Standard_E32_v3": 2.816000, + "Standard_E32_v4": 2.675000, + "Standard_E32_v5": 2.675000, + "Standard_E32a_v4": 2.883000, + "Standard_E32ads_v5": 2.997000, + "Standard_E32as_v4": 2.883000, + "Standard_E32as_v5": 2.585000, + "Standard_E32bds_v5": 3.821000, + "Standard_E32bs_v5": 3.409000, + "Standard_E32d_v4": 3.045000, + "Standard_E32d_v5": 3.045000, + "Standard_E32ds_v4": 3.045000, + "Standard_E32ds_v5": 3.045000, + "Standard_E32s_v3": 2.816000, + "Standard_E32s_v4": 2.675000, + "Standard_E32s_v5": 2.675000, + "Standard_E4-2ads_v5": 0.375000, + "Standard_E4-2as_v4": 0.360000, + "Standard_E4-2as_v5": 0.323000, + "Standard_E4-2ds_v4": 0.381000, + "Standard_E4-2ds_v5": 0.381000, + "Standard_E4-2s_v3": 0.352000, + "Standard_E4-2s_v4": 0.334000, + "Standard_E4-2s_v5": 0.334000, + "Standard_E48_v3": 4.012000, + "Standard_E48_v4": 4.013000, + "Standard_E48_v5": 4.013000, + "Standard_E48a_v4": 4.324000, + "Standard_E48ads_v5": 4.496000, + "Standard_E48as_v4": 4.324000, + "Standard_E48as_v5": 3.924000, + "Standard_E48bds_v5": 5.731000, + "Standard_E48bs_v5": 5.114000, + "Standard_E48d_v4": 4.567000, + "Standard_E48d_v5": 4.567000, + "Standard_E48ds_v4": 4.567000, + "Standard_E48ds_v5": 4.567000, + "Standard_E48s_v3": 4.012000, + "Standard_E48s_v4": 4.013000, + "Standard_E48s_v5": 4.013000, + "Standard_E4_v3": 0.352000, + "Standard_E4_v4": 0.334000, + "Standard_E4_v5": 0.334000, + "Standard_E4a_v4": 0.360000, + "Standard_E4ads_v5": 0.421000, + "Standard_E4as_v4": 0.360000, + "Standard_E4as_v5": 0.323000, + "Standard_E4bds_v5": 0.478000, + "Standard_E4bs_v5": 0.426000, + "Standard_E4d_v4": 0.381000, + "Standard_E4d_v5": 0.381000, + "Standard_E4ds_v4": 0.381000, + "Standard_E4ds_v5": 0.381000, + "Standard_E4s_v3": 0.352000, + "Standard_E4s_v4": 0.334000, + "Standard_E4s_v5": 0.334000, + "Standard_E64-16ads_v5": 5.995000, + "Standard_E64-16as_v4": 5.766000, + "Standard_E64-16as_v5": 8.115000, + "Standard_E64-16ds_v4": 6.090000, + "Standard_E64-16ds_v5": 6.090000, + "Standard_E64-16s_v3": 4.814000, + "Standard_E64-16s_v4": 5.350000, + "Standard_E64-16s_v5": 5.350000, + "Standard_E64-32ads_v5": 8.939000, + "Standard_E64-32as_v4": 5.766000, + "Standard_E64-32as_v5": 5.171000, + "Standard_E64-32ds_v4": 6.090000, + "Standard_E64-32ds_v5": 6.090000, + "Standard_E64-32s_v3": 4.814000, + "Standard_E64-32s_v4": 5.350000, + "Standard_E64-32s_v5": 5.350000, + "Standard_E64_v3": 4.814000, + "Standard_E64_v4": 5.350000, + "Standard_E64_v5": 5.350000, + "Standard_E64a_v4": 5.766000, + "Standard_E64ads_v5": 5.995000, + "Standard_E64as_v4": 5.766000, + "Standard_E64as_v5": 5.217000, + "Standard_E64bds_v5": 7.642000, + "Standard_E64bs_v5": 6.818000, + "Standard_E64d_v4": 6.090000, + "Standard_E64d_v5": 6.090000, + "Standard_E64ds_v4": 6.090000, + "Standard_E64ds_v5": 6.090000, + "Standard_E64i_v3": 4.814000, + "Standard_E64is_v3": 4.814000, + "Standard_E64s_v3": 4.814000, + "Standard_E64s_v4": 5.350000, + "Standard_E64s_v5": 5.350000, + "Standard_E8-2ads_v5": 1.117000, + "Standard_E8-2as_v4": 0.721000, + "Standard_E8-2as_v5": 1.014000, + "Standard_E8-2ds_v4": 0.761000, + "Standard_E8-2ds_v5": 0.761000, + "Standard_E8-2s_v3": 0.704000, + "Standard_E8-2s_v4": 0.669000, + "Standard_E8-2s_v5": 0.669000, + "Standard_E8-4ads_v5": 1.117000, + "Standard_E8-4as_v4": 0.721000, + "Standard_E8-4as_v5": 0.646000, + "Standard_E8-4ds_v4": 0.761000, + "Standard_E8-4ds_v5": 0.761000, + "Standard_E8-4s_v3": 0.704000, + "Standard_E8-4s_v4": 0.669000, + "Standard_E8-4s_v5": 0.669000, + "Standard_E80ids_v4": 7.612000, + "Standard_E80is_v4": 6.688000, + "Standard_E8_v3": 0.704000, + "Standard_E8_v4": 0.669000, + "Standard_E8_v5": 0.669000, + "Standard_E8a_v4": 0.721000, + "Standard_E8ads_v5": 0.795000, + "Standard_E8as_v4": 0.721000, + "Standard_E8as_v5": 0.692000, + "Standard_E8bds_v5": 0.955000, + "Standard_E8bs_v5": 0.852000, + "Standard_E8d_v4": 0.761000, + "Standard_E8d_v5": 0.761000, + "Standard_E8ds_v4": 0.761000, + "Standard_E8ds_v5": 0.761000, + "Standard_E8s_v3": 0.704000, + "Standard_E8s_v4": 0.669000, + "Standard_E8s_v5": 0.669000, + "Standard_E96-24ads_v5": 8.992000, + "Standard_E96-24as_v4": 8.649000, + "Standard_E96-24as_v5": 12.172000, + "Standard_E96-24ds_v5": 9.134000, + "Standard_E96-24s_v5": 8.026000, + "Standard_E96-48ads_v5": 8.992000, + "Standard_E96-48as_v4": 8.649000, + "Standard_E96-48as_v5": 7.756000, + "Standard_E96-48ds_v5": 9.134000, + "Standard_E96-48s_v5": 8.026000, + "Standard_E96_v5": 8.026000, + "Standard_E96a_v4": 8.649000, + "Standard_E96ads_v5": 8.992000, + "Standard_E96as_v4": 8.649000, + "Standard_E96as_v5": 7.802000, + "Standard_E96bds_v5": 11.463000, + "Standard_E96bs_v5": 10.227000, + "Standard_E96d_v5": 9.134000, + "Standard_E96ds_v5": 9.134000, + "Standard_E96iads_v5": 9.891000, + "Standard_E96ias_v4": 8.649000, + "Standard_E96ias_v5": 8.532000, + "Standard_E96s_v5": 8.026000, + "Standard_F1": 0.062500, + "Standard_F16": 1.000000, + "Standard_F16s": 1.000000, + "Standard_F16s_v2": 0.854000, + "Standard_F1s": 0.062500, + "Standard_F2": 0.125000, + "Standard_F2s": 0.125000, + "Standard_F2s_v2": 0.107000, + "Standard_F32s_v2": 1.709000, + "Standard_F4": 0.250000, + "Standard_F48s_v2": 2.563000, + "Standard_F4s": 0.250000, + "Standard_F4s_v2": 0.214000, + "Standard_F64s_v2": 3.418000, + "Standard_F72s_v2": 3.845000, + "Standard_F8": 0.499000, + "Standard_F8s": 0.499000, + "Standard_F8s_v2": 0.427000, + "Standard_L16as_v3": 1.785000, + "Standard_L16s_v3": 1.996000, + "Standard_L32as_v3": 3.569000, + "Standard_L32s_v3": 3.993000, + "Standard_L48as_v3": 5.354000, + "Standard_L48s_v3": 5.989000, + "Standard_L64as_v3": 7.139000, + "Standard_L64s_v3": 7.985000, + "Standard_L80as_v3": 8.923000, + "Standard_L80s_v3": 9.981000, + "Standard_L8as_v3": 0.892000, + "Standard_L8s_v3": 0.998000, + "Standard_M128": 19.073000, + "Standard_M128-32ms": 38.164000, + "Standard_M128-64ms": 38.164000, + "Standard_M128dms_v2": 38.167000, + "Standard_M128ds_v2": 19.076000, + "Standard_M128m": 38.164000, + "Standard_M128ms": 38.164000, + "Standard_M128ms_v2": 37.565000, + "Standard_M128s": 19.073000, + "Standard_M128s_v2": 18.474000, + "Standard_M16-4ms": 4.394000, + "Standard_M16-8ms": 4.394000, + "Standard_M16ms": 4.394000, + "Standard_M16s": 3.413000, + "Standard_M192idms_v2": 45.851000, + "Standard_M192ids_v2": 22.925000, + "Standard_M192ims_v2": 45.249000, + "Standard_M192is_v2": 22.324000, + "Standard_M32-16ms": 8.789000, + "Standard_M32-8ms": 8.789000, + "Standard_M32dms_v2": 8.789000, + "Standard_M32ls": 4.108000, + "Standard_M32ms": 8.789000, + "Standard_M32ms_v2": 8.638000, + "Standard_M32s": 6.827000, + "Standard_M32ts": 3.871000, + "Standard_M64": 9.537000, + "Standard_M64-16ms": 19.082000, + "Standard_M64-32ms": 19.082000, + "Standard_M64dms_v2": 14.786000, + "Standard_M64ds_v2": 9.537000, + "Standard_M64ls": 8.217000, + "Standard_M64m": 19.082000, + "Standard_M64ms": 19.082000, + "Standard_M64ms_v2": 14.485000, + "Standard_M64s": 9.537000, + "Standard_M64s_v2": 9.236000, + "Standard_M8-2ms": 2.197000, + "Standard_M8-4ms": 2.197000, + "Standard_M8ms": 2.197000, + "Standard_ND96ams_A100_v4": 46.861000, + "Standard_ND96amsr_A100_v4": 46.861000, + "Standard_NV12s_v3": 1.568000, + "Standard_NV24s_v3": 3.135000, + "Standard_NV48s_v3": 6.270000, + } + // norwaywest + initialOnDemandPrices["norwaywest"] = map[string]float64{ + "Dadsv5_Type1": 11.795000, + "Dasv4_Type1": 9.816000, + "Dasv4_Type2": 9.816000, + "Dasv5_Type1": 9.848000, + "Ddsv4_Type 1": 6.846000, + "Ddsv5_Type1": 10.269000, + "Dsv3_Type3": 7.858000, + "Dsv3_Type4": 9.823000, + "Dsv4_Type1": 7.236000, + "Dsv5_Type1": 9.045000, + "Eadsv5_Type1": 12.858000, + "Easv4_Type1": 12.368000, + "Easv4_Type2": 12.368000, + "Easv5_Type1": 11.092000, + "Ebdsv5-Type1": 10.928000, + "Ebsv5-Type1": 9.750000, + "Edsv4_Type 1": 8.708000, + "Edsv5_Type1": 13.062000, + "Esv3_Type3": 7.992000, + "Esv3_Type4": 10.828000, + "Esv4_Type1": 7.651000, + "Esv5_Type1": 11.477000, + "Fsv2 Type3": 8.343000, + "Fsv2_Type2": 8.343000, + "Lasv3_Type1": 12.760000, + "Lsv3_Type1": 14.273000, + "Mdmsv2MedMem _Type1": 54.608000, + "Mdsv2MedMem_Type1": 27.294000, + "Msmv2MedMem Type1": 53.747000, + "Msv2MedMem Type1": 26.433000, + "NVsv3_Type1": 12.634000, + "Standard_A1_v2": 0.058600, + "Standard_A2_v2": 0.124000, + "Standard_A2m_v2": 0.177000, + "Standard_A4_v2": 0.262000, + "Standard_A4m_v2": 0.372000, + "Standard_A8_v2": 0.548000, + "Standard_A8m_v2": 0.781000, + "Standard_B12ms": 0.824000, + "Standard_B16als_v2": 0.949000, + "Standard_B16as_v2": 1.062000, + "Standard_B16ls_v2": 0.973000, + "Standard_B16ms": 1.098000, + "Standard_B16s_v2": 1.172000, + "Standard_B1ls": 0.008580, + "Standard_B1ms": 0.034300, + "Standard_B1s": 0.017200, + "Standard_B20ms": 1.373000, + "Standard_B2als_v2": 0.071000, + "Standard_B2as_v2": 0.133000, + "Standard_B2ats_v2": 0.015400, + "Standard_B2ls_v2": 0.077800, + "Standard_B2ms": 0.137000, + "Standard_B2s": 0.068600, + "Standard_B2s_v2": 0.146000, + "Standard_B2ts_v2": 0.026400, + "Standard_B32als_v2": 1.898000, + "Standard_B32as_v2": 1.977000, + "Standard_B32ls_v2": 2.092000, + "Standard_B32s_v2": 2.344000, + "Standard_B4als_v2": 0.219000, + "Standard_B4as_v2": 0.266000, + "Standard_B4ls_v2": 0.243000, + "Standard_B4ms": 0.275000, + "Standard_B4s_v2": 0.293000, + "Standard_B8als_v2": 0.475000, + "Standard_B8as_v2": 0.494000, + "Standard_B8ls_v2": 0.523000, + "Standard_B8ms": 0.549000, + "Standard_B8s_v2": 0.549000, + "Standard_D11_v2": 0.272000, + "Standard_D12_v2": 0.542000, + "Standard_D13_v2": 1.085000, + "Standard_D14_v2": 2.171000, + "Standard_D15_v2": 2.713000, + "Standard_D15i_v2": 2.713000, + "Standard_D16_v3": 1.373000, + "Standard_D16_v4": 1.316000, + "Standard_D16_v5": 1.316000, + "Standard_D16a_v4": 1.428000, + "Standard_D16ads_v5": 1.532000, + "Standard_D16as_v4": 1.428000, + "Standard_D16as_v5": 1.279000, + "Standard_D16d_v4": 1.556000, + "Standard_D16d_v5": 1.556000, + "Standard_D16ds_v4": 1.556000, + "Standard_D16ds_v5": 1.556000, + "Standard_D16lds_v5": 1.428000, + "Standard_D16ls_v5": 1.264000, + "Standard_D16s_v3": 1.373000, + "Standard_D16s_v4": 1.316000, + "Standard_D16s_v5": 1.316000, + "Standard_D1_v2": 0.097100, + "Standard_D2_v2": 0.194000, + "Standard_D2_v3": 0.172000, + "Standard_D2_v4": 0.164000, + "Standard_D2_v5": 0.164000, + "Standard_D2a_v4": 0.178000, + "Standard_D2ads_v5": 0.191000, + "Standard_D2as_v4": 0.178000, + "Standard_D2as_v5": 0.252000, + "Standard_D2d_v4": 0.194000, + "Standard_D2d_v5": 0.194000, + "Standard_D2ds_v4": 0.194000, + "Standard_D2ds_v5": 0.194000, + "Standard_D2lds_v5": 0.178000, + "Standard_D2ls_v5": 0.158000, + "Standard_D2s_v3": 0.172000, + "Standard_D2s_v4": 0.164000, + "Standard_D2s_v5": 0.164000, + "Standard_D32-16s_v3": 2.746000, + "Standard_D32-8s_v3": 2.746000, + "Standard_D32_v3": 2.746000, + "Standard_D32_v4": 2.631000, + "Standard_D32_v5": 2.631000, + "Standard_D32a_v4": 2.855000, + "Standard_D32ads_v5": 4.536000, + "Standard_D32as_v4": 2.855000, + "Standard_D32as_v5": 2.558000, + "Standard_D32d_v4": 3.112000, + "Standard_D32d_v5": 3.112000, + "Standard_D32ds_v4": 3.112000, + "Standard_D32ds_v5": 3.112000, + "Standard_D32lds_v5": 2.855000, + "Standard_D32ls_v5": 2.528000, + "Standard_D32s_v3": 2.746000, + "Standard_D32s_v4": 2.631000, + "Standard_D32s_v5": 2.631000, + "Standard_D3_v2": 0.389000, + "Standard_D48_v3": 4.118000, + "Standard_D48_v4": 3.947000, + "Standard_D48_v5": 3.947000, + "Standard_D48a_v4": 4.283000, + "Standard_D48ads_v5": 4.595000, + "Standard_D48as_v4": 4.283000, + "Standard_D48as_v5": 6.045000, + "Standard_D48d_v4": 4.668000, + "Standard_D48d_v5": 4.668000, + "Standard_D48ds_v4": 4.668000, + "Standard_D48ds_v5": 4.668000, + "Standard_D48lds_v5": 4.283000, + "Standard_D48ls_v5": 3.792000, + "Standard_D48s_v3": 4.118000, + "Standard_D48s_v4": 3.947000, + "Standard_D48s_v5": 3.947000, + "Standard_D4_v2": 0.778000, + "Standard_D4_v3": 0.343000, + "Standard_D4_v4": 0.329000, + "Standard_D4_v5": 0.329000, + "Standard_D4a_v4": 0.357000, + "Standard_D4ads_v5": 0.567000, + "Standard_D4as_v4": 0.357000, + "Standard_D4as_v5": 0.504000, + "Standard_D4d_v4": 0.389000, + "Standard_D4d_v5": 0.389000, + "Standard_D4ds_v4": 0.389000, + "Standard_D4ds_v5": 0.389000, + "Standard_D4lds_v5": 0.357000, + "Standard_D4ls_v5": 0.316000, + "Standard_D4s_v3": 0.343000, + "Standard_D4s_v4": 0.329000, + "Standard_D4s_v5": 0.329000, + "Standard_D5_v2": 1.554000, + "Standard_D64-16s_v3": 5.491000, + "Standard_D64-32s_v3": 5.491000, + "Standard_D64_v3": 5.491000, + "Standard_D64_v4": 5.262000, + "Standard_D64_v5": 5.262000, + "Standard_D64a_v4": 5.711000, + "Standard_D64ads_v5": 6.127000, + "Standard_D64as_v4": 5.711000, + "Standard_D64as_v5": 5.116000, + "Standard_D64d_v4": 6.223000, + "Standard_D64d_v5": 6.223000, + "Standard_D64ds_v4": 6.223000, + "Standard_D64ds_v5": 6.223000, + "Standard_D64lds_v5": 5.711000, + "Standard_D64ls_v5": 5.057000, + "Standard_D64s_v3": 5.491000, + "Standard_D64s_v4": 5.262000, + "Standard_D64s_v5": 5.262000, + "Standard_D8_v3": 0.686000, + "Standard_D8_v4": 0.658000, + "Standard_D8_v5": 0.658000, + "Standard_D8a_v4": 0.714000, + "Standard_D8ads_v5": 1.134000, + "Standard_D8as_v4": 0.714000, + "Standard_D8as_v5": 0.639000, + "Standard_D8d_v4": 0.778000, + "Standard_D8d_v5": 0.778000, + "Standard_D8ds_v4": 0.778000, + "Standard_D8ds_v5": 0.778000, + "Standard_D8lds_v5": 0.714000, + "Standard_D8ls_v5": 0.632000, + "Standard_D8s_v3": 0.686000, + "Standard_D8s_v4": 0.658000, + "Standard_D8s_v5": 0.658000, + "Standard_D96_v5": 7.894000, + "Standard_D96a_v4": 8.566000, + "Standard_D96ads_v5": 9.191000, + "Standard_D96as_v4": 8.566000, + "Standard_D96as_v5": 7.674000, + "Standard_D96d_v5": 9.335000, + "Standard_D96ds_v5": 9.335000, + "Standard_D96lds_v5": 8.566000, + "Standard_D96ls_v5": 7.585000, + "Standard_D96s_v5": 7.894000, + "Standard_DS11-1_v2": 0.272000, + "Standard_DS11_v2": 0.272000, + "Standard_DS12-1_v2": 0.542000, + "Standard_DS12-2_v2": 0.542000, + "Standard_DS12_v2": 0.542000, + "Standard_DS13-2_v2": 1.085000, + "Standard_DS13-4_v2": 1.085000, + "Standard_DS13_v2": 1.085000, + "Standard_DS14-4_v2": 2.171000, + "Standard_DS14-8_v2": 2.171000, + "Standard_DS14_v2": 2.171000, + "Standard_DS15_v2": 2.713000, + "Standard_DS15i_v2": 2.713000, + "Standard_DS1_v2": 0.097100, + "Standard_DS2_v2": 0.194000, + "Standard_DS3_v2": 0.389000, + "Standard_DS4_v2": 0.778000, + "Standard_DS5_v2": 1.554000, + "Standard_E104i_v5": 12.433000, + "Standard_E104id_v5": 14.151000, + "Standard_E104ids_v5": 14.151000, + "Standard_E104is_v5": 12.433000, + "Standard_E112iads_v5": 15.001000, + "Standard_E112ias_v5": 12.940000, + "Standard_E112ibds_v5": 19.124000, + "Standard_E112ibs_v5": 17.063000, + "Standard_E16-4ads_v5": 1.948000, + "Standard_E16-4as_v4": 1.874000, + "Standard_E16-4as_v5": 1.681000, + "Standard_E16-4ds_v4": 1.979000, + "Standard_E16-4ds_v5": 1.979000, + "Standard_E16-4s_v3": 1.830000, + "Standard_E16-4s_v4": 1.739000, + "Standard_E16-4s_v5": 1.739000, + "Standard_E16-8ads_v5": 1.948000, + "Standard_E16-8as_v4": 1.874000, + "Standard_E16-8as_v5": 1.681000, + "Standard_E16-8ds_v4": 1.979000, + "Standard_E16-8ds_v5": 1.979000, + "Standard_E16-8s_v3": 1.830000, + "Standard_E16-8s_v4": 1.739000, + "Standard_E16-8s_v5": 1.739000, + "Standard_E16_v3": 1.830000, + "Standard_E16_v4": 1.739000, + "Standard_E16_v5": 1.739000, + "Standard_E16a_v4": 1.874000, + "Standard_E16ads_v5": 1.994000, + "Standard_E16as_v4": 1.874000, + "Standard_E16as_v5": 1.727000, + "Standard_E16bds_v5": 2.484000, + "Standard_E16bs_v5": 2.216000, + "Standard_E16d_v4": 1.979000, + "Standard_E16d_v5": 1.979000, + "Standard_E16ds_v4": 1.979000, + "Standard_E16ds_v5": 1.979000, + "Standard_E16s_v3": 1.830000, + "Standard_E16s_v4": 1.739000, + "Standard_E16s_v5": 1.739000, + "Standard_E20_v3": 2.288000, + "Standard_E20_v4": 2.174000, + "Standard_E20_v5": 2.174000, + "Standard_E20a_v4": 2.342000, + "Standard_E20ads_v5": 2.481000, + "Standard_E20as_v4": 2.342000, + "Standard_E20as_v5": 2.147000, + "Standard_E20d_v4": 2.474000, + "Standard_E20d_v5": 2.474000, + "Standard_E20ds_v4": 2.474000, + "Standard_E20ds_v5": 2.474000, + "Standard_E20s_v3": 2.288000, + "Standard_E20s_v4": 2.174000, + "Standard_E20s_v5": 2.174000, + "Standard_E2_v3": 0.229000, + "Standard_E2_v4": 0.217000, + "Standard_E2_v5": 0.217000, + "Standard_E2a_v4": 0.234000, + "Standard_E2ads_v5": 0.244000, + "Standard_E2as_v4": 0.234000, + "Standard_E2as_v5": 0.210000, + "Standard_E2bds_v5": 0.310000, + "Standard_E2bs_v5": 0.277000, + "Standard_E2d_v4": 0.247000, + "Standard_E2d_v5": 0.247000, + "Standard_E2ds_v4": 0.247000, + "Standard_E2ds_v5": 0.247000, + "Standard_E2s_v3": 0.229000, + "Standard_E2s_v4": 0.217000, + "Standard_E2s_v5": 0.217000, + "Standard_E32-16ads_v5": 3.896000, + "Standard_E32-16as_v4": 3.748000, + "Standard_E32-16as_v5": 4.833000, + "Standard_E32-16ds_v4": 3.958000, + "Standard_E32-16ds_v5": 3.958000, + "Standard_E32-16s_v3": 3.661000, + "Standard_E32-16s_v4": 3.478000, + "Standard_E32-16s_v5": 3.478000, + "Standard_E32-8ads_v5": 5.368000, + "Standard_E32-8as_v4": 3.748000, + "Standard_E32-8as_v5": 3.361000, + "Standard_E32-8ds_v4": 3.958000, + "Standard_E32-8ds_v5": 3.958000, + "Standard_E32-8s_v3": 3.661000, + "Standard_E32-8s_v4": 3.478000, + "Standard_E32-8s_v5": 3.478000, + "Standard_E32_v3": 3.661000, + "Standard_E32_v4": 3.478000, + "Standard_E32_v5": 3.478000, + "Standard_E32a_v4": 3.748000, + "Standard_E32ads_v5": 3.942000, + "Standard_E32as_v4": 3.748000, + "Standard_E32as_v5": 3.361000, + "Standard_E32bds_v5": 4.967000, + "Standard_E32bs_v5": 4.432000, + "Standard_E32d_v4": 3.958000, + "Standard_E32d_v5": 3.958000, + "Standard_E32ds_v4": 3.958000, + "Standard_E32ds_v5": 3.958000, + "Standard_E32s_v3": 3.661000, + "Standard_E32s_v4": 3.478000, + "Standard_E32s_v5": 3.478000, + "Standard_E4-2ads_v5": 0.671000, + "Standard_E4-2as_v4": 0.468000, + "Standard_E4-2as_v5": 0.420000, + "Standard_E4-2ds_v4": 0.495000, + "Standard_E4-2ds_v5": 0.495000, + "Standard_E4-2s_v3": 0.458000, + "Standard_E4-2s_v4": 0.435000, + "Standard_E4-2s_v5": 0.435000, + "Standard_E48_v3": 5.215000, + "Standard_E48_v4": 5.217000, + "Standard_E48_v5": 5.217000, + "Standard_E48a_v4": 5.622000, + "Standard_E48ads_v5": 5.845000, + "Standard_E48as_v4": 5.622000, + "Standard_E48as_v5": 5.088000, + "Standard_E48bds_v5": 7.451000, + "Standard_E48bs_v5": 6.648000, + "Standard_E48d_v4": 5.937000, + "Standard_E48d_v5": 5.937000, + "Standard_E48ds_v4": 5.937000, + "Standard_E48ds_v5": 5.937000, + "Standard_E48s_v3": 5.215000, + "Standard_E48s_v4": 5.217000, + "Standard_E48s_v5": 5.217000, + "Standard_E4_v3": 0.458000, + "Standard_E4_v4": 0.435000, + "Standard_E4_v5": 0.435000, + "Standard_E4a_v4": 0.468000, + "Standard_E4ads_v5": 0.533000, + "Standard_E4as_v4": 0.468000, + "Standard_E4as_v5": 0.466000, + "Standard_E4bds_v5": 0.621000, + "Standard_E4bs_v5": 0.554000, + "Standard_E4d_v4": 0.495000, + "Standard_E4d_v5": 0.495000, + "Standard_E4ds_v4": 0.495000, + "Standard_E4ds_v5": 0.495000, + "Standard_E4s_v3": 0.458000, + "Standard_E4s_v4": 0.435000, + "Standard_E4s_v5": 0.435000, + "Standard_E64-16ads_v5": 7.793000, + "Standard_E64-16as_v4": 7.495000, + "Standard_E64-16as_v5": 6.722000, + "Standard_E64-16ds_v4": 7.916000, + "Standard_E64-16ds_v5": 7.916000, + "Standard_E64-16s_v3": 6.258000, + "Standard_E64-16s_v4": 6.956000, + "Standard_E64-16s_v5": 6.956000, + "Standard_E64-32ads_v5": 10.737000, + "Standard_E64-32as_v4": 7.495000, + "Standard_E64-32as_v5": 9.666000, + "Standard_E64-32ds_v4": 7.916000, + "Standard_E64-32ds_v5": 7.916000, + "Standard_E64-32s_v3": 6.258000, + "Standard_E64-32s_v4": 6.956000, + "Standard_E64-32s_v5": 6.956000, + "Standard_E64_v3": 6.258000, + "Standard_E64_v4": 6.956000, + "Standard_E64_v5": 6.956000, + "Standard_E64a_v4": 7.495000, + "Standard_E64ads_v5": 7.839000, + "Standard_E64as_v4": 7.495000, + "Standard_E64as_v5": 6.722000, + "Standard_E64bds_v5": 9.935000, + "Standard_E64bs_v5": 8.864000, + "Standard_E64d_v4": 7.916000, + "Standard_E64d_v5": 7.916000, + "Standard_E64ds_v4": 7.916000, + "Standard_E64ds_v5": 7.916000, + "Standard_E64i_v3": 6.258000, + "Standard_E64is_v3": 6.258000, + "Standard_E64s_v3": 6.258000, + "Standard_E64s_v4": 6.956000, + "Standard_E64s_v5": 6.956000, + "Standard_E8-2ads_v5": 0.974000, + "Standard_E8-2as_v4": 0.937000, + "Standard_E8-2as_v5": 1.208000, + "Standard_E8-2ds_v4": 0.990000, + "Standard_E8-2ds_v5": 0.990000, + "Standard_E8-2s_v3": 0.915000, + "Standard_E8-2s_v4": 0.869000, + "Standard_E8-2s_v5": 0.869000, + "Standard_E8-4ads_v5": 1.342000, + "Standard_E8-4as_v4": 0.937000, + "Standard_E8-4as_v5": 1.208000, + "Standard_E8-4ds_v4": 0.990000, + "Standard_E8-4ds_v5": 0.990000, + "Standard_E8-4s_v3": 0.915000, + "Standard_E8-4s_v4": 0.869000, + "Standard_E8-4s_v5": 0.869000, + "Standard_E80ids_v4": 9.896000, + "Standard_E80is_v4": 8.694000, + "Standard_E8_v3": 0.915000, + "Standard_E8_v4": 0.869000, + "Standard_E8_v5": 0.869000, + "Standard_E8a_v4": 0.937000, + "Standard_E8ads_v5": 0.974000, + "Standard_E8as_v4": 0.937000, + "Standard_E8as_v5": 0.886000, + "Standard_E8bds_v5": 1.242000, + "Standard_E8bs_v5": 1.108000, + "Standard_E8d_v4": 0.990000, + "Standard_E8d_v5": 0.990000, + "Standard_E8ds_v4": 0.990000, + "Standard_E8ds_v5": 0.990000, + "Standard_E8s_v3": 0.915000, + "Standard_E8s_v4": 0.869000, + "Standard_E8s_v5": 0.869000, + "Standard_E96-24ads_v5": 11.689000, + "Standard_E96-24as_v4": 11.243000, + "Standard_E96-24as_v5": 10.083000, + "Standard_E96-24ds_v5": 11.875000, + "Standard_E96-24s_v5": 10.433000, + "Standard_E96-48ads_v5": 16.105000, + "Standard_E96-48as_v4": 11.243000, + "Standard_E96-48as_v5": 14.499000, + "Standard_E96-48ds_v5": 11.875000, + "Standard_E96-48s_v5": 10.433000, + "Standard_E96_v5": 10.433000, + "Standard_E96a_v4": 11.243000, + "Standard_E96ads_v5": 11.689000, + "Standard_E96as_v4": 11.243000, + "Standard_E96as_v5": 10.129000, + "Standard_E96bds_v5": 14.902000, + "Standard_E96bs_v5": 13.296000, + "Standard_E96d_v5": 11.875000, + "Standard_E96ds_v5": 11.875000, + "Standard_E96iads_v5": 12.858000, + "Standard_E96ias_v4": 11.243000, + "Standard_E96ias_v5": 11.092000, + "Standard_E96s_v5": 10.433000, + "Standard_F1": 0.081200, + "Standard_F16": 1.300000, + "Standard_F16s": 1.300000, + "Standard_F16s_v2": 1.264000, + "Standard_F1s": 0.081200, + "Standard_F2": 0.163000, + "Standard_F2s": 0.163000, + "Standard_F2s_v2": 0.158000, + "Standard_F32s_v2": 2.528000, + "Standard_F4": 0.325000, + "Standard_F48s_v2": 3.792000, + "Standard_F4s": 0.325000, + "Standard_F4s_v2": 0.316000, + "Standard_F64s_v2": 5.057000, + "Standard_F72s_v2": 5.689000, + "Standard_F8": 0.649000, + "Standard_F8s": 0.649000, + "Standard_F8s_v2": 0.632000, + "Standard_L16as_v3": 2.320000, + "Standard_L16s_v3": 2.595000, + "Standard_L32as_v3": 4.640000, + "Standard_L32s_v3": 5.190000, + "Standard_L48as_v3": 6.960000, + "Standard_L48s_v3": 7.786000, + "Standard_L64as_v3": 9.280000, + "Standard_L64s_v3": 10.381000, + "Standard_L80as_v3": 11.600000, + "Standard_L80s_v3": 12.976000, + "Standard_L8as_v3": 1.160000, + "Standard_L8s_v3": 1.298000, + "Standard_M128": 24.809000, + "Standard_M128-32ms": 49.640000, + "Standard_M128-64ms": 49.640000, + "Standard_M128dms_v2": 49.643000, + "Standard_M128ds_v2": 24.812000, + "Standard_M128m": 49.640000, + "Standard_M128ms": 49.640000, + "Standard_M128ms_v2": 48.861000, + "Standard_M128s": 24.809000, + "Standard_M128s_v2": 24.030000, + "Standard_M16-4ms": 5.716000, + "Standard_M16-8ms": 5.716000, + "Standard_M16ms": 5.716000, + "Standard_M16s": 4.440000, + "Standard_M192idms_v2": 59.639000, + "Standard_M192ids_v2": 29.819000, + "Standard_M192ims_v2": 58.856000, + "Standard_M192is_v2": 29.037000, + "Standard_M32-16ms": 11.432000, + "Standard_M32-8ms": 11.432000, + "Standard_M32dms_v2": 11.432000, + "Standard_M32ls": 5.344000, + "Standard_M32ms": 11.432000, + "Standard_M32ms_v2": 11.236000, + "Standard_M32s": 6.203100, + "Standard_M32ts": 5.035000, + "Standard_M64": 12.404000, + "Standard_M64-16ms": 19.226820, + "Standard_M64-32ms": 19.226820, + "Standard_M64dms_v2": 19.232000, + "Standard_M64ds_v2": 12.404000, + "Standard_M64ls": 10.071900, + "Standard_M64m": 19.226820, + "Standard_M64ms": 19.226820, + "Standard_M64ms_v2": 18.841000, + "Standard_M64s": 12.404000, + "Standard_M64s_v2": 12.013000, + "Standard_M8-2ms": 2.858000, + "Standard_M8-4ms": 2.858000, + "Standard_M8ms": 2.858000, + "Standard_NV12s_v3": 2.038000, + "Standard_NV24s_v3": 4.076000, + "Standard_NV48s_v3": 8.151000, + } + // polandcentral + initialOnDemandPrices["polandcentral"] = map[string]float64{ + "Basic_A0": 0.023200, + "Basic_A1": 0.029700, + "Basic_A2": 0.102000, + "Basic_A3": 0.227000, + "Basic_A4": 0.454000, + "Dadsv5_Type1": 8.470000, + "Dasv4_Type1": 6.958000, + "Dasv4_Type2": 6.958000, + "Dasv5_Type1": 7.047000, + "Ddsv4_Type 1": 5.280000, + "Ddsv4_Type2": 6.249000, + "Ddsv5_Type1": 7.899000, + "Dsv3_Type1": 6.970000, + "Dsv3_Type2": 6.970000, + "Dsv3_Type3": 6.970000, + "Dsv3_Type4": 6.970000, + "Dsv4_Type1": 5.566000, + "Dsv4_Type2": 6.679000, + "Dsv5_Type1": 6.958000, + "Eadsv5_Type1": 9.176000, + "Easv4_Type1": 8.828000, + "Easv4_Type2": 8.828000, + "Easv5_Type1": 7.957000, + "Ebdsv5-Type1": 7.705000, + "Ebsv5-Type1": 6.892000, + "Edsv4_Type 1": 6.706000, + "Edsv4_Type2": 7.963000, + "Edsv5_Type1": 10.047000, + "Esv3_Type1": 8.131000, + "Esv3_Type2": 8.131000, + "Esv3_Type3": 8.131000, + "Esv3_Type4": 8.131000, + "Esv4_Type1": 5.887000, + "Esv4_Type2": 7.715000, + "Esv5_Type1": 8.829000, + "Fsv2 Type3": 5.636000, + "Fsv2_Type2": 5.636000, + "Fsv2_Type4": 5.636000, + "Lasv3_Type1": 9.002000, + "Standard_A0": 0.021900, + "Standard_A1": 0.066000, + "Standard_A1_v2": 0.045100, + "Standard_A2": 0.132000, + "Standard_A2_v2": 0.094700, + "Standard_A2m_v2": 0.124000, + "Standard_A3": 0.264000, + "Standard_A4": 0.528000, + "Standard_A4_v2": 0.200000, + "Standard_A4m_v2": 0.249000, + "Standard_A5": 0.275000, + "Standard_A6": 0.550000, + "Standard_A7": 1.100000, + "Standard_A8_v2": 0.419000, + "Standard_A8m_v2": 0.498000, + "Standard_B12ms": 0.634000, + "Standard_B16ms": 0.845000, + "Standard_B1ls": 0.006600, + "Standard_B1ms": 0.026400, + "Standard_B1s": 0.013200, + "Standard_B20ms": 1.056000, + "Standard_B2ms": 0.106000, + "Standard_B2s": 0.052800, + "Standard_B4ms": 0.211000, + "Standard_B8ms": 0.422000, + "Standard_D11_v2": 0.188000, + "Standard_D12_v2": 0.380000, + "Standard_D13_v2": 0.760000, + "Standard_D14_v2": 1.520000, + "Standard_D15_v2": 1.900000, + "Standard_D15i_v2": 1.900000, + "Standard_D16_v3": 1.056000, + "Standard_D16_v4": 1.012000, + "Standard_D16_v5": 1.012000, + "Standard_D16a_v4": 1.012000, + "Standard_D16ads_v5": 1.100000, + "Standard_D16as_v4": 1.012000, + "Standard_D16as_v5": 0.915000, + "Standard_D16d_v4": 1.197000, + "Standard_D16d_v5": 1.197000, + "Standard_D16ds_v4": 1.197000, + "Standard_D16ds_v5": 1.197000, + "Standard_D16lds_v5": 0.977000, + "Standard_D16ls_v5": 0.854000, + "Standard_D16s_v3": 1.056000, + "Standard_D16s_v4": 1.012000, + "Standard_D16s_v5": 1.012000, + "Standard_D1_v2": 0.074800, + "Standard_D2_v2": 0.150000, + "Standard_D2_v3": 0.132000, + "Standard_D2_v4": 0.126000, + "Standard_D2_v5": 0.127000, + "Standard_D2a_v4": 0.126000, + "Standard_D2ads_v5": 0.230000, + "Standard_D2as_v4": 0.126000, + "Standard_D2as_v5": 0.206000, + "Standard_D2d_v4": 0.150000, + "Standard_D2d_v5": 0.150000, + "Standard_D2ds_v4": 0.150000, + "Standard_D2ds_v5": 0.150000, + "Standard_D2lds_v5": 0.122000, + "Standard_D2ls_v5": 0.107000, + "Standard_D2s_v3": 0.132000, + "Standard_D2s_v4": 0.126000, + "Standard_D2s_v5": 0.127000, + "Standard_D32-16s_v3": 2.112000, + "Standard_D32-8s_v3": 2.112000, + "Standard_D32_v3": 2.112000, + "Standard_D32_v4": 2.024000, + "Standard_D32_v5": 2.024000, + "Standard_D32a_v4": 2.024000, + "Standard_D32ads_v5": 2.200000, + "Standard_D32as_v4": 2.024000, + "Standard_D32as_v5": 3.302000, + "Standard_D32d_v4": 2.394000, + "Standard_D32d_v5": 2.394000, + "Standard_D32ds_v4": 2.394000, + "Standard_D32ds_v5": 2.394000, + "Standard_D32lds_v5": 1.954000, + "Standard_D32ls_v5": 1.707000, + "Standard_D32s_v3": 2.112000, + "Standard_D32s_v4": 2.024000, + "Standard_D32s_v5": 2.024000, + "Standard_D3_v2": 0.299000, + "Standard_D48_v3": 3.168000, + "Standard_D48_v4": 3.036000, + "Standard_D48_v5": 3.036000, + "Standard_D48a_v4": 3.036000, + "Standard_D48ads_v5": 5.508000, + "Standard_D48as_v4": 3.036000, + "Standard_D48as_v5": 4.954000, + "Standard_D48d_v4": 3.590000, + "Standard_D48d_v5": 3.590000, + "Standard_D48ds_v4": 3.590000, + "Standard_D48ds_v5": 3.590000, + "Standard_D48lds_v5": 2.930000, + "Standard_D48ls_v5": 2.561000, + "Standard_D48s_v3": 3.168000, + "Standard_D48s_v4": 3.036000, + "Standard_D48s_v5": 3.036000, + "Standard_D4_v2": 0.598000, + "Standard_D4_v3": 0.264000, + "Standard_D4_v4": 0.253000, + "Standard_D4_v5": 0.253000, + "Standard_D4a_v4": 0.253000, + "Standard_D4ads_v5": 0.275000, + "Standard_D4as_v4": 0.253000, + "Standard_D4as_v5": 0.413000, + "Standard_D4d_v4": 0.299000, + "Standard_D4d_v5": 0.299000, + "Standard_D4ds_v4": 0.299000, + "Standard_D4ds_v5": 0.299000, + "Standard_D4lds_v5": 0.244000, + "Standard_D4ls_v5": 0.213000, + "Standard_D4s_v3": 0.264000, + "Standard_D4s_v4": 0.253000, + "Standard_D4s_v5": 0.253000, + "Standard_D5_v2": 1.197000, + "Standard_D64-16s_v3": 4.224000, + "Standard_D64-32s_v3": 4.224000, + "Standard_D64_v3": 4.224000, + "Standard_D64_v4": 4.048000, + "Standard_D64_v5": 4.048000, + "Standard_D64a_v4": 4.048000, + "Standard_D64ads_v5": 7.344000, + "Standard_D64as_v4": 4.048000, + "Standard_D64as_v5": 3.661000, + "Standard_D64d_v4": 4.787000, + "Standard_D64d_v5": 4.787000, + "Standard_D64ds_v4": 4.787000, + "Standard_D64ds_v5": 4.787000, + "Standard_D64lds_v5": 3.907000, + "Standard_D64ls_v5": 3.414000, + "Standard_D64s_v3": 4.224000, + "Standard_D64s_v4": 4.048000, + "Standard_D64s_v5": 4.048000, + "Standard_D8_v3": 0.528000, + "Standard_D8_v4": 0.506000, + "Standard_D8_v5": 0.506000, + "Standard_D8a_v4": 0.506000, + "Standard_D8ads_v5": 0.550000, + "Standard_D8as_v4": 0.506000, + "Standard_D8as_v5": 0.458000, + "Standard_D8d_v4": 0.598000, + "Standard_D8d_v5": 0.598000, + "Standard_D8ds_v4": 0.598000, + "Standard_D8ds_v5": 0.598000, + "Standard_D8lds_v5": 0.488000, + "Standard_D8ls_v5": 0.427000, + "Standard_D8s_v3": 0.528000, + "Standard_D8s_v4": 0.506000, + "Standard_D8s_v5": 0.506000, + "Standard_D96_v5": 6.072000, + "Standard_D96a_v4": 6.072000, + "Standard_D96ads_v5": 11.016000, + "Standard_D96as_v4": 6.072000, + "Standard_D96as_v5": 5.491000, + "Standard_D96d_v5": 7.181000, + "Standard_D96ds_v5": 7.181000, + "Standard_D96lds_v5": 5.861000, + "Standard_D96ls_v5": 5.122000, + "Standard_D96s_v5": 6.072000, + "Standard_DS11-1_v2": 0.188000, + "Standard_DS11_v2": 0.188000, + "Standard_DS12-1_v2": 0.380000, + "Standard_DS12-2_v2": 0.380000, + "Standard_DS12_v2": 0.380000, + "Standard_DS13-2_v2": 0.760000, + "Standard_DS13-4_v2": 0.760000, + "Standard_DS13_v2": 0.760000, + "Standard_DS14-4_v2": 1.520000, + "Standard_DS14-8_v2": 1.520000, + "Standard_DS14_v2": 1.520000, + "Standard_DS15_v2": 1.900000, + "Standard_DS15i_v2": 1.900000, + "Standard_DS1_v2": 0.074800, + "Standard_DS2_v2": 0.150000, + "Standard_DS3_v2": 0.299000, + "Standard_DS4_v2": 0.598000, + "Standard_DS5_v2": 1.197000, + "Standard_E104i_v5": 9.564000, + "Standard_E104id_v5": 10.885000, + "Standard_E104ids_v5": 10.885000, + "Standard_E104is_v5": 9.564000, + "Standard_E112iads_v5": 10.706000, + "Standard_E112ias_v5": 9.283000, + "Standard_E112ibds_v5": 13.484000, + "Standard_E112ibs_v5": 12.061000, + "Standard_E16-4ads_v5": 1.390000, + "Standard_E16-4as_v4": 1.338000, + "Standard_E16-4as_v5": 1.942000, + "Standard_E16-4ds_v4": 1.522000, + "Standard_E16-4ds_v5": 1.522000, + "Standard_E16-4s_v3": 1.408000, + "Standard_E16-4s_v4": 1.338000, + "Standard_E16-4s_v5": 1.338000, + "Standard_E16-8ads_v5": 2.126000, + "Standard_E16-8as_v4": 1.338000, + "Standard_E16-8as_v5": 1.942000, + "Standard_E16-8ds_v4": 1.522000, + "Standard_E16-8ds_v5": 1.522000, + "Standard_E16-8s_v3": 1.408000, + "Standard_E16-8s_v4": 1.338000, + "Standard_E16-8s_v5": 1.338000, + "Standard_E16_v3": 1.408000, + "Standard_E16_v4": 1.338000, + "Standard_E16_v5": 1.338000, + "Standard_E16a_v4": 1.338000, + "Standard_E16ads_v5": 1.390000, + "Standard_E16as_v4": 1.338000, + "Standard_E16as_v5": 1.206000, + "Standard_E16bds_v5": 1.751000, + "Standard_E16bs_v5": 1.566000, + "Standard_E16d_v4": 1.522000, + "Standard_E16d_v5": 1.522000, + "Standard_E16ds_v4": 1.522000, + "Standard_E16ds_v5": 1.522000, + "Standard_E16s_v3": 1.408000, + "Standard_E16s_v4": 1.338000, + "Standard_E16s_v5": 1.338000, + "Standard_E20_v3": 1.760000, + "Standard_E20_v4": 1.672000, + "Standard_E20_v5": 1.672000, + "Standard_E20a_v4": 1.672000, + "Standard_E20ads_v5": 1.784000, + "Standard_E20as_v4": 1.672000, + "Standard_E20as_v5": 1.553000, + "Standard_E20d_v4": 1.903000, + "Standard_E20d_v5": 1.903000, + "Standard_E20ds_v4": 1.903000, + "Standard_E20ds_v5": 1.903000, + "Standard_E20s_v3": 1.760000, + "Standard_E20s_v4": 1.672000, + "Standard_E20s_v5": 1.672000, + "Standard_E2_v3": 0.176000, + "Standard_E2_v4": 0.167000, + "Standard_E2_v5": 0.167000, + "Standard_E2a_v4": 0.167000, + "Standard_E2ads_v5": 0.174000, + "Standard_E2as_v4": 0.167000, + "Standard_E2as_v5": 0.151000, + "Standard_E2bds_v5": 0.219000, + "Standard_E2bs_v5": 0.196000, + "Standard_E2d_v4": 0.190000, + "Standard_E2d_v5": 0.190000, + "Standard_E2ds_v4": 0.190000, + "Standard_E2ds_v5": 0.190000, + "Standard_E2s_v3": 0.176000, + "Standard_E2s_v4": 0.167000, + "Standard_E2s_v5": 0.167000, + "Standard_E32-16ads_v5": 4.253000, + "Standard_E32-16as_v4": 2.675000, + "Standard_E32-16as_v5": 3.883000, + "Standard_E32-16ds_v4": 3.045000, + "Standard_E32-16ds_v5": 3.045000, + "Standard_E32-16s_v3": 2.816000, + "Standard_E32-16s_v4": 2.675000, + "Standard_E32-16s_v5": 2.675000, + "Standard_E32-8ads_v5": 4.253000, + "Standard_E32-8as_v4": 2.675000, + "Standard_E32-8as_v5": 2.411000, + "Standard_E32-8ds_v4": 3.045000, + "Standard_E32-8ds_v5": 3.045000, + "Standard_E32-8s_v3": 2.816000, + "Standard_E32-8s_v4": 2.675000, + "Standard_E32-8s_v5": 2.675000, + "Standard_E32_v3": 2.816000, + "Standard_E32_v4": 2.675000, + "Standard_E32_v5": 2.675000, + "Standard_E32a_v4": 2.675000, + "Standard_E32ads_v5": 2.781000, + "Standard_E32as_v4": 2.675000, + "Standard_E32as_v5": 2.457000, + "Standard_E32bds_v5": 3.502000, + "Standard_E32bs_v5": 3.133000, + "Standard_E32d_v4": 3.045000, + "Standard_E32d_v5": 3.045000, + "Standard_E32ds_v4": 3.045000, + "Standard_E32ds_v5": 3.045000, + "Standard_E32s_v3": 2.816000, + "Standard_E32s_v4": 2.675000, + "Standard_E32s_v5": 2.675000, + "Standard_E4-2ads_v5": 0.532000, + "Standard_E4-2as_v4": 0.334000, + "Standard_E4-2as_v5": 0.485000, + "Standard_E4-2ds_v4": 0.381000, + "Standard_E4-2ds_v5": 0.381000, + "Standard_E4-2s_v3": 0.352000, + "Standard_E4-2s_v4": 0.334000, + "Standard_E4-2s_v5": 0.334000, + "Standard_E48_v3": 4.224000, + "Standard_E48_v4": 4.013000, + "Standard_E48_v5": 4.013000, + "Standard_E48a_v4": 4.013000, + "Standard_E48ads_v5": 4.171000, + "Standard_E48as_v4": 4.013000, + "Standard_E48as_v5": 3.617000, + "Standard_E48bds_v5": 5.254000, + "Standard_E48bs_v5": 4.699000, + "Standard_E48d_v4": 4.567000, + "Standard_E48d_v5": 4.567000, + "Standard_E48ds_v4": 4.567000, + "Standard_E48ds_v5": 4.567000, + "Standard_E48s_v3": 4.224000, + "Standard_E48s_v4": 4.013000, + "Standard_E48s_v5": 4.013000, + "Standard_E4_v3": 0.352000, + "Standard_E4_v4": 0.334000, + "Standard_E4_v5": 0.334000, + "Standard_E4a_v4": 0.334000, + "Standard_E4ads_v5": 0.394000, + "Standard_E4as_v4": 0.334000, + "Standard_E4as_v5": 0.301000, + "Standard_E4bds_v5": 0.438000, + "Standard_E4bs_v5": 0.392000, + "Standard_E4d_v4": 0.381000, + "Standard_E4d_v5": 0.381000, + "Standard_E4ds_v4": 0.381000, + "Standard_E4ds_v5": 0.381000, + "Standard_E4s_v3": 0.352000, + "Standard_E4s_v4": 0.334000, + "Standard_E4s_v5": 0.334000, + "Standard_E64-16ads_v5": 5.562000, + "Standard_E64-16as_v4": 5.350000, + "Standard_E64-16as_v5": 4.822000, + "Standard_E64-16ds_v4": 6.090000, + "Standard_E64-16ds_v5": 6.090000, + "Standard_E64-16s_v3": 5.069000, + "Standard_E64-16s_v4": 5.350000, + "Standard_E64-16s_v5": 5.350000, + "Standard_E64-32ads_v5": 5.562000, + "Standard_E64-32as_v4": 5.350000, + "Standard_E64-32as_v5": 7.766000, + "Standard_E64-32ds_v4": 6.090000, + "Standard_E64-32ds_v5": 6.090000, + "Standard_E64-32s_v3": 5.069000, + "Standard_E64-32s_v4": 5.350000, + "Standard_E64-32s_v5": 5.350000, + "Standard_E64_v3": 5.069000, + "Standard_E64_v4": 5.350000, + "Standard_E64_v5": 5.350000, + "Standard_E64a_v4": 5.350000, + "Standard_E64ads_v5": 5.562000, + "Standard_E64as_v4": 5.350000, + "Standard_E64as_v5": 4.868000, + "Standard_E64bds_v5": 7.005000, + "Standard_E64bs_v5": 6.266000, + "Standard_E64d_v4": 6.090000, + "Standard_E64d_v5": 6.090000, + "Standard_E64ds_v4": 6.090000, + "Standard_E64ds_v5": 6.090000, + "Standard_E64i_v3": 5.069000, + "Standard_E64is_v3": 5.069000, + "Standard_E64s_v3": 5.069000, + "Standard_E64s_v4": 5.350000, + "Standard_E64s_v5": 5.350000, + "Standard_E8-2ads_v5": 0.695000, + "Standard_E8-2as_v4": 0.669000, + "Standard_E8-2as_v5": 0.971000, + "Standard_E8-2ds_v4": 0.761000, + "Standard_E8-2ds_v5": 0.761000, + "Standard_E8-2s_v3": 0.704000, + "Standard_E8-2s_v4": 0.669000, + "Standard_E8-2s_v5": 0.669000, + "Standard_E8-4ads_v5": 1.063000, + "Standard_E8-4as_v4": 0.669000, + "Standard_E8-4as_v5": 0.603000, + "Standard_E8-4ds_v4": 0.761000, + "Standard_E8-4ds_v5": 0.761000, + "Standard_E8-4s_v3": 0.704000, + "Standard_E8-4s_v4": 0.669000, + "Standard_E8-4s_v5": 0.669000, + "Standard_E80ids_v4": 7.612000, + "Standard_E80is_v4": 6.688000, + "Standard_E8_v3": 0.704000, + "Standard_E8_v4": 0.669000, + "Standard_E8_v5": 0.669000, + "Standard_E8a_v4": 0.669000, + "Standard_E8ads_v5": 0.741000, + "Standard_E8as_v4": 0.669000, + "Standard_E8as_v5": 0.603000, + "Standard_E8bds_v5": 0.876000, + "Standard_E8bs_v5": 0.783000, + "Standard_E8d_v4": 0.761000, + "Standard_E8d_v5": 0.761000, + "Standard_E8ds_v4": 0.761000, + "Standard_E8ds_v5": 0.761000, + "Standard_E8s_v3": 0.704000, + "Standard_E8s_v4": 0.669000, + "Standard_E8s_v5": 0.669000, + "Standard_E96-24ads_v5": 8.342000, + "Standard_E96-24as_v4": 8.026000, + "Standard_E96-24as_v5": 11.650000, + "Standard_E96-24ds_v5": 9.134000, + "Standard_E96-24s_v5": 8.026000, + "Standard_E96-48ads_v5": 12.758000, + "Standard_E96-48as_v4": 8.026000, + "Standard_E96-48as_v5": 7.234000, + "Standard_E96-48ds_v5": 9.134000, + "Standard_E96-48s_v5": 8.026000, + "Standard_E96_v5": 8.026000, + "Standard_E96a_v4": 8.026000, + "Standard_E96ads_v5": 8.388000, + "Standard_E96as_v4": 8.026000, + "Standard_E96as_v5": 7.280000, + "Standard_E96bds_v5": 10.507000, + "Standard_E96bs_v5": 9.398000, + "Standard_E96d_v5": 9.134000, + "Standard_E96ds_v5": 9.134000, + "Standard_E96iads_v5": 9.177000, + "Standard_E96ias_v4": 8.026000, + "Standard_E96ias_v5": 7.957000, + "Standard_E96s_v5": 8.026000, + "Standard_F1": 0.062400, + "Standard_F16": 0.999000, + "Standard_F16s": 0.999000, + "Standard_F16s_v2": 0.854000, + "Standard_F1s": 0.062400, + "Standard_F2": 0.125000, + "Standard_F2s": 0.125000, + "Standard_F2s_v2": 0.107000, + "Standard_F32s_v2": 1.707000, + "Standard_F4": 0.250000, + "Standard_F48s_v2": 2.561000, + "Standard_F4s": 0.250000, + "Standard_F4s_v2": 0.213000, + "Standard_F64s_v2": 3.414000, + "Standard_F72s_v2": 3.841000, + "Standard_F8": 0.499000, + "Standard_F8s": 0.499000, + "Standard_F8s_v2": 0.427000, + "Standard_L112ias_v3": 12.603000, + "Standard_L16as_v3": 1.637000, + "Standard_L32as_v3": 3.274000, + "Standard_L48as_v3": 4.910000, + "Standard_L64as_v3": 6.547000, + "Standard_L80as_v3": 8.184000, + "Standard_L8as_v3": 0.818000, + "Standard_L96ias_v3": 10.803000, + "Standard_ND96ams_A100_v4": 46.861000, + "Standard_ND96amsr_A100_v4": 46.861000, + "Standard_SQLG7_AMD_IaaS": 10.701000, + "Standard_SQLG7_AMD_NVME": 10.701000, + } + // qatarcentral + initialOnDemandPrices["qatarcentral"] = map[string]float64{ + "Basic_A0": 0.019800, + "Basic_A1": 0.029700, + "Basic_A2": 0.085800, + "Basic_A3": 0.224000, + "Basic_A4": 0.449000, + "Dadsv5_Type1": 7.817000, + "Dasv4_Type1": 6.474000, + "Dasv4_Type2": 6.474000, + "Dasv5_Type1": 6.530000, + "Ddsv4_Type 1": 4.136000, + "Ddsv4_Type2": 5.789000, + "Ddsv5_Type1": 7.318000, + "Dsv3_Type3": 5.170000, + "Dsv3_Type4": 6.460000, + "Dsv4_Type1": 5.170000, + "Dsv4_Type2": 6.217000, + "Dsv5_Type1": 6.474000, + "Eadsv5_Type1": 8.496000, + "Easv4_Type1": 8.189000, + "Easv4_Type2": 8.189000, + "Easv5_Type1": 7.392000, + "Ebdsv5-Type1": 7.251000, + "Ebsv5-Type1": 6.512000, + "Edsv4_Type 1": 6.195000, + "Edsv4_Type2": 7.942000, + "Edsv5_Type1": 9.293000, + "Esv3_Type3": 6.653000, + "Esv3_Type4": 6.986000, + "Esv4_Type1": 5.456000, + "Esv4_Type2": 7.161000, + "Esv5_Type1": 8.189000, + "Fsv2 Type3": 4.874000, + "Fsv2_Type2": 4.178000, + "Fsv2_Type4": 5.386000, + "Lasv3_Type1": 8.316000, + "Lsv3_Type1": 9.284000, + "Standard_A0": 0.022000, + "Standard_A1": 0.066000, + "Standard_A1_v2": 0.043200, + "Standard_A2": 0.132000, + "Standard_A2_v2": 0.091200, + "Standard_A2m_v2": 0.119000, + "Standard_A3": 0.264000, + "Standard_A4": 0.528000, + "Standard_A4_v2": 0.191000, + "Standard_A4m_v2": 0.250000, + "Standard_A5": 0.297000, + "Standard_A6": 0.594000, + "Standard_A7": 1.188000, + "Standard_A8_v2": 0.400000, + "Standard_A8m_v2": 0.524000, + "Standard_B12ms": 0.602000, + "Standard_B16als_v2": 0.713000, + "Standard_B16as_v2": 0.796000, + "Standard_B16ls_v2": 0.784000, + "Standard_B16ms": 0.802000, + "Standard_B16s_v2": 0.876000, + "Standard_B1ls": 0.006300, + "Standard_B1ms": 0.025100, + "Standard_B1s": 0.012500, + "Standard_B20ms": 1.003000, + "Standard_B2als_v2": 0.054300, + "Standard_B2as_v2": 0.090300, + "Standard_B2ats_v2": 0.011300, + "Standard_B2ls_v2": 0.059400, + "Standard_B2ms": 0.100000, + "Standard_B2s": 0.050200, + "Standard_B2s_v2": 0.110000, + "Standard_B2ts_v2": 0.021700, + "Standard_B32als_v2": 1.279000, + "Standard_B32as_v2": 1.592000, + "Standard_B32ls_v2": 1.421000, + "Standard_B32s_v2": 1.605000, + "Standard_B4als_v2": 0.178000, + "Standard_B4as_v2": 0.181000, + "Standard_B4ls_v2": 0.178000, + "Standard_B4ms": 0.201000, + "Standard_B4s_v2": 0.219000, + "Standard_B8als_v2": 0.320000, + "Standard_B8as_v2": 0.361000, + "Standard_B8ls_v2": 0.392000, + "Standard_B8ms": 0.401000, + "Standard_B8s_v2": 0.401000, + "Standard_D11_v2": 0.179000, + "Standard_D11_v2_Promo": 0.179000, + "Standard_D12_v2": 0.359000, + "Standard_D12_v2_Promo": 0.359000, + "Standard_D13_v2": 0.718000, + "Standard_D13_v2_Promo": 0.718000, + "Standard_D14_v2": 1.435000, + "Standard_D14_v2_Promo": 1.435000, + "Standard_D15_v2": 1.794000, + "Standard_D15i_v2": 1.794000, + "Standard_D16_v3": 0.942000, + "Standard_D16_v4": 0.942000, + "Standard_D16_v5": 0.942000, + "Standard_D16a_v4": 0.942000, + "Standard_D16ads_v5": 1.751000, + "Standard_D16as_v4": 0.942000, + "Standard_D16as_v5": 1.584000, + "Standard_D16d_v4": 1.109000, + "Standard_D16d_v5": 1.109000, + "Standard_D16ds_v4": 1.109000, + "Standard_D16ds_v5": 1.109000, + "Standard_D16lds_v5": 0.960000, + "Standard_D16ls_v5": 0.845000, + "Standard_D16s_v3": 0.942000, + "Standard_D16s_v4": 0.942000, + "Standard_D16s_v5": 0.942000, + "Standard_D1_v2": 0.068700, + "Standard_D2_v2": 0.137000, + "Standard_D2_v2_Promo": 0.137000, + "Standard_D2_v3": 0.118000, + "Standard_D2_v4": 0.118000, + "Standard_D2_v5": 0.118000, + "Standard_D2a_v4": 0.118000, + "Standard_D2ads_v5": 0.219000, + "Standard_D2as_v4": 0.118000, + "Standard_D2as_v5": 0.198000, + "Standard_D2d_v4": 0.139000, + "Standard_D2d_v5": 0.139000, + "Standard_D2ds_v4": 0.139000, + "Standard_D2ds_v5": 0.139000, + "Standard_D2lds_v5": 0.120000, + "Standard_D2ls_v5": 0.106000, + "Standard_D2s_v3": 0.118000, + "Standard_D2s_v4": 0.118000, + "Standard_D2s_v5": 0.118000, + "Standard_D32-16s_v3": 1.883000, + "Standard_D32-8s_v3": 1.883000, + "Standard_D32_v3": 1.883000, + "Standard_D32_v4": 1.883000, + "Standard_D32_v5": 1.883000, + "Standard_D32a_v4": 1.883000, + "Standard_D32ads_v5": 3.502000, + "Standard_D32as_v4": 1.883000, + "Standard_D32as_v5": 3.168000, + "Standard_D32d_v4": 2.218000, + "Standard_D32d_v5": 2.218000, + "Standard_D32ds_v4": 2.218000, + "Standard_D32ds_v5": 2.218000, + "Standard_D32lds_v5": 1.920000, + "Standard_D32ls_v5": 1.690000, + "Standard_D32s_v3": 1.883000, + "Standard_D32s_v4": 1.883000, + "Standard_D32s_v5": 1.883000, + "Standard_D3_v2": 0.275000, + "Standard_D3_v2_Promo": 0.275000, + "Standard_D48_v3": 2.825000, + "Standard_D48_v4": 2.825000, + "Standard_D48_v5": 2.825000, + "Standard_D48a_v4": 2.825000, + "Standard_D48ads_v5": 3.046000, + "Standard_D48as_v4": 2.825000, + "Standard_D48as_v5": 2.544000, + "Standard_D48d_v4": 3.326000, + "Standard_D48d_v5": 3.326000, + "Standard_D48ds_v4": 3.326000, + "Standard_D48ds_v5": 3.326000, + "Standard_D48lds_v5": 2.880000, + "Standard_D48ls_v5": 2.534000, + "Standard_D48s_v3": 2.825000, + "Standard_D48s_v4": 2.825000, + "Standard_D48s_v5": 2.825000, + "Standard_D4_v2": 0.550000, + "Standard_D4_v2_Promo": 0.550000, + "Standard_D4_v3": 0.235000, + "Standard_D4_v4": 0.235000, + "Standard_D4_v5": 0.235000, + "Standard_D4a_v4": 0.235000, + "Standard_D4ads_v5": 0.438000, + "Standard_D4as_v4": 0.235000, + "Standard_D4as_v5": 0.396000, + "Standard_D4d_v4": 0.277000, + "Standard_D4d_v5": 0.277000, + "Standard_D4ds_v4": 0.277000, + "Standard_D4ds_v5": 0.277000, + "Standard_D4lds_v5": 0.240000, + "Standard_D4ls_v5": 0.211000, + "Standard_D4s_v3": 0.235000, + "Standard_D4s_v4": 0.235000, + "Standard_D4s_v5": 0.235000, + "Standard_D5_v2": 1.099000, + "Standard_D5_v2_Promo": 1.099000, + "Standard_D64-16s_v3": 3.766000, + "Standard_D64-32s_v3": 3.766000, + "Standard_D64_v3": 3.766000, + "Standard_D64_v4": 3.766000, + "Standard_D64_v5": 3.766000, + "Standard_D64a_v4": 3.766000, + "Standard_D64ads_v5": 7.005000, + "Standard_D64as_v4": 3.766000, + "Standard_D64as_v5": 6.336000, + "Standard_D64d_v4": 4.435000, + "Standard_D64d_v5": 4.435000, + "Standard_D64ds_v4": 4.435000, + "Standard_D64ds_v5": 4.435000, + "Standard_D64lds_v5": 3.840000, + "Standard_D64ls_v5": 3.379000, + "Standard_D64s_v3": 3.766000, + "Standard_D64s_v4": 3.766000, + "Standard_D64s_v5": 3.766000, + "Standard_D8_v3": 0.471000, + "Standard_D8_v4": 0.471000, + "Standard_D8_v5": 0.471000, + "Standard_D8a_v4": 0.471000, + "Standard_D8ads_v5": 0.876000, + "Standard_D8as_v4": 0.471000, + "Standard_D8as_v5": 0.424000, + "Standard_D8d_v4": 0.554000, + "Standard_D8d_v5": 0.554000, + "Standard_D8ds_v4": 0.554000, + "Standard_D8ds_v5": 0.554000, + "Standard_D8lds_v5": 0.480000, + "Standard_D8ls_v5": 0.422000, + "Standard_D8s_v3": 0.471000, + "Standard_D8s_v4": 0.471000, + "Standard_D8s_v5": 0.471000, + "Standard_D96_v5": 5.650000, + "Standard_D96a_v4": 5.650000, + "Standard_D96ads_v5": 6.091000, + "Standard_D96as_v4": 5.650000, + "Standard_D96as_v5": 9.504000, + "Standard_D96d_v5": 6.653000, + "Standard_D96ds_v5": 6.653000, + "Standard_D96lds_v5": 5.760000, + "Standard_D96ls_v5": 5.069000, + "Standard_D96s_v5": 5.650000, + "Standard_DS1": 0.080400, + "Standard_DS11": 0.208000, + "Standard_DS11-1_v2": 0.179000, + "Standard_DS11_v2": 0.179000, + "Standard_DS11_v2_Promo": 0.179000, + "Standard_DS12": 0.416000, + "Standard_DS12-1_v2": 0.359000, + "Standard_DS12-2_v2": 0.359000, + "Standard_DS12_v2": 0.359000, + "Standard_DS12_v2_Promo": 0.359000, + "Standard_DS13": 0.832000, + "Standard_DS13-2_v2": 0.718000, + "Standard_DS13-4_v2": 0.718000, + "Standard_DS13_v2": 0.718000, + "Standard_DS13_v2_Promo": 0.718000, + "Standard_DS14": 1.665000, + "Standard_DS14-4_v2": 1.435000, + "Standard_DS14-8_v2": 1.435000, + "Standard_DS14_v2": 1.435000, + "Standard_DS14_v2_Promo": 1.435000, + "Standard_DS15_v2": 1.794000, + "Standard_DS15i_v2": 1.794000, + "Standard_DS1_v2": 0.068700, + "Standard_DS2": 0.161000, + "Standard_DS2_v2": 0.137000, + "Standard_DS2_v2_Promo": 0.137000, + "Standard_DS3": 0.322000, + "Standard_DS3_v2": 0.275000, + "Standard_DS3_v2_Promo": 0.275000, + "Standard_DS4": 0.643000, + "Standard_DS4_v2": 0.550000, + "Standard_DS4_v2_Promo": 0.550000, + "Standard_DS5_v2": 1.099000, + "Standard_DS5_v2_Promo": 1.099000, + "Standard_E104i_v5": 8.872000, + "Standard_E104id_v5": 10.067000, + "Standard_E104ids_v5": 10.067000, + "Standard_E104is_v5": 8.872000, + "Standard_E112iads_v5": 9.911000, + "Standard_E112ias_v5": 8.624000, + "Standard_E112ibds_v5": 12.690000, + "Standard_E112ibs_v5": 11.396000, + "Standard_E16-4ads_v5": 2.023000, + "Standard_E16-4as_v4": 1.241000, + "Standard_E16-4as_v5": 1.856000, + "Standard_E16-4ds_v4": 1.408000, + "Standard_E16-4ds_v5": 1.408000, + "Standard_E16-4s_v3": 1.241000, + "Standard_E16-4s_v4": 1.241000, + "Standard_E16-4s_v5": 1.241000, + "Standard_E16-8ads_v5": 2.023000, + "Standard_E16-8as_v4": 1.241000, + "Standard_E16-8as_v5": 1.120000, + "Standard_E16-8ds_v4": 1.408000, + "Standard_E16-8ds_v5": 1.408000, + "Standard_E16-8s_v3": 1.241000, + "Standard_E16-8s_v4": 1.241000, + "Standard_E16-8s_v5": 1.241000, + "Standard_E16_v3": 1.241000, + "Standard_E16_v4": 1.241000, + "Standard_E16_v5": 1.241000, + "Standard_E16a_v4": 1.241000, + "Standard_E16ads_v5": 1.333000, + "Standard_E16as_v4": 1.241000, + "Standard_E16as_v5": 1.120000, + "Standard_E16bds_v5": 1.648000, + "Standard_E16bs_v5": 1.480000, + "Standard_E16d_v4": 1.408000, + "Standard_E16d_v5": 1.408000, + "Standard_E16ds_v4": 1.408000, + "Standard_E16ds_v5": 1.408000, + "Standard_E16s_v3": 1.241000, + "Standard_E16s_v4": 1.241000, + "Standard_E16s_v5": 1.241000, + "Standard_E20_v3": 1.550000, + "Standard_E20_v4": 1.550000, + "Standard_E20_v5": 1.551000, + "Standard_E20a_v4": 1.551000, + "Standard_E20ads_v5": 1.609000, + "Standard_E20as_v4": 1.551000, + "Standard_E20as_v5": 1.446000, + "Standard_E20d_v4": 1.760000, + "Standard_E20d_v5": 1.760000, + "Standard_E20ds_v4": 1.760000, + "Standard_E20ds_v5": 1.760000, + "Standard_E20s_v3": 1.550000, + "Standard_E20s_v4": 1.550000, + "Standard_E20s_v5": 1.551000, + "Standard_E2_v3": 0.155000, + "Standard_E2_v4": 0.155000, + "Standard_E2_v5": 0.155000, + "Standard_E2a_v4": 0.155000, + "Standard_E2ads_v5": 0.207000, + "Standard_E2as_v4": 0.155000, + "Standard_E2as_v5": 0.140000, + "Standard_E2bds_v5": 0.206000, + "Standard_E2bs_v5": 0.185000, + "Standard_E2d_v4": 0.176000, + "Standard_E2d_v5": 0.176000, + "Standard_E2ds_v4": 0.176000, + "Standard_E2ds_v5": 0.176000, + "Standard_E2s_v3": 0.155000, + "Standard_E2s_v4": 0.155000, + "Standard_E2s_v5": 0.155000, + "Standard_E32-16ads_v5": 2.574000, + "Standard_E32-16as_v4": 2.482000, + "Standard_E32-16as_v5": 3.712000, + "Standard_E32-16ds_v4": 2.816000, + "Standard_E32-16ds_v5": 2.816000, + "Standard_E32-16s_v3": 2.482000, + "Standard_E32-16s_v4": 2.482000, + "Standard_E32-16s_v5": 2.482000, + "Standard_E32-8ads_v5": 2.574000, + "Standard_E32-8as_v4": 2.482000, + "Standard_E32-8as_v5": 2.240000, + "Standard_E32-8ds_v4": 2.816000, + "Standard_E32-8ds_v5": 2.816000, + "Standard_E32-8s_v3": 2.482000, + "Standard_E32-8s_v4": 2.482000, + "Standard_E32-8s_v5": 2.482000, + "Standard_E32_v3": 2.482000, + "Standard_E32_v4": 2.482000, + "Standard_E32_v5": 2.482000, + "Standard_E32a_v4": 2.482000, + "Standard_E32ads_v5": 2.620000, + "Standard_E32as_v4": 2.482000, + "Standard_E32as_v5": 2.240000, + "Standard_E32bds_v5": 3.296000, + "Standard_E32bs_v5": 2.960000, + "Standard_E32d_v4": 2.816000, + "Standard_E32d_v5": 2.816000, + "Standard_E32ds_v4": 2.816000, + "Standard_E32ds_v5": 2.816000, + "Standard_E32s_v3": 2.482000, + "Standard_E32s_v4": 2.482000, + "Standard_E32s_v5": 2.482000, + "Standard_E4-2ads_v5": 0.506000, + "Standard_E4-2as_v4": 0.310000, + "Standard_E4-2as_v5": 0.280000, + "Standard_E4-2ds_v4": 0.352000, + "Standard_E4-2ds_v5": 0.352000, + "Standard_E4-2s_v3": 0.310000, + "Standard_E4-2s_v4": 0.310000, + "Standard_E4-2s_v5": 0.310000, + "Standard_E48_v3": 3.722000, + "Standard_E48_v4": 3.722000, + "Standard_E48_v5": 3.722000, + "Standard_E48a_v4": 3.722000, + "Standard_E48ads_v5": 3.908000, + "Standard_E48as_v4": 3.722000, + "Standard_E48as_v5": 3.406000, + "Standard_E48bds_v5": 4.944000, + "Standard_E48bs_v5": 4.440000, + "Standard_E48d_v4": 4.224000, + "Standard_E48d_v5": 4.224000, + "Standard_E48ds_v4": 4.224000, + "Standard_E48ds_v5": 4.224000, + "Standard_E48s_v3": 3.722000, + "Standard_E48s_v4": 3.722000, + "Standard_E48s_v5": 3.722000, + "Standard_E4_v3": 0.310000, + "Standard_E4_v4": 0.310000, + "Standard_E4_v5": 0.310000, + "Standard_E4a_v4": 0.310000, + "Standard_E4ads_v5": 0.368000, + "Standard_E4as_v4": 0.310000, + "Standard_E4as_v5": 0.326000, + "Standard_E4bds_v5": 0.412000, + "Standard_E4bs_v5": 0.370000, + "Standard_E4d_v4": 0.352000, + "Standard_E4d_v5": 0.352000, + "Standard_E4ds_v4": 0.352000, + "Standard_E4ds_v5": 0.352000, + "Standard_E4s_v3": 0.310000, + "Standard_E4s_v4": 0.310000, + "Standard_E4s_v5": 0.310000, + "Standard_E64-16ads_v5": 8.093000, + "Standard_E64-16as_v4": 4.963000, + "Standard_E64-16as_v5": 4.480000, + "Standard_E64-16ds_v4": 5.632000, + "Standard_E64-16ds_v5": 5.632000, + "Standard_E64-16s_v3": 4.963000, + "Standard_E64-16s_v4": 4.963000, + "Standard_E64-16s_v5": 4.963000, + "Standard_E64-32ads_v5": 8.093000, + "Standard_E64-32as_v4": 4.963000, + "Standard_E64-32as_v5": 4.480000, + "Standard_E64-32ds_v4": 5.632000, + "Standard_E64-32ds_v5": 5.632000, + "Standard_E64-32s_v3": 4.963000, + "Standard_E64-32s_v4": 4.963000, + "Standard_E64-32s_v5": 4.963000, + "Standard_E64_v3": 4.963000, + "Standard_E64_v4": 4.963000, + "Standard_E64_v5": 4.963000, + "Standard_E64a_v4": 4.963000, + "Standard_E64ads_v5": 5.149000, + "Standard_E64as_v4": 4.963000, + "Standard_E64as_v5": 4.526000, + "Standard_E64bds_v5": 6.592000, + "Standard_E64bs_v5": 5.920000, + "Standard_E64d_v4": 5.632000, + "Standard_E64d_v5": 5.632000, + "Standard_E64ds_v4": 5.632000, + "Standard_E64ds_v5": 5.632000, + "Standard_E64i_v3": 4.963000, + "Standard_E64is_v3": 4.963000, + "Standard_E64is_v4_SPECIAL": 4.963000, + "Standard_E64s_v3": 4.963000, + "Standard_E64s_v4": 4.963000, + "Standard_E64s_v5": 4.963000, + "Standard_E8-2ads_v5": 1.012000, + "Standard_E8-2as_v4": 0.620000, + "Standard_E8-2as_v5": 0.928000, + "Standard_E8-2ds_v4": 0.704000, + "Standard_E8-2ds_v5": 0.704000, + "Standard_E8-2s_v3": 0.620000, + "Standard_E8-2s_v4": 0.620000, + "Standard_E8-2s_v5": 0.620000, + "Standard_E8-4ads_v5": 1.012000, + "Standard_E8-4as_v4": 0.620000, + "Standard_E8-4as_v5": 0.928000, + "Standard_E8-4ds_v4": 0.704000, + "Standard_E8-4ds_v5": 0.704000, + "Standard_E8-4s_v3": 0.620000, + "Standard_E8-4s_v4": 0.620000, + "Standard_E8-4s_v5": 0.620000, + "Standard_E80ids_v4": 7.040000, + "Standard_E80is_v4": 6.200000, + "Standard_E8_v3": 0.620000, + "Standard_E8_v4": 0.620000, + "Standard_E8_v5": 0.620000, + "Standard_E8a_v4": 0.620000, + "Standard_E8ads_v5": 0.644000, + "Standard_E8as_v4": 0.620000, + "Standard_E8as_v5": 0.560000, + "Standard_E8bds_v5": 0.824000, + "Standard_E8bs_v5": 0.740000, + "Standard_E8d_v4": 0.704000, + "Standard_E8d_v5": 0.704000, + "Standard_E8ds_v4": 0.704000, + "Standard_E8ds_v5": 0.704000, + "Standard_E8s_v3": 0.620000, + "Standard_E8s_v4": 0.620000, + "Standard_E8s_v5": 0.620000, + "Standard_E96-24ads_v5": 12.139000, + "Standard_E96-24as_v4": 7.445000, + "Standard_E96-24as_v5": 11.136000, + "Standard_E96-24ds_v5": 8.448000, + "Standard_E96-24s_v5": 7.445000, + "Standard_E96-48ads_v5": 12.139000, + "Standard_E96-48as_v4": 7.445000, + "Standard_E96-48as_v5": 6.720000, + "Standard_E96-48ds_v5": 8.448000, + "Standard_E96-48s_v5": 7.445000, + "Standard_E96_v5": 7.445000, + "Standard_E96a_v4": 7.445000, + "Standard_E96ads_v5": 7.769000, + "Standard_E96as_v4": 7.445000, + "Standard_E96as_v5": 6.766000, + "Standard_E96bds_v5": 9.888000, + "Standard_E96bs_v5": 8.880000, + "Standard_E96d_v5": 8.448000, + "Standard_E96ds_v5": 8.448000, + "Standard_E96iads_v5": 8.496000, + "Standard_E96ias_v4": 7.445000, + "Standard_E96ias_v5": 7.392000, + "Standard_E96s_v5": 7.445000, + "Standard_F1": 0.059700, + "Standard_F16": 0.955000, + "Standard_F16s": 0.955000, + "Standard_F16s_v2": 0.845000, + "Standard_F1s": 0.059700, + "Standard_F2": 0.119000, + "Standard_F2s": 0.119000, + "Standard_F2s_v2": 0.106000, + "Standard_F32s_v2": 1.688000, + "Standard_F4": 0.239000, + "Standard_F48s_v2": 2.534000, + "Standard_F4s": 0.239000, + "Standard_F4s_v2": 0.211000, + "Standard_F64s_v2": 3.376000, + "Standard_F72s_v2": 3.802000, + "Standard_F8": 0.478000, + "Standard_F8s": 0.478000, + "Standard_F8s_v2": 0.422000, + "Standard_HB120-16rs_v2": 5.148000, + "Standard_HB120-32rs_v2": 5.148000, + "Standard_HB120-64rs_v2": 5.148000, + "Standard_HB120-96rs_v2": 5.148000, + "Standard_HB120rs_v2": 5.148000, + "Standard_L16as_v3": 1.512000, + "Standard_L16s_v3": 1.688000, + "Standard_L32as_v3": 3.024000, + "Standard_L32s_v3": 3.376000, + "Standard_L48as_v3": 4.536000, + "Standard_L48s_v3": 5.064000, + "Standard_L64as_v3": 6.048000, + "Standard_L64s_v3": 6.752000, + "Standard_L80as_v3": 7.560000, + "Standard_L80s_v3": 8.440000, + "Standard_L8as_v3": 0.756000, + "Standard_L8s_v3": 0.844000, + "Standard_M128": 17.610000, + "Standard_M128-32ms": 35.230000, + "Standard_M128-64ms": 35.230000, + "Standard_M128dms_v2": 35.230000, + "Standard_M128ds_v2": 17.610000, + "Standard_M128m": 35.230000, + "Standard_M128ms": 35.230000, + "Standard_M128ms_v2": 34.680000, + "Standard_M128s": 17.610000, + "Standard_M128s_v2": 17.050000, + "Standard_M16-4ms": 4.060000, + "Standard_M16-8ms": 4.060000, + "Standard_M16ms": 4.060000, + "Standard_M16s": 3.150000, + "Standard_M192idms_v2": 42.320000, + "Standard_M192ids_v2": 21.160000, + "Standard_M192ims_v2": 41.770000, + "Standard_M192is_v2": 20.610000, + "Standard_M32-16ms": 8.110000, + "Standard_M32-8ms": 8.110000, + "Standard_M32dms_v2": 8.110000, + "Standard_M32ls": 3.540000, + "Standard_M32ms": 8.110000, + "Standard_M32ms_v2": 7.970000, + "Standard_M32s": 4.400000, + "Standard_M32ts": 3.330000, + "Standard_M64": 8.800000, + "Standard_M64-16ms": 13.650000, + "Standard_M64-32ms": 13.650000, + "Standard_M64dms_v2": 13.650000, + "Standard_M64ds_v2": 8.800000, + "Standard_M64ls": 6.660000, + "Standard_M64m": 13.650000, + "Standard_M64ms": 13.650000, + "Standard_M64ms_v2": 13.370000, + "Standard_M64s": 8.800000, + "Standard_M64s_v2": 8.530000, + "Standard_M8-2ms": 2.030000, + "Standard_M8-4ms": 2.030000, + "Standard_M8ms": 2.030000, + "Standard_NC12s_v3": 8.411000, + "Standard_NC24rs_v3": 18.503000, + "Standard_NC24s_v3": 16.821000, + "Standard_NC6s_v3": 4.205000, + "Standard_NV12ads_A10_v5": 1.298000, + "Standard_NV18ads_A10_v5": 2.288000, + "Standard_NV36adms_A10_v5": 6.464000, + "Standard_NV36ads_A10_v5": 4.576000, + "Standard_NV6ads_A10_v5": 0.649000, + "Standard_NV72ads_A10_v5": 9.324000, + } + // southafricanorth + initialOnDemandPrices["southafricanorth"] = map[string]float64{ + "Dadsv5_Type1": 8.470000, + "Dasv4_Type1": 6.985000, + "Dasv4_Type2": 6.985000, + "Dasv5_Type1": 7.053000, + "Ddsv4_Type 1": 5.280000, + "Ddsv4_Type2": 6.270000, + "Ddsv5_Type1": 7.920000, + "Dsv3_Type3": 5.662000, + "Dsv3_Type4": 7.077000, + "Dsv4_Type1": 5.588000, + "Dsv4_Type2": 6.706000, + "Dsv5_Type1": 6.985000, + "Eadsv5_Type1": 9.134000, + "Easv4_Type1": 8.870000, + "Easv4_Type2": 8.870000, + "Easv5_Type1": 7.973000, + "Ebdsv5-Type1": 7.814000, + "Ebsv5-Type1": 7.040000, + "Edsv4_Type 1": 6.688000, + "Edsv4_Type2": 8.067000, + "Edsv5_Type1": 10.032000, + "Esv3_Type3": 5.758000, + "Esv3_Type4": 7.801000, + "Esv4_Type1": 5.913000, + "Esv4_Type2": 7.762000, + "Esv5_Type1": 8.870000, + "Fsv2 Type3": 5.267000, + "Fsv2_Type2": 4.510000, + "Fsv2_Type4": 6.019000, + "Lasv3_Type1": 9.020000, + "Lsv3_Type1": 10.076000, + "Mdmsv2MedMem _Type1": 39.341000, + "Mdsv2MedMem_Type1": 19.663000, + "Ms_Type1": 19.656000, + "Msm_Type1": 39.347000, + "Msmv2MedMem Type1": 38.721000, + "Msmv2_Type1": 109.060000, + "Msv2MedMem Type1": 19.043000, + "Msv2_Type1": 54.538000, + "NVsv3_Type1": 6.420000, + "Standard_A1_v2": 0.048200, + "Standard_A2_v2": 0.102000, + "Standard_A2m_v2": 0.133000, + "Standard_A4_v2": 0.213000, + "Standard_A4m_v2": 0.279000, + "Standard_A8_v2": 0.446000, + "Standard_A8m_v2": 0.586000, + "Standard_B12ms": 0.651000, + "Standard_B16als_v2": 0.765000, + "Standard_B16as_v2": 0.855000, + "Standard_B16ls_v2": 0.769000, + "Standard_B16ms": 0.868000, + "Standard_B16s_v2": 0.868000, + "Standard_B1ls": 0.006800, + "Standard_B1ms": 0.027100, + "Standard_B1s": 0.013600, + "Standard_B20ms": 1.085000, + "Standard_B2als_v2": 0.048800, + "Standard_B2as_v2": 0.097700, + "Standard_B2ats_v2": 0.021400, + "Standard_B2ls_v2": 0.063500, + "Standard_B2ms": 0.108000, + "Standard_B2s": 0.054200, + "Standard_B2s_v2": 0.109000, + "Standard_B2ts_v2": 0.013600, + "Standard_B32als_v2": 1.531000, + "Standard_B32as_v2": 1.710000, + "Standard_B32ls_v2": 1.684000, + "Standard_B32s_v2": 1.736000, + "Standard_B4als_v2": 0.191000, + "Standard_B4as_v2": 0.195000, + "Standard_B4ls_v2": 0.211000, + "Standard_B4ms": 0.217000, + "Standard_B4s_v2": 0.217000, + "Standard_B8als_v2": 0.383000, + "Standard_B8as_v2": 0.427000, + "Standard_B8ls_v2": 0.421000, + "Standard_B8ms": 0.434000, + "Standard_B8s_v2": 0.471000, + "Standard_D1": 0.089800, + "Standard_D11": 0.232000, + "Standard_D11_v2": 0.200000, + "Standard_D12": 0.465000, + "Standard_D12_v2": 0.401000, + "Standard_D13": 0.930000, + "Standard_D13_v2": 0.801000, + "Standard_D14": 1.859000, + "Standard_D14_v2": 1.603000, + "Standard_D15_v2": 2.003000, + "Standard_D15i_v2": 2.003000, + "Standard_D16_v3": 1.016000, + "Standard_D16_v4": 1.016000, + "Standard_D16_v5": 1.016000, + "Standard_D16a_v4": 1.016000, + "Standard_D16ads_v5": 1.100000, + "Standard_D16as_v4": 1.016000, + "Standard_D16as_v5": 1.652000, + "Standard_D16as_v5_Promo": 0.816000, + "Standard_D16d_v4": 1.200000, + "Standard_D16d_v5": 1.200000, + "Standard_D16ds_v4": 1.200000, + "Standard_D16ds_v5": 1.200000, + "Standard_D16lds_v5": 1.040000, + "Standard_D16ls_v5": 0.912000, + "Standard_D16s_v3": 1.016000, + "Standard_D16s_v4": 1.016000, + "Standard_D16s_v5": 1.016000, + "Standard_D1_v2": 0.076700, + "Standard_D2": 0.180000, + "Standard_D2_v2": 0.153000, + "Standard_D2_v3": 0.127000, + "Standard_D2_v4": 0.127000, + "Standard_D2_v5": 0.127000, + "Standard_D2a_v4": 0.127000, + "Standard_D2ads_v5": 0.230000, + "Standard_D2as_v4": 0.127000, + "Standard_D2as_v5": 0.207000, + "Standard_D2as_v5_Promo": 0.102000, + "Standard_D2d_v4": 0.150000, + "Standard_D2d_v5": 0.150000, + "Standard_D2ds_v4": 0.150000, + "Standard_D2ds_v5": 0.150000, + "Standard_D2lds_v5": 0.130000, + "Standard_D2ls_v5": 0.114000, + "Standard_D2s_v3": 0.127000, + "Standard_D2s_v4": 0.127000, + "Standard_D2s_v5": 0.127000, + "Standard_D3": 0.359000, + "Standard_D32-16s_v3": 2.032000, + "Standard_D32-8s_v3": 2.032000, + "Standard_D32_v3": 2.032000, + "Standard_D32_v4": 2.032000, + "Standard_D32_v5": 2.032000, + "Standard_D32a_v4": 2.032000, + "Standard_D32ads_v5": 2.200000, + "Standard_D32as_v4": 2.032000, + "Standard_D32as_v5": 3.304000, + "Standard_D32as_v5_Promo": 1.632000, + "Standard_D32d_v4": 2.400000, + "Standard_D32d_v5": 2.400000, + "Standard_D32ds_v4": 2.400000, + "Standard_D32ds_v5": 2.400000, + "Standard_D32lds_v5": 2.080000, + "Standard_D32ls_v5": 1.824000, + "Standard_D32s_v3": 2.032000, + "Standard_D32s_v4": 2.032000, + "Standard_D32s_v5": 2.032000, + "Standard_D3_v2": 0.307000, + "Standard_D4": 0.718000, + "Standard_D48_v3": 3.048000, + "Standard_D48_v4": 3.048000, + "Standard_D48_v5": 3.048000, + "Standard_D48a_v4": 3.048000, + "Standard_D48ads_v5": 3.300000, + "Standard_D48as_v4": 3.048000, + "Standard_D48as_v5": 4.956000, + "Standard_D48as_v5_Promo": 2.448000, + "Standard_D48d_v4": 3.600000, + "Standard_D48d_v5": 3.600000, + "Standard_D48ds_v4": 3.600000, + "Standard_D48ds_v5": 3.600000, + "Standard_D48lds_v5": 3.120000, + "Standard_D48ls_v5": 2.736000, + "Standard_D48s_v3": 3.048000, + "Standard_D48s_v4": 3.048000, + "Standard_D48s_v5": 3.048000, + "Standard_D4_v2": 0.614000, + "Standard_D4_v3": 0.254000, + "Standard_D4_v4": 0.254000, + "Standard_D4_v5": 0.254000, + "Standard_D4a_v4": 0.254000, + "Standard_D4ads_v5": 0.459000, + "Standard_D4as_v4": 0.254000, + "Standard_D4as_v5": 0.413000, + "Standard_D4as_v5_Promo": 0.204000, + "Standard_D4d_v4": 0.300000, + "Standard_D4d_v5": 0.300000, + "Standard_D4ds_v4": 0.300000, + "Standard_D4ds_v5": 0.300000, + "Standard_D4lds_v5": 0.260000, + "Standard_D4ls_v5": 0.228000, + "Standard_D4s_v3": 0.254000, + "Standard_D4s_v4": 0.254000, + "Standard_D4s_v5": 0.254000, + "Standard_D5_v2": 1.227000, + "Standard_D64-16s_v3": 4.064000, + "Standard_D64-32s_v3": 4.064000, + "Standard_D64_v3": 4.064000, + "Standard_D64_v4": 4.064000, + "Standard_D64_v5": 4.064000, + "Standard_D64a_v4": 4.064000, + "Standard_D64ads_v5": 4.400000, + "Standard_D64as_v4": 4.064000, + "Standard_D64as_v5": 6.608000, + "Standard_D64as_v5_Promo": 3.264000, + "Standard_D64d_v4": 4.800000, + "Standard_D64d_v5": 4.800000, + "Standard_D64ds_v4": 4.800000, + "Standard_D64ds_v5": 4.800000, + "Standard_D64lds_v5": 4.160000, + "Standard_D64ls_v5": 3.648000, + "Standard_D64s_v3": 4.064000, + "Standard_D64s_v4": 4.064000, + "Standard_D64s_v5": 4.064000, + "Standard_D8_v3": 0.508000, + "Standard_D8_v4": 0.508000, + "Standard_D8_v5": 0.508000, + "Standard_D8a_v4": 0.508000, + "Standard_D8ads_v5": 0.550000, + "Standard_D8as_v4": 0.508000, + "Standard_D8as_v5": 0.458000, + "Standard_D8as_v5_Promo": 0.408000, + "Standard_D8d_v4": 0.600000, + "Standard_D8d_v5": 0.600000, + "Standard_D8ds_v4": 0.600000, + "Standard_D8ds_v5": 0.600000, + "Standard_D8lds_v5": 0.520000, + "Standard_D8ls_v5": 0.456000, + "Standard_D8s_v3": 0.508000, + "Standard_D8s_v4": 0.508000, + "Standard_D8s_v5": 0.508000, + "Standard_D96_v5": 6.096000, + "Standard_D96a_v4": 6.096000, + "Standard_D96ads_v5": 6.600000, + "Standard_D96as_v4": 6.096000, + "Standard_D96as_v5": 9.912000, + "Standard_D96as_v5_Promo": 4.896000, + "Standard_D96d_v5": 7.200000, + "Standard_D96ds_v5": 7.200000, + "Standard_D96lds_v5": 6.240000, + "Standard_D96ls_v5": 5.472000, + "Standard_D96s_v5": 6.096000, + "Standard_DS1": 0.089800, + "Standard_DS11": 0.232000, + "Standard_DS11-1_v2": 0.200000, + "Standard_DS11_v2": 0.200000, + "Standard_DS11_v2_Promo": 0.000000, + "Standard_DS12": 0.465000, + "Standard_DS12-1_v2": 0.401000, + "Standard_DS12-2_v2": 0.401000, + "Standard_DS12_v2": 0.401000, + "Standard_DS12_v2_Promo": 0.000000, + "Standard_DS13": 0.930000, + "Standard_DS13-2_v2": 0.801000, + "Standard_DS13-4_v2": 0.801000, + "Standard_DS13_v2": 0.801000, + "Standard_DS13_v2_Promo": 0.000000, + "Standard_DS14": 1.859000, + "Standard_DS14-4_v2": 1.603000, + "Standard_DS14-8_v2": 1.603000, + "Standard_DS14_v2": 1.603000, + "Standard_DS14_v2_Promo": 0.000000, + "Standard_DS15_v2": 2.003000, + "Standard_DS15i_v2": 2.003000, + "Standard_DS1_v2": 0.076700, + "Standard_DS2": 0.180000, + "Standard_DS2_v2": 0.153000, + "Standard_DS2_v2_Promo": 0.000000, + "Standard_DS3": 0.359000, + "Standard_DS3_v2": 0.307000, + "Standard_DS3_v2_Promo": 0.000000, + "Standard_DS4": 0.718000, + "Standard_DS4_v2": 0.614000, + "Standard_DS4_v2_Promo": 0.000000, + "Standard_DS5_v2": 1.227000, + "Standard_DS5_v2_Promo": 0.000000, + "Standard_E104i_v5": 9.610000, + "Standard_E104id_v5": 10.868000, + "Standard_E104ids_v5": 10.868000, + "Standard_E104is_v5": 9.610000, + "Standard_E112iads_v5": 10.657000, + "Standard_E112ias_v5": 9.302000, + "Standard_E112ibds_v5": 13.675000, + "Standard_E112ibs_v5": 12.320000, + "Standard_E16-4ads_v5": 1.384000, + "Standard_E16-4as_v4": 1.344000, + "Standard_E16-4as_v5": 1.208000, + "Standard_E16-4ds_v4": 1.520000, + "Standard_E16-4ds_v5": 1.520000, + "Standard_E16-4s_v3": 1.344000, + "Standard_E16-4s_v4": 1.344000, + "Standard_E16-4s_v5": 1.344000, + "Standard_E16-8ads_v5": 2.120000, + "Standard_E16-8as_v4": 1.344000, + "Standard_E16-8as_v5": 1.208000, + "Standard_E16-8ds_v4": 1.520000, + "Standard_E16-8ds_v5": 1.520000, + "Standard_E16-8s_v3": 1.344000, + "Standard_E16-8s_v4": 1.344000, + "Standard_E16-8s_v5": 1.344000, + "Standard_E16_v3": 1.344000, + "Standard_E16_v4": 1.344000, + "Standard_E16_v5": 1.344000, + "Standard_E16a_v4": 1.344000, + "Standard_E16ads_v5": 1.384000, + "Standard_E16as_v4": 1.344000, + "Standard_E16as_v5": 1.208000, + "Standard_E16bds_v5": 1.776000, + "Standard_E16bs_v5": 1.600000, + "Standard_E16d_v4": 1.520000, + "Standard_E16d_v5": 1.520000, + "Standard_E16ds_v4": 1.520000, + "Standard_E16ds_v5": 1.520000, + "Standard_E16s_v3": 1.344000, + "Standard_E16s_v4": 1.344000, + "Standard_E16s_v5": 1.344000, + "Standard_E20_v3": 1.680000, + "Standard_E20_v4": 1.680000, + "Standard_E20_v5": 1.680000, + "Standard_E20a_v4": 1.680000, + "Standard_E20ads_v5": 1.730000, + "Standard_E20as_v4": 1.680000, + "Standard_E20as_v5": 1.556000, + "Standard_E20d_v4": 1.900000, + "Standard_E20d_v5": 1.900000, + "Standard_E20ds_v4": 1.900000, + "Standard_E20ds_v5": 1.900000, + "Standard_E20s_v3": 1.680000, + "Standard_E20s_v4": 1.680000, + "Standard_E20s_v5": 1.680000, + "Standard_E2_v3": 0.168000, + "Standard_E2_v4": 0.168000, + "Standard_E2_v5": 0.168000, + "Standard_E2a_v4": 0.168000, + "Standard_E2ads_v5": 0.173000, + "Standard_E2as_v4": 0.168000, + "Standard_E2as_v5": 0.151000, + "Standard_E2bds_v5": 0.222000, + "Standard_E2bs_v5": 0.200000, + "Standard_E2d_v4": 0.190000, + "Standard_E2d_v5": 0.190000, + "Standard_E2ds_v4": 0.190000, + "Standard_E2ds_v5": 0.190000, + "Standard_E2s_v3": 0.168000, + "Standard_E2s_v4": 0.168000, + "Standard_E2s_v5": 0.168000, + "Standard_E32-16ads_v5": 4.240000, + "Standard_E32-16as_v4": 2.688000, + "Standard_E32-16as_v5": 3.888000, + "Standard_E32-16ds_v4": 3.040000, + "Standard_E32-16ds_v5": 3.040000, + "Standard_E32-16s_v3": 2.688000, + "Standard_E32-16s_v4": 2.688000, + "Standard_E32-16s_v5": 2.688000, + "Standard_E32-8ads_v5": 4.240000, + "Standard_E32-8as_v4": 2.688000, + "Standard_E32-8as_v5": 3.888000, + "Standard_E32-8ds_v4": 3.040000, + "Standard_E32-8ds_v5": 3.040000, + "Standard_E32-8s_v3": 2.688000, + "Standard_E32-8s_v4": 2.688000, + "Standard_E32-8s_v5": 2.688000, + "Standard_E32_v3": 2.688000, + "Standard_E32_v4": 2.688000, + "Standard_E32_v5": 2.688000, + "Standard_E32a_v4": 2.688000, + "Standard_E32ads_v5": 2.768000, + "Standard_E32as_v4": 2.688000, + "Standard_E32as_v5": 2.462000, + "Standard_E32bds_v5": 3.552000, + "Standard_E32bs_v5": 3.200000, + "Standard_E32d_v4": 3.040000, + "Standard_E32d_v5": 3.040000, + "Standard_E32ds_v4": 3.040000, + "Standard_E32ds_v5": 3.040000, + "Standard_E32s_v3": 2.688000, + "Standard_E32s_v4": 2.688000, + "Standard_E32s_v5": 2.688000, + "Standard_E4-2ads_v5": 0.346000, + "Standard_E4-2as_v4": 0.336000, + "Standard_E4-2as_v5": 0.486000, + "Standard_E4-2ds_v4": 0.380000, + "Standard_E4-2ds_v5": 0.380000, + "Standard_E4-2s_v3": 0.336000, + "Standard_E4-2s_v4": 0.336000, + "Standard_E4-2s_v5": 0.336000, + "Standard_E48_v3": 4.032000, + "Standard_E48_v4": 4.032000, + "Standard_E48_v5": 4.032000, + "Standard_E48a_v4": 4.032000, + "Standard_E48ads_v5": 4.152000, + "Standard_E48as_v4": 4.032000, + "Standard_E48as_v5": 3.670000, + "Standard_E48bds_v5": 5.328000, + "Standard_E48bs_v5": 4.800000, + "Standard_E48d_v4": 4.560000, + "Standard_E48d_v5": 4.560000, + "Standard_E48ds_v4": 4.560000, + "Standard_E48ds_v5": 4.560000, + "Standard_E48s_v3": 4.032000, + "Standard_E48s_v4": 4.032000, + "Standard_E48s_v5": 4.032000, + "Standard_E4_v3": 0.336000, + "Standard_E4_v4": 0.336000, + "Standard_E4_v5": 0.336000, + "Standard_E4a_v4": 0.336000, + "Standard_E4ads_v5": 0.346000, + "Standard_E4as_v4": 0.336000, + "Standard_E4as_v5": 0.302000, + "Standard_E4bds_v5": 0.444000, + "Standard_E4bs_v5": 0.400000, + "Standard_E4d_v4": 0.380000, + "Standard_E4d_v5": 0.380000, + "Standard_E4ds_v4": 0.380000, + "Standard_E4ds_v5": 0.380000, + "Standard_E4s_v3": 0.336000, + "Standard_E4s_v4": 0.336000, + "Standard_E4s_v5": 0.336000, + "Standard_E64-16ads_v5": 8.480000, + "Standard_E64-16as_v4": 5.376000, + "Standard_E64-16as_v5": 7.776000, + "Standard_E64-16ds_v4": 6.080000, + "Standard_E64-16ds_v5": 6.080000, + "Standard_E64-16s_v3": 4.863000, + "Standard_E64-16s_v4": 5.376000, + "Standard_E64-16s_v5": 5.376000, + "Standard_E64-32ads_v5": 8.480000, + "Standard_E64-32as_v4": 5.376000, + "Standard_E64-32as_v5": 4.832000, + "Standard_E64-32ds_v4": 6.080000, + "Standard_E64-32ds_v5": 6.080000, + "Standard_E64-32s_v3": 4.863000, + "Standard_E64-32s_v4": 5.376000, + "Standard_E64-32s_v5": 5.376000, + "Standard_E64_v3": 4.863000, + "Standard_E64_v4": 5.376000, + "Standard_E64_v5": 5.376000, + "Standard_E64a_v4": 5.376000, + "Standard_E64ads_v5": 5.536000, + "Standard_E64as_v4": 5.376000, + "Standard_E64as_v5": 4.832000, + "Standard_E64bds_v5": 7.104000, + "Standard_E64bs_v5": 6.400000, + "Standard_E64d_v4": 6.080000, + "Standard_E64d_v5": 6.080000, + "Standard_E64ds_v4": 6.080000, + "Standard_E64ds_v5": 6.080000, + "Standard_E64i_v3": 4.863000, + "Standard_E64is_v3": 4.863000, + "Standard_E64s_v3": 4.863000, + "Standard_E64s_v4": 5.376000, + "Standard_E64s_v5": 5.376000, + "Standard_E8-2ads_v5": 1.060000, + "Standard_E8-2as_v4": 0.672000, + "Standard_E8-2as_v5": 0.604000, + "Standard_E8-2ds_v4": 0.760000, + "Standard_E8-2ds_v5": 0.760000, + "Standard_E8-2s_v3": 0.672000, + "Standard_E8-2s_v4": 0.672000, + "Standard_E8-2s_v5": 0.672000, + "Standard_E8-4ads_v5": 1.060000, + "Standard_E8-4as_v4": 0.672000, + "Standard_E8-4as_v5": 0.972000, + "Standard_E8-4ds_v4": 0.760000, + "Standard_E8-4ds_v5": 0.760000, + "Standard_E8-4s_v3": 0.672000, + "Standard_E8-4s_v4": 0.672000, + "Standard_E8-4s_v5": 0.672000, + "Standard_E80ids_v4": 7.600000, + "Standard_E80is_v4": 6.720000, + "Standard_E8_v3": 0.672000, + "Standard_E8_v4": 0.672000, + "Standard_E8_v5": 0.672000, + "Standard_E8a_v4": 0.672000, + "Standard_E8ads_v5": 0.692000, + "Standard_E8as_v4": 0.672000, + "Standard_E8as_v5": 0.650000, + "Standard_E8bds_v5": 0.888000, + "Standard_E8bs_v5": 0.800000, + "Standard_E8d_v4": 0.760000, + "Standard_E8d_v5": 0.760000, + "Standard_E8ds_v4": 0.760000, + "Standard_E8ds_v5": 0.760000, + "Standard_E8s_v3": 0.672000, + "Standard_E8s_v4": 0.672000, + "Standard_E8s_v5": 0.672000, + "Standard_E96-24ads_v5": 8.304000, + "Standard_E96-24as_v4": 8.064000, + "Standard_E96-24as_v5": 11.664000, + "Standard_E96-24ds_v5": 9.120000, + "Standard_E96-24s_v5": 8.064000, + "Standard_E96-48ads_v5": 12.720000, + "Standard_E96-48as_v4": 8.064000, + "Standard_E96-48as_v5": 7.248000, + "Standard_E96-48ds_v5": 9.120000, + "Standard_E96-48s_v5": 8.064000, + "Standard_E96_v5": 8.064000, + "Standard_E96a_v4": 8.064000, + "Standard_E96ads_v5": 8.350000, + "Standard_E96as_v4": 8.064000, + "Standard_E96as_v5": 7.248000, + "Standard_E96bds_v5": 10.656000, + "Standard_E96bs_v5": 9.600000, + "Standard_E96d_v5": 9.120000, + "Standard_E96ds_v5": 9.120000, + "Standard_E96iads_v5": 9.134000, + "Standard_E96ias_v4": 8.064000, + "Standard_E96ias_v5": 7.973000, + "Standard_E96s_v5": 8.064000, + "Standard_F1": 0.066700, + "Standard_F16": 1.066000, + "Standard_F16s": 1.066000, + "Standard_F16s_v2": 0.911000, + "Standard_F1s": 0.066700, + "Standard_F2": 0.133000, + "Standard_F2s": 0.133000, + "Standard_F2s_v2": 0.114000, + "Standard_F32s_v2": 1.822000, + "Standard_F4": 0.267000, + "Standard_F48s_v2": 2.734000, + "Standard_F4s": 0.267000, + "Standard_F4s_v2": 0.228000, + "Standard_F64s_v2": 3.645000, + "Standard_F72s_v2": 4.100000, + "Standard_F8": 0.533000, + "Standard_F8s": 0.533000, + "Standard_F8s_v2": 0.456000, + "Standard_L16as_v3": 1.640000, + "Standard_L16s_v3": 1.832000, + "Standard_L32as_v3": 3.280000, + "Standard_L32s_v3": 3.664000, + "Standard_L48as_v3": 4.920000, + "Standard_L48s_v3": 5.496000, + "Standard_L64as_v3": 6.560000, + "Standard_L64s_v3": 7.328000, + "Standard_L80as_v3": 8.200000, + "Standard_L80s_v3": 9.160000, + "Standard_L8as_v3": 0.820000, + "Standard_L8s_v3": 0.916000, + "Standard_M128": 17.873000, + "Standard_M128-32ms": 35.762000, + "Standard_M128-64ms": 35.762000, + "Standard_M128dms_v2": 35.767000, + "Standard_M128ds_v2": 17.877000, + "Standard_M128m": 35.762000, + "Standard_M128ms": 35.762000, + "Standard_M128ms_v2": 35.203000, + "Standard_M128s": 17.873000, + "Standard_M128s_v2": 17.313000, + "Standard_M16-4ms": 4.118000, + "Standard_M16-8ms": 4.118000, + "Standard_M16ms": 4.118000, + "Standard_M16s": 3.199000, + "Standard_M192idms_v2": 42.969000, + "Standard_M192ids_v2": 21.484000, + "Standard_M192ims_v2": 42.405000, + "Standard_M192is_v2": 20.920000, + "Standard_M208-104ms_v2": 59.791000, + "Standard_M208-104s_v2": 29.895000, + "Standard_M208-52ms_v2": 59.791000, + "Standard_M208-52s_v2": 29.895000, + "Standard_M208ms_v2": 59.791000, + "Standard_M208s_v2": 29.895000, + "Standard_M32-16ms": 8.236000, + "Standard_M32-8ms": 8.236000, + "Standard_M32dms_v2": 8.236000, + "Standard_M32ls": 3.829709, + "Standard_M32ms": 8.236000, + "Standard_M32ms_v2": 8.095000, + "Standard_M32s": 4.469000, + "Standard_M32ts": 3.608431, + "Standard_M416-104ms_v2": 132.860000, + "Standard_M416-104s_v2": 66.437000, + "Standard_M416-208ms_v2": 132.860000, + "Standard_M416-208s_v2": 66.437000, + "Standard_M416is_v2": 66.437000, + "Standard_M416ms_v2": 132.860000, + "Standard_M416s_8_v2": 96.520000, + "Standard_M416s_v2": 66.437000, + "Standard_M64": 8.936000, + "Standard_M64-16ms": 13.852000, + "Standard_M64-32ms": 13.852000, + "Standard_M64dms_v2": 13.857000, + "Standard_M64ds_v2": 8.937000, + "Standard_M64ls": 7.218195, + "Standard_M64m": 13.852000, + "Standard_M64ms": 13.852000, + "Standard_M64ms_v2": 13.575000, + "Standard_M64s": 8.936000, + "Standard_M64s_v2": 8.655000, + "Standard_M8-2ms": 2.059000, + "Standard_M8-4ms": 2.059000, + "Standard_M8ms": 2.059000, + "Standard_ND96ams_A100_v4": 47.844000, + "Standard_ND96amsr_A100_v4": 47.844000, + "Standard_NV12ads_A10_v5": 1.326000, + "Standard_NV12s_v3": 1.460000, + "Standard_NV18ads_A10_v5": 2.336000, + "Standard_NV24s_v3": 2.921000, + "Standard_NV36adms_A10_v5": 6.599000, + "Standard_NV36ads_A10_v5": 4.672000, + "Standard_NV48s_v3": 5.841000, + "Standard_NV6ads_A10_v5": 0.663000, + "Standard_NV72ads_A10_v5": 9.519000, + } + // southafricawest + initialOnDemandPrices["southafricawest"] = map[string]float64{ + "Dadsv5_Type1": 11.011000, + "Dasv4_Type1": 9.080000, + "Dasv4_Type2": 9.080000, + "Dasv5_Type1": 9.169000, + "Ddsv5_Type1": 10.296000, + "Dsv3_Type3": 7.098000, + "Dsv3_Type4": 8.872000, + "Dsv5_Type1": 9.081000, + "Eadsv5_Type1": 11.875000, + "Easv4_Type1": 11.532000, + "Easv4_Type2": 11.532000, + "Easv5_Type1": 10.365000, + "Ebdsv5-Type1": 10.159000, + "Ebsv5-Type1": 9.152000, + "Edsv5_Type1": 13.042000, + "Esv3_Type3": 7.219000, + "Esv3_Type4": 9.781000, + "Esv5_Type1": 11.532000, + "Fsv2_Type2": 5.655000, + "Lasv3_Type1": 11.726000, + "Lsv3_Type1": 13.099000, + "Standard_A1_v2": 0.060500, + "Standard_A2_v2": 0.128000, + "Standard_A2m_v2": 0.166000, + "Standard_A4_v2": 0.267000, + "Standard_A4m_v2": 0.349000, + "Standard_A8_v2": 0.559000, + "Standard_A8m_v2": 0.734000, + "Standard_B12ms": 0.813750, + "Standard_B16als_v2": 0.973000, + "Standard_B16as_v2": 1.089000, + "Standard_B16ls_v2": 1.073000, + "Standard_B16ms": 1.085000, + "Standard_B16s_v2": 1.202000, + "Standard_B1ls": 0.008500, + "Standard_B1ms": 0.033875, + "Standard_B1s": 0.017000, + "Standard_B20ms": 1.356250, + "Standard_B2als_v2": 0.063500, + "Standard_B2as_v2": 0.136000, + "Standard_B2ats_v2": 0.025100, + "Standard_B2ls_v2": 0.079700, + "Standard_B2ms": 0.135625, + "Standard_B2s": 0.067750, + "Standard_B2s_v2": 0.141000, + "Standard_B2ts_v2": 0.017600, + "Standard_B32als_v2": 1.946000, + "Standard_B32as_v2": 2.178000, + "Standard_B32ls_v2": 1.998000, + "Standard_B32s_v2": 2.257000, + "Standard_B4als_v2": 0.243000, + "Standard_B4as_v2": 0.254000, + "Standard_B4ls_v2": 0.250000, + "Standard_B4ms": 0.271250, + "Standard_B4s_v2": 0.282000, + "Standard_B8als_v2": 0.487000, + "Standard_B8as_v2": 0.545000, + "Standard_B8ls_v2": 0.536000, + "Standard_B8ms": 0.542375, + "Standard_B8s_v2": 0.601000, + "Standard_D1": 0.113000, + "Standard_D11": 0.291000, + "Standard_D11_v2": 0.251000, + "Standard_D12": 0.583000, + "Standard_D12_v2": 0.502000, + "Standard_D13": 1.165000, + "Standard_D13_v2": 1.005000, + "Standard_D14": 2.331000, + "Standard_D14_v2": 2.009000, + "Standard_D15_v2": 2.512000, + "Standard_D15i_v2": 2.512000, + "Standard_D16_v3": 1.270000, + "Standard_D16_v4": 1.344000, + "Standard_D16_v5": 1.321000, + "Standard_D16a_v4": 1.321000, + "Standard_D16ads_v5": 2.166000, + "Standard_D16as_v4": 1.321000, + "Standard_D16as_v5": 1.191000, + "Standard_D16d_v4": 1.582000, + "Standard_D16d_v5": 1.560000, + "Standard_D16ds_v4": 1.582000, + "Standard_D16ds_v5": 1.560000, + "Standard_D16lds_v5": 1.352000, + "Standard_D16ls_v5": 1.186000, + "Standard_D16s_v3": 1.270000, + "Standard_D16s_v4": 1.344000, + "Standard_D16s_v5": 1.321000, + "Standard_D1_v2": 0.096200, + "Standard_D2": 0.225000, + "Standard_D2_v2": 0.192000, + "Standard_D2_v3": 0.158750, + "Standard_D2_v4": 0.168000, + "Standard_D2_v5": 0.165000, + "Standard_D2a_v4": 0.165000, + "Standard_D2ads_v5": 0.179000, + "Standard_D2as_v4": 0.165000, + "Standard_D2as_v5": 0.241000, + "Standard_D2d_v4": 0.198000, + "Standard_D2d_v5": 0.195000, + "Standard_D2ds_v4": 0.198000, + "Standard_D2ds_v5": 0.195000, + "Standard_D2lds_v5": 0.169000, + "Standard_D2ls_v5": 0.148000, + "Standard_D2s_v3": 0.158750, + "Standard_D2s_v4": 0.168000, + "Standard_D2s_v5": 0.165000, + "Standard_D3": 0.450000, + "Standard_D32-16s_v3": 2.540000, + "Standard_D32-8s_v3": 2.540000, + "Standard_D32_v3": 2.540000, + "Standard_D32_v4": 2.688000, + "Standard_D32_v5": 2.642000, + "Standard_D32a_v4": 2.642000, + "Standard_D32ads_v5": 4.332000, + "Standard_D32as_v4": 2.642000, + "Standard_D32as_v5": 3.854000, + "Standard_D32d_v4": 3.164000, + "Standard_D32d_v5": 3.120000, + "Standard_D32ds_v4": 3.164000, + "Standard_D32ds_v5": 3.120000, + "Standard_D32lds_v5": 2.704000, + "Standard_D32ls_v5": 2.371000, + "Standard_D32s_v3": 2.540000, + "Standard_D32s_v4": 2.688000, + "Standard_D32s_v5": 2.642000, + "Standard_D3_v2": 0.385000, + "Standard_D4": 0.900000, + "Standard_D48_v3": 3.810000, + "Standard_D48_v4": 4.032000, + "Standard_D48_v5": 3.962000, + "Standard_D48a_v4": 3.962000, + "Standard_D48ads_v5": 6.498000, + "Standard_D48as_v4": 3.962000, + "Standard_D48as_v5": 5.780000, + "Standard_D48d_v4": 4.746000, + "Standard_D48d_v5": 4.680000, + "Standard_D48ds_v4": 4.746000, + "Standard_D48ds_v5": 4.680000, + "Standard_D48lds_v5": 4.056000, + "Standard_D48ls_v5": 3.557000, + "Standard_D48s_v3": 3.810000, + "Standard_D48s_v4": 4.032000, + "Standard_D48s_v5": 3.962000, + "Standard_D4_v2": 0.769000, + "Standard_D4_v3": 0.317500, + "Standard_D4_v4": 0.336000, + "Standard_D4_v5": 0.330000, + "Standard_D4a_v4": 0.330000, + "Standard_D4ads_v5": 0.542000, + "Standard_D4as_v4": 0.330000, + "Standard_D4as_v5": 0.298000, + "Standard_D4d_v4": 0.396000, + "Standard_D4d_v5": 0.390000, + "Standard_D4ds_v4": 0.396000, + "Standard_D4ds_v5": 0.390000, + "Standard_D4lds_v5": 0.338000, + "Standard_D4ls_v5": 0.296000, + "Standard_D4s_v3": 0.317500, + "Standard_D4s_v4": 0.336000, + "Standard_D4s_v5": 0.330000, + "Standard_D5_v2": 1.539000, + "Standard_D64-16s_v3": 5.080000, + "Standard_D64-32s_v3": 5.080000, + "Standard_D64_v3": 5.080000, + "Standard_D64_v4": 5.376000, + "Standard_D64_v5": 5.283000, + "Standard_D64a_v4": 5.283000, + "Standard_D64ads_v5": 5.720000, + "Standard_D64as_v4": 5.283000, + "Standard_D64as_v5": 4.763000, + "Standard_D64d_v4": 6.328000, + "Standard_D64d_v5": 6.240000, + "Standard_D64ds_v4": 6.328000, + "Standard_D64ds_v5": 6.240000, + "Standard_D64lds_v5": 5.408000, + "Standard_D64ls_v5": 4.742000, + "Standard_D64s_v3": 5.080000, + "Standard_D64s_v4": 5.376000, + "Standard_D64s_v5": 5.283000, + "Standard_D8_v3": 0.635000, + "Standard_D8_v4": 0.672000, + "Standard_D8_v5": 0.660000, + "Standard_D8a_v4": 0.660000, + "Standard_D8ads_v5": 1.083000, + "Standard_D8as_v4": 0.660000, + "Standard_D8as_v5": 0.963000, + "Standard_D8d_v4": 0.791000, + "Standard_D8d_v5": 0.780000, + "Standard_D8ds_v4": 0.791000, + "Standard_D8ds_v5": 0.780000, + "Standard_D8lds_v5": 0.676000, + "Standard_D8ls_v5": 0.593000, + "Standard_D8s_v3": 0.635000, + "Standard_D8s_v4": 0.672000, + "Standard_D8s_v5": 0.660000, + "Standard_D96_v5": 7.925000, + "Standard_D96a_v4": 7.925000, + "Standard_D96ads_v5": 8.580000, + "Standard_D96as_v4": 7.925000, + "Standard_D96as_v5": 11.561000, + "Standard_D96d_v5": 9.360000, + "Standard_D96ds_v5": 9.360000, + "Standard_D96lds_v5": 8.112000, + "Standard_D96ls_v5": 7.114000, + "Standard_D96s_v5": 7.925000, + "Standard_DS1": 0.113000, + "Standard_DS11": 0.291000, + "Standard_DS11-1_v2": 0.251000, + "Standard_DS11_v2": 0.251000, + "Standard_DS11_v2_Promo": 0.000000, + "Standard_DS12": 0.583000, + "Standard_DS12-1_v2": 0.502000, + "Standard_DS12-2_v2": 0.502000, + "Standard_DS12_v2": 0.502000, + "Standard_DS12_v2_Promo": 0.000000, + "Standard_DS13": 1.165000, + "Standard_DS13-2_v2": 1.005000, + "Standard_DS13-4_v2": 1.005000, + "Standard_DS13_v2": 1.005000, + "Standard_DS13_v2_Promo": 0.000000, + "Standard_DS14": 2.331000, + "Standard_DS14-4_v2": 2.009000, + "Standard_DS14-8_v2": 2.009000, + "Standard_DS14_v2": 2.009000, + "Standard_DS14_v2_Promo": 0.000000, + "Standard_DS15_v2": 2.512000, + "Standard_DS15i_v2": 2.512000, + "Standard_DS1_v2": 0.096200, + "Standard_DS2": 0.225000, + "Standard_DS2_v2": 0.192000, + "Standard_DS2_v2_Promo": 0.000000, + "Standard_DS3": 0.450000, + "Standard_DS3_v2": 0.385000, + "Standard_DS3_v2_Promo": 0.000000, + "Standard_DS4": 0.900000, + "Standard_DS4_v2": 0.769000, + "Standard_DS4_v2_Promo": 0.000000, + "Standard_DS5_v2": 1.539000, + "Standard_DS5_v2_Promo": 0.000000, + "Standard_E104i_v5": 12.492000, + "Standard_E104id_v5": 14.128000, + "Standard_E104ids_v5": 14.128000, + "Standard_E104is_v5": 12.492000, + "Standard_E112iads_v5": 13.854000, + "Standard_E112ias_v5": 12.092000, + "Standard_E112ibds_v5": 17.778000, + "Standard_E112ibs_v5": 16.016000, + "Standard_E16-4ads_v5": 2.535000, + "Standard_E16-4as_v4": 1.747000, + "Standard_E16-4as_v5": 1.570000, + "Standard_E16-4ds_v4": 1.935000, + "Standard_E16-4ds_v5": 1.976000, + "Standard_E16-4s_v3": 1.680000, + "Standard_E16-4s_v4": 1.693000, + "Standard_E16-4s_v5": 1.747000, + "Standard_E16-8ads_v5": 1.799000, + "Standard_E16-8as_v4": 1.747000, + "Standard_E16-8as_v5": 1.570000, + "Standard_E16-8ds_v4": 1.935000, + "Standard_E16-8ds_v5": 1.976000, + "Standard_E16-8s_v3": 1.680000, + "Standard_E16-8s_v4": 1.693000, + "Standard_E16-8s_v5": 1.747000, + "Standard_E16_v3": 1.680000, + "Standard_E16_v4": 1.693000, + "Standard_E16_v5": 1.747000, + "Standard_E16a_v4": 1.747000, + "Standard_E16ads_v5": 1.799000, + "Standard_E16as_v4": 1.747000, + "Standard_E16as_v5": 1.570000, + "Standard_E16bds_v5": 2.309000, + "Standard_E16bs_v5": 2.080000, + "Standard_E16d_v4": 1.935000, + "Standard_E16d_v5": 1.976000, + "Standard_E16ds_v4": 1.935000, + "Standard_E16ds_v5": 1.976000, + "Standard_E16s_v3": 1.680000, + "Standard_E16s_v4": 1.693000, + "Standard_E16s_v5": 1.747000, + "Standard_E20_v3": 2.100000, + "Standard_E20_v4": 2.117000, + "Standard_E20_v5": 2.184000, + "Standard_E20a_v4": 2.184000, + "Standard_E20ads_v5": 2.295000, + "Standard_E20as_v4": 2.184000, + "Standard_E20as_v5": 1.963000, + "Standard_E20d_v4": 2.419000, + "Standard_E20d_v5": 2.470000, + "Standard_E20ds_v4": 2.419000, + "Standard_E20ds_v5": 2.470000, + "Standard_E20s_v3": 2.100000, + "Standard_E20s_v4": 2.117000, + "Standard_E20s_v5": 2.184000, + "Standard_E2_v3": 0.210000, + "Standard_E2_v4": 0.212000, + "Standard_E2_v5": 0.218000, + "Standard_E2a_v4": 0.218000, + "Standard_E2ads_v5": 0.225000, + "Standard_E2as_v4": 0.218000, + "Standard_E2as_v5": 0.196000, + "Standard_E2bds_v5": 0.289000, + "Standard_E2bs_v5": 0.260000, + "Standard_E2d_v4": 0.242000, + "Standard_E2d_v5": 0.247000, + "Standard_E2ds_v4": 0.242000, + "Standard_E2ds_v5": 0.247000, + "Standard_E2s_v3": 0.210000, + "Standard_E2s_v4": 0.212000, + "Standard_E2s_v5": 0.218000, + "Standard_E32-16ads_v5": 3.598000, + "Standard_E32-16as_v4": 3.494000, + "Standard_E32-16as_v5": 3.141000, + "Standard_E32-16ds_v4": 3.871000, + "Standard_E32-16ds_v5": 3.952000, + "Standard_E32-16s_v3": 3.360000, + "Standard_E32-16s_v4": 3.387000, + "Standard_E32-16s_v5": 3.494000, + "Standard_E32-8ads_v5": 5.070000, + "Standard_E32-8as_v4": 3.494000, + "Standard_E32-8as_v5": 3.141000, + "Standard_E32-8ds_v4": 3.871000, + "Standard_E32-8ds_v5": 3.952000, + "Standard_E32-8s_v3": 3.360000, + "Standard_E32-8s_v4": 3.387000, + "Standard_E32-8s_v5": 3.494000, + "Standard_E32_v3": 3.360000, + "Standard_E32_v4": 3.387000, + "Standard_E32_v5": 3.494000, + "Standard_E32a_v4": 3.494000, + "Standard_E32ads_v5": 3.598000, + "Standard_E32as_v4": 3.494000, + "Standard_E32as_v5": 3.141000, + "Standard_E32bds_v5": 4.618000, + "Standard_E32bs_v5": 4.160000, + "Standard_E32d_v4": 3.871000, + "Standard_E32d_v5": 3.952000, + "Standard_E32ds_v4": 3.871000, + "Standard_E32ds_v5": 3.952000, + "Standard_E32s_v3": 3.360000, + "Standard_E32s_v4": 3.387000, + "Standard_E32s_v5": 3.494000, + "Standard_E4-2ads_v5": 0.634000, + "Standard_E4-2as_v4": 0.437000, + "Standard_E4-2as_v5": 0.393000, + "Standard_E4-2ds_v4": 0.484000, + "Standard_E4-2ds_v5": 0.494000, + "Standard_E4-2s_v3": 0.420000, + "Standard_E4-2s_v4": 0.423000, + "Standard_E4-2s_v5": 0.437000, + "Standard_E48_v3": 5.040000, + "Standard_E48_v4": 5.080000, + "Standard_E48_v5": 5.242000, + "Standard_E48a_v4": 5.242000, + "Standard_E48ads_v5": 5.444000, + "Standard_E48as_v4": 5.242000, + "Standard_E48as_v5": 4.711000, + "Standard_E48bds_v5": 6.926000, + "Standard_E48bs_v5": 6.240000, + "Standard_E48d_v4": 5.806000, + "Standard_E48d_v5": 5.928000, + "Standard_E48ds_v4": 5.806000, + "Standard_E48ds_v5": 5.928000, + "Standard_E48s_v3": 5.040000, + "Standard_E48s_v4": 5.080000, + "Standard_E48s_v5": 5.242000, + "Standard_E4_v3": 0.420000, + "Standard_E4_v4": 0.423000, + "Standard_E4_v5": 0.437000, + "Standard_E4a_v4": 0.437000, + "Standard_E4ads_v5": 0.496000, + "Standard_E4as_v4": 0.437000, + "Standard_E4as_v5": 0.439000, + "Standard_E4bds_v5": 0.577000, + "Standard_E4bs_v5": 0.520000, + "Standard_E4d_v4": 0.484000, + "Standard_E4d_v5": 0.494000, + "Standard_E4ds_v4": 0.484000, + "Standard_E4ds_v5": 0.494000, + "Standard_E4s_v3": 0.420000, + "Standard_E4s_v4": 0.423000, + "Standard_E4s_v5": 0.437000, + "Standard_E64-16ads_v5": 10.141000, + "Standard_E64-16as_v4": 6.989000, + "Standard_E64-16as_v5": 9.226000, + "Standard_E64-16ds_v4": 7.741000, + "Standard_E64-16ds_v5": 7.904000, + "Standard_E64-16s_v3": 6.096000, + "Standard_E64-16s_v4": 6.774000, + "Standard_E64-16s_v5": 6.989000, + "Standard_E64-32ads_v5": 7.197000, + "Standard_E64-32as_v4": 6.989000, + "Standard_E64-32as_v5": 9.226000, + "Standard_E64-32ds_v4": 7.741000, + "Standard_E64-32ds_v5": 7.904000, + "Standard_E64-32s_v3": 6.096000, + "Standard_E64-32s_v4": 6.774000, + "Standard_E64-32s_v5": 6.989000, + "Standard_E64_v3": 6.096000, + "Standard_E64_v4": 6.774000, + "Standard_E64_v5": 6.989000, + "Standard_E64a_v4": 6.989000, + "Standard_E64ads_v5": 7.243000, + "Standard_E64as_v4": 6.989000, + "Standard_E64as_v5": 6.328000, + "Standard_E64bds_v5": 9.235000, + "Standard_E64bs_v5": 8.320000, + "Standard_E64d_v4": 7.741000, + "Standard_E64d_v5": 7.904000, + "Standard_E64ds_v4": 7.741000, + "Standard_E64ds_v5": 7.904000, + "Standard_E64i_v3": 6.096000, + "Standard_E64is_v3": 6.096000, + "Standard_E64s_v3": 6.096000, + "Standard_E64s_v4": 6.774000, + "Standard_E64s_v5": 6.989000, + "Standard_E8-2ads_v5": 1.268000, + "Standard_E8-2as_v4": 0.874000, + "Standard_E8-2as_v5": 1.153000, + "Standard_E8-2ds_v4": 0.968000, + "Standard_E8-2ds_v5": 0.988000, + "Standard_E8-2s_v3": 0.840000, + "Standard_E8-2s_v4": 0.847000, + "Standard_E8-2s_v5": 0.874000, + "Standard_E8-4ads_v5": 0.900000, + "Standard_E8-4as_v4": 0.874000, + "Standard_E8-4as_v5": 1.153000, + "Standard_E8-4ds_v4": 0.968000, + "Standard_E8-4ds_v5": 0.988000, + "Standard_E8-4s_v3": 0.840000, + "Standard_E8-4s_v4": 0.847000, + "Standard_E8-4s_v5": 0.874000, + "Standard_E80ids_v4": 9.677000, + "Standard_E80is_v4": 8.467000, + "Standard_E8_v3": 0.840000, + "Standard_E8_v4": 0.847000, + "Standard_E8_v5": 0.874000, + "Standard_E8a_v4": 0.874000, + "Standard_E8ads_v5": 0.900000, + "Standard_E8as_v4": 0.874000, + "Standard_E8as_v5": 0.785000, + "Standard_E8bds_v5": 1.154000, + "Standard_E8bs_v5": 1.040000, + "Standard_E8d_v4": 0.968000, + "Standard_E8d_v5": 0.988000, + "Standard_E8ds_v4": 0.968000, + "Standard_E8ds_v5": 0.988000, + "Standard_E8s_v3": 0.840000, + "Standard_E8s_v4": 0.847000, + "Standard_E8s_v5": 0.874000, + "Standard_E96-24ads_v5": 10.795000, + "Standard_E96-24as_v4": 10.483000, + "Standard_E96-24as_v5": 9.422000, + "Standard_E96-24ds_v5": 11.856000, + "Standard_E96-24s_v5": 10.483000, + "Standard_E96-48ads_v5": 10.795000, + "Standard_E96-48as_v4": 10.483000, + "Standard_E96-48as_v5": 13.838000, + "Standard_E96-48ds_v5": 11.856000, + "Standard_E96-48s_v5": 10.483000, + "Standard_E96_v5": 10.483000, + "Standard_E96a_v4": 10.483000, + "Standard_E96ads_v5": 10.795000, + "Standard_E96as_v4": 10.483000, + "Standard_E96as_v5": 9.422000, + "Standard_E96bds_v5": 13.853000, + "Standard_E96bs_v5": 12.480000, + "Standard_E96d_v5": 11.856000, + "Standard_E96ds_v5": 11.856000, + "Standard_E96iads_v5": 11.875000, + "Standard_E96ias_v4": 10.483000, + "Standard_E96ias_v5": 10.365000, + "Standard_E96s_v5": 10.483000, + "Standard_F1": 0.083600, + "Standard_F16": 1.337000, + "Standard_F16s": 1.337000, + "Standard_F16s_v2": 1.140000, + "Standard_F1s": 0.083600, + "Standard_F2": 0.167000, + "Standard_F2s": 0.167000, + "Standard_F2s_v2": 0.142500, + "Standard_F32s_v2": 2.280000, + "Standard_F4": 0.334000, + "Standard_F48s_v2": 3.420000, + "Standard_F4s": 0.334000, + "Standard_F4s_v2": 0.285000, + "Standard_F64s_v2": 4.560000, + "Standard_F72s_v2": 5.130000, + "Standard_F8": 0.669000, + "Standard_F8s": 0.669000, + "Standard_F8s_v2": 0.570000, + "Standard_L16as_v3": 2.132000, + "Standard_L16s_v3": 2.382000, + "Standard_L32as_v3": 4.264000, + "Standard_L32s_v3": 4.763000, + "Standard_L48as_v3": 6.396000, + "Standard_L48s_v3": 7.145000, + "Standard_L64as_v3": 8.528000, + "Standard_L64s_v3": 9.526000, + "Standard_L80as_v3": 10.660000, + "Standard_L80s_v3": 11.908000, + "Standard_L8as_v3": 1.066000, + "Standard_L8s_v3": 1.191000, + "Standard_M128": 22.408000, + "Standard_M128-32ms": 44.836000, + "Standard_M128-64ms": 44.836000, + "Standard_M128m": 44.836000, + "Standard_M128ms": 44.836000, + "Standard_M128s": 22.408000, + "Standard_M16-4ms": 5.163000, + "Standard_M16-8ms": 5.163000, + "Standard_M16ms": 5.163000, + "Standard_M16s": 4.010000, + "Standard_M32-16ms": 10.325000, + "Standard_M32-8ms": 10.325000, + "Standard_M32ls": 4.827000, + "Standard_M32ms": 10.325000, + "Standard_M32s": 5.603000, + "Standard_M32ts": 4.548000, + "Standard_M64": 11.204000, + "Standard_M64-16ms": 17.367000, + "Standard_M64-32ms": 17.367000, + "Standard_M64ls": 9.097000, + "Standard_M64m": 17.367000, + "Standard_M64ms": 17.367000, + "Standard_M64s": 11.204000, + "Standard_M8-2ms": 2.581000, + "Standard_M8-4ms": 2.581000, + "Standard_M8ms": 2.581000, + "Standard_ND96ams_A100_v4": 62.197000, + "Standard_ND96amsr_A100_v4": 62.197000, + } + // southcentralus + initialOnDemandPrices["southcentralus"] = map[string]float64{ + "Basic_A0": 0.018000, + "Basic_A1": 0.025000, + "Basic_A2": 0.079000, + "Basic_A3": 0.176000, + "Basic_A4": 0.352000, + "DCdsv3 Type1": 7.160000, + "DCsv2 Type 1": 1.014000, + "DCsv3 Type1": 6.083000, + "Dadsv5_Type1": 7.614000, + "Dasv4_Type1": 6.083000, + "Dasv4_Type2": 6.083000, + "Dasv5_Type1": 6.357000, + "Ddsv4_Type 1": 4.773000, + "Ddsv4_Type2": 5.668000, + "Ddsv5_Type1": 7.160000, + "Dsv3_Type3": 5.069000, + "Dsv3_Type4": 6.336000, + "Dsv4_Type1": 5.069000, + "Dsv4_Type2": 6.336000, + "Dsv5_Type1": 6.336000, + "Eadsv5_Type1": 8.300000, + "Easv4_Type1": 7.983000, + "Easv4_Type2": 7.983000, + "Easv5_Type1": 7.160000, + "Ebdsv5-Type1": 7.054000, + "Ebsv5-Type1": 6.294000, + "Edsv4_Type 1": 6.083000, + "Edsv4_Type2": 7.223000, + "Edsv5_Type1": 9.124000, + "Esv3_Type3": 4.972000, + "Esv3_Type4": 6.736000, + "Esv4_Type1": 5.322000, + "Esv4_Type2": 6.985000, + "Esv5_Type1": 7.983000, + "FXmds Type1": 5.892000, + "Fsv2 Type3": 5.161000, + "Fsv2_Type2": 4.039000, + "Fsv2_Type4": 5.610000, + "Lsv2_Type1": 8.237000, + "Lsv3_Type1": 9.187000, + "Mdmsv2MedMem _Type1": 35.231000, + "Mdsv2MedMem_Type1": 17.609000, + "Ms_Type1": 17.603000, + "Msm_Type1": 35.236000, + "Msmv2MedMem Type1": 34.675000, + "Msmv2_Type1": 130.880000, + "Msv2MedMem Type1": 17.053000, + "Msv2_Type1": 65.446000, + "NDamsrA100v4_Type1": 43.256000, + "NDasrA100v4_Type1": 35.893000, + "NVasv4_Type1": 8.612000, + "NVsv3_Type1": 6.019000, + "Standard_A0": 0.020000, + "Standard_A1": 0.060000, + "Standard_A1_v2": 0.043000, + "Standard_A2": 0.120000, + "Standard_A2_v2": 0.091000, + "Standard_A2m_v2": 0.124000, + "Standard_A3": 0.240000, + "Standard_A4": 0.480000, + "Standard_A4_v2": 0.191000, + "Standard_A4m_v2": 0.248000, + "Standard_A5": 0.220000, + "Standard_A6": 0.440000, + "Standard_A7": 0.880000, + "Standard_A8_v2": 0.400000, + "Standard_A8m_v2": 0.495000, + "Standard_B12ms": 0.599000, + "Standard_B16als_v2": 0.639000, + "Standard_B16as_v2": 0.722000, + "Standard_B16ls_v2": 0.781000, + "Standard_B16ms": 0.799000, + "Standard_B16pls_v2": 0.571000, + "Standard_B16ps_v2": 0.645000, + "Standard_B16s_v2": 0.872000, + "Standard_B1ls": 0.006240, + "Standard_B1ms": 0.025000, + "Standard_B1s": 0.012500, + "Standard_B20ms": 0.998000, + "Standard_B2als_v2": 0.054300, + "Standard_B2as_v2": 0.099400, + "Standard_B2ats_v2": 0.011300, + "Standard_B2ls_v2": 0.059100, + "Standard_B2ms": 0.099800, + "Standard_B2pls_v2": 0.049500, + "Standard_B2ps_v2": 0.080600, + "Standard_B2pts_v2": 0.010100, + "Standard_B2s": 0.049900, + "Standard_B2s_v2": 0.109000, + "Standard_B2ts_v2": 0.021700, + "Standard_B32als_v2": 1.426000, + "Standard_B32as_v2": 1.444000, + "Standard_B32ls_v2": 1.415000, + "Standard_B32s_v2": 1.745000, + "Standard_B4als_v2": 0.178000, + "Standard_B4as_v2": 0.180000, + "Standard_B4ls_v2": 0.195000, + "Standard_B4ms": 0.200000, + "Standard_B4pls_v2": 0.161000, + "Standard_B4ps_v2": 0.180000, + "Standard_B4s_v2": 0.218000, + "Standard_B8als_v2": 0.320000, + "Standard_B8as_v2": 0.361000, + "Standard_B8ls_v2": 0.354000, + "Standard_B8ms": 0.399000, + "Standard_B8pls_v2": 0.322000, + "Standard_B8ps_v2": 0.359000, + "Standard_B8s_v2": 0.436000, + "Standard_D1": 0.067000, + "Standard_D11": 0.173000, + "Standard_D11_v2": 0.166000, + "Standard_D11_v2_Promo": 0.166000, + "Standard_D12": 0.347000, + "Standard_D12_v2": 0.332000, + "Standard_D12_v2_Promo": 0.332000, + "Standard_D13": 0.694000, + "Standard_D13_v2": 0.665000, + "Standard_D13_v2_Promo": 0.665000, + "Standard_D14": 1.387000, + "Standard_D14_v2": 1.330000, + "Standard_D14_v2_Promo": 1.330000, + "Standard_D15_v2": 1.662000, + "Standard_D15i_v2": 1.662000, + "Standard_D16_v3": 0.880000, + "Standard_D16_v4": 0.922000, + "Standard_D16_v5": 0.922000, + "Standard_D16a_v4": 0.922000, + "Standard_D16ads_v5": 0.989000, + "Standard_D16as_v4": 0.922000, + "Standard_D16as_v5": 0.826000, + "Standard_D16d_v4": 1.175000, + "Standard_D16d_v5": 1.085000, + "Standard_D16ds_v4": 1.175000, + "Standard_D16ds_v5": 1.085000, + "Standard_D16lds_v5": 0.922000, + "Standard_D16ls_v5": 0.816000, + "Standard_D16pds_v5": 0.867000, + "Standard_D16plds_v5": 0.737000, + "Standard_D16pls_v5": 0.653000, + "Standard_D16ps_v5": 0.739000, + "Standard_D16s_v3": 0.880000, + "Standard_D16s_v4": 0.922000, + "Standard_D16s_v5": 0.922000, + "Standard_D1_v2": 0.063600, + "Standard_D2": 0.134000, + "Standard_D2_v2": 0.127000, + "Standard_D2_v2_Promo": 0.127000, + "Standard_D2_v3": 0.110000, + "Standard_D2_v4": 0.115000, + "Standard_D2_v5": 0.115000, + "Standard_D2a_v4": 0.115000, + "Standard_D2ads_v5": 0.124000, + "Standard_D2as_v4": 0.115000, + "Standard_D2as_v5": 0.103000, + "Standard_D2d_v4": 0.147000, + "Standard_D2d_v5": 0.136000, + "Standard_D2ds_v4": 0.147000, + "Standard_D2ds_v5": 0.136000, + "Standard_D2lds_v5": 0.115000, + "Standard_D2ls_v5": 0.102000, + "Standard_D2pds_v5": 0.108000, + "Standard_D2plds_v5": 0.092200, + "Standard_D2pls_v5": 0.081600, + "Standard_D2ps_v5": 0.092400, + "Standard_D2s_v3": 0.110000, + "Standard_D2s_v4": 0.115000, + "Standard_D2s_v5": 0.115000, + "Standard_D3": 0.268000, + "Standard_D32-16s_v3": 1.760000, + "Standard_D32-8s_v3": 1.760000, + "Standard_D32_v3": 1.760000, + "Standard_D32_v4": 1.843000, + "Standard_D32_v5": 1.843000, + "Standard_D32a_v4": 1.843000, + "Standard_D32ads_v5": 3.450000, + "Standard_D32as_v4": 1.843000, + "Standard_D32as_v5": 1.651000, + "Standard_D32d_v4": 2.350000, + "Standard_D32d_v5": 2.170000, + "Standard_D32ds_v4": 2.350000, + "Standard_D32ds_v5": 2.170000, + "Standard_D32lds_v5": 1.843000, + "Standard_D32ls_v5": 1.632000, + "Standard_D32pds_v5": 1.734000, + "Standard_D32plds_v5": 1.475000, + "Standard_D32pls_v5": 1.306000, + "Standard_D32ps_v5": 1.478000, + "Standard_D32s_v3": 1.760000, + "Standard_D32s_v4": 1.843000, + "Standard_D32s_v5": 1.843000, + "Standard_D3_v2": 0.254000, + "Standard_D3_v2_Promo": 0.254000, + "Standard_D4": 0.536000, + "Standard_D48_v3": 2.640000, + "Standard_D48_v4": 2.765000, + "Standard_D48_v5": 2.765000, + "Standard_D48a_v4": 2.765000, + "Standard_D48ads_v5": 5.174000, + "Standard_D48as_v4": 2.765000, + "Standard_D48as_v5": 4.685000, + "Standard_D48d_v4": 3.526000, + "Standard_D48d_v5": 3.254000, + "Standard_D48ds_v4": 3.526000, + "Standard_D48ds_v5": 3.254000, + "Standard_D48lds_v5": 2.765000, + "Standard_D48ls_v5": 2.448000, + "Standard_D48pds_v5": 2.602000, + "Standard_D48plds_v5": 2.212000, + "Standard_D48pls_v5": 1.958000, + "Standard_D48ps_v5": 2.218000, + "Standard_D48s_v3": 2.640000, + "Standard_D48s_v4": 2.765000, + "Standard_D48s_v5": 2.765000, + "Standard_D4_v2": 0.509000, + "Standard_D4_v2_Promo": 0.509000, + "Standard_D4_v3": 0.220000, + "Standard_D4_v4": 0.230000, + "Standard_D4_v5": 0.230000, + "Standard_D4a_v4": 0.230000, + "Standard_D4ads_v5": 0.431000, + "Standard_D4as_v4": 0.230000, + "Standard_D4as_v5": 0.390000, + "Standard_D4d_v4": 0.294000, + "Standard_D4d_v5": 0.271000, + "Standard_D4ds_v4": 0.294000, + "Standard_D4ds_v5": 0.271000, + "Standard_D4lds_v5": 0.230000, + "Standard_D4ls_v5": 0.204000, + "Standard_D4pds_v5": 0.217000, + "Standard_D4plds_v5": 0.184000, + "Standard_D4pls_v5": 0.163000, + "Standard_D4ps_v5": 0.185000, + "Standard_D4s_v3": 0.220000, + "Standard_D4s_v4": 0.230000, + "Standard_D4s_v5": 0.230000, + "Standard_D5_v2": 1.017000, + "Standard_D5_v2_Promo": 1.017000, + "Standard_D64-16s_v3": 3.520000, + "Standard_D64-32s_v3": 3.520000, + "Standard_D64_v3": 3.520000, + "Standard_D64_v4": 3.686000, + "Standard_D64_v5": 3.686000, + "Standard_D64a_v4": 3.686000, + "Standard_D64ads_v5": 3.955000, + "Standard_D64as_v4": 3.686000, + "Standard_D64as_v5": 6.246000, + "Standard_D64d_v4": 4.701000, + "Standard_D64d_v5": 4.339000, + "Standard_D64ds_v4": 4.701000, + "Standard_D64ds_v5": 4.339000, + "Standard_D64lds_v5": 3.686000, + "Standard_D64ls_v5": 3.264000, + "Standard_D64pds_v5": 3.469000, + "Standard_D64plds_v5": 2.949000, + "Standard_D64pls_v5": 2.611000, + "Standard_D64ps_v5": 2.957000, + "Standard_D64s_v3": 3.520000, + "Standard_D64s_v4": 3.686000, + "Standard_D64s_v5": 3.686000, + "Standard_D8_v3": 0.440000, + "Standard_D8_v4": 0.461000, + "Standard_D8_v5": 0.461000, + "Standard_D8a_v4": 0.461000, + "Standard_D8ads_v5": 0.862000, + "Standard_D8as_v4": 0.461000, + "Standard_D8as_v5": 0.413000, + "Standard_D8d_v4": 0.588000, + "Standard_D8d_v5": 0.542000, + "Standard_D8ds_v4": 0.588000, + "Standard_D8ds_v5": 0.542000, + "Standard_D8lds_v5": 0.461000, + "Standard_D8ls_v5": 0.408000, + "Standard_D8pds_v5": 0.434000, + "Standard_D8plds_v5": 0.369000, + "Standard_D8pls_v5": 0.326000, + "Standard_D8ps_v5": 0.370000, + "Standard_D8s_v3": 0.440000, + "Standard_D8s_v4": 0.461000, + "Standard_D8s_v5": 0.461000, + "Standard_D96_v5": 5.530000, + "Standard_D96a_v4": 5.530000, + "Standard_D96ads_v5": 10.349000, + "Standard_D96as_v4": 5.530000, + "Standard_D96as_v5": 4.954000, + "Standard_D96d_v5": 6.509000, + "Standard_D96ds_v5": 6.509000, + "Standard_D96lds_v5": 5.530000, + "Standard_D96ls_v5": 4.896000, + "Standard_D96s_v5": 5.530000, + "Standard_DC16ds_v3": 2.170000, + "Standard_DC16s_v3": 1.843000, + "Standard_DC1ds_v3": 0.136000, + "Standard_DC1s_v2": 0.115000, + "Standard_DC1s_v3": 0.115000, + "Standard_DC24ds_v3": 3.254000, + "Standard_DC24s_v3": 2.765000, + "Standard_DC2ds_v3": 0.271000, + "Standard_DC2s_v2": 0.230000, + "Standard_DC2s_v3": 0.230000, + "Standard_DC32ds_v3": 4.339000, + "Standard_DC32s_v3": 3.686000, + "Standard_DC48ds_v3": 6.509000, + "Standard_DC48s_v3": 5.530000, + "Standard_DC4ds_v3": 0.542000, + "Standard_DC4s_v2": 0.461000, + "Standard_DC4s_v3": 0.461000, + "Standard_DC8_v2": 0.922000, + "Standard_DC8ds_v3": 1.085000, + "Standard_DC8s_v3": 0.922000, + "Standard_DS1": 0.067000, + "Standard_DS11": 0.173000, + "Standard_DS11-1_v2": 0.166000, + "Standard_DS11_v2": 0.166000, + "Standard_DS11_v2_Promo": 0.166000, + "Standard_DS12": 0.347000, + "Standard_DS12-1_v2": 0.332000, + "Standard_DS12-2_v2": 0.332000, + "Standard_DS12_v2": 0.332000, + "Standard_DS12_v2_Promo": 0.332000, + "Standard_DS13": 0.694000, + "Standard_DS13-2_v2": 0.665000, + "Standard_DS13-4_v2": 0.665000, + "Standard_DS13_v2": 0.665000, + "Standard_DS13_v2_Promo": 0.665000, + "Standard_DS14": 1.387000, + "Standard_DS14-4_v2": 1.330000, + "Standard_DS14-8_v2": 1.330000, + "Standard_DS14_v2": 1.330000, + "Standard_DS14_v2_Promo": 1.330000, + "Standard_DS15_v2": 1.662000, + "Standard_DS15i_v2": 1.662000, + "Standard_DS1_v2": 0.063600, + "Standard_DS2": 0.134000, + "Standard_DS2_v2": 0.127000, + "Standard_DS2_v2_Promo": 0.127000, + "Standard_DS3": 0.268000, + "Standard_DS3_v2": 0.254000, + "Standard_DS3_v2_Promo": 0.254000, + "Standard_DS4": 0.536000, + "Standard_DS4_v2": 0.509000, + "Standard_DS4_v2_Promo": 0.509000, + "Standard_DS5_v2": 1.017000, + "Standard_DS5_v2_Promo": 1.017000, + "Standard_E104i_v5": 8.649000, + "Standard_E104id_v5": 9.884000, + "Standard_E104ids_v5": 9.884000, + "Standard_E104is_v5": 8.649000, + "Standard_E112iads_v5": 9.684000, + "Standard_E112ias_v5": 8.353000, + "Standard_E112ibds_v5": 12.345000, + "Standard_E112ibs_v5": 11.014000, + "Standard_E16-4ads_v5": 1.258000, + "Standard_E16-4as_v4": 1.210000, + "Standard_E16-4as_v5": 1.085000, + "Standard_E16-4ds_v4": 1.498000, + "Standard_E16-4ds_v5": 1.382000, + "Standard_E16-4s_v3": 1.166000, + "Standard_E16-4s_v4": 1.210000, + "Standard_E16-4s_v5": 1.210000, + "Standard_E16-8ads_v5": 1.994000, + "Standard_E16-8as_v4": 1.210000, + "Standard_E16-8as_v5": 1.821000, + "Standard_E16-8ds_v4": 1.498000, + "Standard_E16-8ds_v5": 1.382000, + "Standard_E16-8s_v3": 1.166000, + "Standard_E16-8s_v4": 1.210000, + "Standard_E16-8s_v5": 1.210000, + "Standard_E16_v3": 1.166000, + "Standard_E16_v4": 1.210000, + "Standard_E16_v5": 1.210000, + "Standard_E16a_v4": 1.210000, + "Standard_E16ads_v5": 1.304000, + "Standard_E16as_v4": 1.210000, + "Standard_E16as_v5": 1.085000, + "Standard_E16bds_v5": 1.603000, + "Standard_E16bs_v5": 1.430000, + "Standard_E16d_v4": 1.498000, + "Standard_E16d_v5": 1.382000, + "Standard_E16ds_v4": 1.498000, + "Standard_E16ds_v5": 1.382000, + "Standard_E16pds_v5": 1.106000, + "Standard_E16ps_v5": 0.968000, + "Standard_E16s_v3": 1.166000, + "Standard_E16s_v4": 1.210000, + "Standard_E16s_v5": 1.210000, + "Standard_E20_v3": 1.460000, + "Standard_E20_v4": 1.512000, + "Standard_E20_v5": 1.512000, + "Standard_E20a_v4": 1.512000, + "Standard_E20ads_v5": 1.618000, + "Standard_E20as_v4": 1.512000, + "Standard_E20as_v5": 1.402000, + "Standard_E20d_v4": 1.872000, + "Standard_E20d_v5": 1.728000, + "Standard_E20ds_v4": 1.872000, + "Standard_E20ds_v5": 1.728000, + "Standard_E20pds_v5": 1.382000, + "Standard_E20ps_v5": 1.210000, + "Standard_E20s_v3": 1.460000, + "Standard_E20s_v4": 1.512000, + "Standard_E20s_v5": 1.512000, + "Standard_E2_v3": 0.146000, + "Standard_E2_v4": 0.151000, + "Standard_E2_v5": 0.151000, + "Standard_E2a_v4": 0.151000, + "Standard_E2ads_v5": 0.157000, + "Standard_E2as_v4": 0.151000, + "Standard_E2as_v5": 0.182000, + "Standard_E2bds_v5": 0.200000, + "Standard_E2bs_v5": 0.179000, + "Standard_E2d_v4": 0.187000, + "Standard_E2d_v5": 0.173000, + "Standard_E2ds_v4": 0.187000, + "Standard_E2ds_v5": 0.173000, + "Standard_E2pds_v5": 0.138000, + "Standard_E2ps_v5": 0.121000, + "Standard_E2s_v3": 0.146000, + "Standard_E2s_v4": 0.151000, + "Standard_E2s_v5": 0.151000, + "Standard_E32-16ads_v5": 3.987000, + "Standard_E32-16as_v4": 2.419000, + "Standard_E32-16as_v5": 3.642000, + "Standard_E32-16ds_v4": 2.995000, + "Standard_E32-16ds_v5": 2.765000, + "Standard_E32-16s_v3": 2.333000, + "Standard_E32-16s_v4": 2.419000, + "Standard_E32-16s_v5": 2.419000, + "Standard_E32-8ads_v5": 3.987000, + "Standard_E32-8as_v4": 2.419000, + "Standard_E32-8as_v5": 2.170000, + "Standard_E32-8ds_v4": 2.995000, + "Standard_E32-8ds_v5": 2.765000, + "Standard_E32-8s_v3": 2.333000, + "Standard_E32-8s_v4": 2.419000, + "Standard_E32-8s_v5": 2.419000, + "Standard_E32_v3": 2.333000, + "Standard_E32_v4": 2.419000, + "Standard_E32_v5": 2.419000, + "Standard_E32a_v4": 2.419000, + "Standard_E32ads_v5": 2.515000, + "Standard_E32as_v4": 2.419000, + "Standard_E32as_v5": 2.216000, + "Standard_E32bds_v5": 3.206000, + "Standard_E32bs_v5": 2.861000, + "Standard_E32d_v4": 2.995000, + "Standard_E32d_v5": 2.765000, + "Standard_E32ds_v4": 2.995000, + "Standard_E32ds_v5": 2.765000, + "Standard_E32pds_v5": 2.212000, + "Standard_E32ps_v5": 1.935000, + "Standard_E32s_v3": 2.333000, + "Standard_E32s_v4": 2.419000, + "Standard_E32s_v5": 2.419000, + "Standard_E4-2ads_v5": 0.314000, + "Standard_E4-2as_v4": 0.302000, + "Standard_E4-2as_v5": 0.271000, + "Standard_E4-2ds_v4": 0.374000, + "Standard_E4-2ds_v5": 0.346000, + "Standard_E4-2s_v3": 0.292000, + "Standard_E4-2s_v4": 0.302000, + "Standard_E4-2s_v5": 0.302000, + "Standard_E48_v3": 3.499000, + "Standard_E48_v4": 3.629000, + "Standard_E48_v5": 3.629000, + "Standard_E48a_v4": 3.629000, + "Standard_E48ads_v5": 3.773000, + "Standard_E48as_v4": 3.629000, + "Standard_E48as_v5": 3.254000, + "Standard_E48bds_v5": 4.810000, + "Standard_E48bs_v5": 4.291000, + "Standard_E48d_v4": 4.493000, + "Standard_E48d_v5": 4.147000, + "Standard_E48ds_v4": 4.493000, + "Standard_E48ds_v5": 4.147000, + "Standard_E48s_v3": 3.499000, + "Standard_E48s_v4": 3.629000, + "Standard_E48s_v5": 3.629000, + "Standard_E4_v3": 0.292000, + "Standard_E4_v4": 0.302000, + "Standard_E4_v5": 0.302000, + "Standard_E4a_v4": 0.302000, + "Standard_E4ads_v5": 0.360000, + "Standard_E4as_v4": 0.302000, + "Standard_E4as_v5": 0.317000, + "Standard_E4bds_v5": 0.401000, + "Standard_E4bs_v5": 0.358000, + "Standard_E4d_v4": 0.374000, + "Standard_E4d_v5": 0.346000, + "Standard_E4ds_v4": 0.374000, + "Standard_E4ds_v5": 0.346000, + "Standard_E4pds_v5": 0.276000, + "Standard_E4ps_v5": 0.242000, + "Standard_E4s_v3": 0.292000, + "Standard_E4s_v4": 0.302000, + "Standard_E4s_v5": 0.302000, + "Standard_E64-16ads_v5": 5.030000, + "Standard_E64-16as_v4": 4.838000, + "Standard_E64-16as_v5": 4.339000, + "Standard_E64-16ds_v4": 5.990000, + "Standard_E64-16ds_v5": 5.530000, + "Standard_E64-16s_v3": 4.199000, + "Standard_E64-16s_v4": 4.838000, + "Standard_E64-16s_v5": 4.838000, + "Standard_E64-32ads_v5": 5.030000, + "Standard_E64-32as_v4": 4.838000, + "Standard_E64-32as_v5": 4.339000, + "Standard_E64-32ds_v4": 5.990000, + "Standard_E64-32ds_v5": 5.530000, + "Standard_E64-32s_v3": 4.199000, + "Standard_E64-32s_v4": 4.838000, + "Standard_E64-32s_v5": 4.838000, + "Standard_E64_v3": 4.199000, + "Standard_E64_v4": 4.838000, + "Standard_E64_v5": 4.838000, + "Standard_E64a_v4": 4.838000, + "Standard_E64ads_v5": 5.030000, + "Standard_E64as_v4": 4.838000, + "Standard_E64as_v5": 4.339000, + "Standard_E64bds_v5": 6.413000, + "Standard_E64bs_v5": 5.722000, + "Standard_E64d_v4": 5.990000, + "Standard_E64d_v5": 5.530000, + "Standard_E64ds_v4": 5.990000, + "Standard_E64ds_v5": 5.530000, + "Standard_E64i_v3": 4.199000, + "Standard_E64is_v3": 4.199000, + "Standard_E64s_v3": 4.199000, + "Standard_E64s_v4": 4.838000, + "Standard_E64s_v5": 4.838000, + "Standard_E8-2ads_v5": 0.629000, + "Standard_E8-2as_v4": 0.605000, + "Standard_E8-2as_v5": 0.910000, + "Standard_E8-2ds_v4": 0.749000, + "Standard_E8-2ds_v5": 0.691000, + "Standard_E8-2s_v3": 0.583000, + "Standard_E8-2s_v4": 0.605000, + "Standard_E8-2s_v5": 0.605000, + "Standard_E8-4ads_v5": 0.629000, + "Standard_E8-4as_v4": 0.605000, + "Standard_E8-4as_v5": 0.910000, + "Standard_E8-4ds_v4": 0.749000, + "Standard_E8-4ds_v5": 0.691000, + "Standard_E8-4s_v3": 0.583000, + "Standard_E8-4s_v4": 0.605000, + "Standard_E8-4s_v5": 0.605000, + "Standard_E80ids_v4": 7.488000, + "Standard_E80is_v4": 6.048000, + "Standard_E8_v3": 0.583000, + "Standard_E8_v4": 0.605000, + "Standard_E8_v5": 0.605000, + "Standard_E8a_v4": 0.605000, + "Standard_E8ads_v5": 0.675000, + "Standard_E8as_v4": 0.605000, + "Standard_E8as_v5": 0.542000, + "Standard_E8bds_v5": 0.802000, + "Standard_E8bs_v5": 0.715000, + "Standard_E8d_v4": 0.749000, + "Standard_E8d_v5": 0.691000, + "Standard_E8ds_v4": 0.749000, + "Standard_E8ds_v5": 0.691000, + "Standard_E8pds_v5": 0.553000, + "Standard_E8ps_v5": 0.484000, + "Standard_E8s_v3": 0.583000, + "Standard_E8s_v4": 0.605000, + "Standard_E8s_v5": 0.605000, + "Standard_E96-24ads_v5": 11.962000, + "Standard_E96-24as_v4": 7.258000, + "Standard_E96-24as_v5": 6.509000, + "Standard_E96-24ds_v5": 8.294000, + "Standard_E96-24s_v5": 7.258000, + "Standard_E96-48ads_v5": 11.962000, + "Standard_E96-48as_v4": 7.258000, + "Standard_E96-48as_v5": 6.509000, + "Standard_E96-48ds_v5": 8.294000, + "Standard_E96-48s_v5": 7.258000, + "Standard_E96_v5": 7.258000, + "Standard_E96a_v4": 7.258000, + "Standard_E96ads_v5": 7.546000, + "Standard_E96as_v4": 7.258000, + "Standard_E96as_v5": 6.555000, + "Standard_E96bds_v5": 9.619000, + "Standard_E96bs_v5": 8.582000, + "Standard_E96d_v5": 8.294000, + "Standard_E96ds_v5": 8.294000, + "Standard_E96iads_v5": 8.300000, + "Standard_E96ias_v5": 7.160000, + "Standard_E96s_v5": 7.258000, + "Standard_F1": 0.054700, + "Standard_F16": 0.875000, + "Standard_F16s": 0.875000, + "Standard_F16s_v2": 0.816000, + "Standard_F1s": 0.054700, + "Standard_F2": 0.109000, + "Standard_F2s": 0.109000, + "Standard_F2s_v2": 0.102000, + "Standard_F32s_v2": 1.632000, + "Standard_F4": 0.219000, + "Standard_F48s_v2": 2.448000, + "Standard_F4s": 0.219000, + "Standard_F4s_v2": 0.204000, + "Standard_F64s_v2": 3.264000, + "Standard_F72s_v2": 3.672000, + "Standard_F8": 0.438000, + "Standard_F8s": 0.438000, + "Standard_F8s_v2": 0.408000, + "Standard_FX12mds": 1.339000, + "Standard_FX24mds": 2.678000, + "Standard_FX36mds": 4.018000, + "Standard_FX48mds": 5.357000, + "Standard_FX4mds": 0.446000, + "Standard_H16": 1.749000, + "Standard_H16m": 2.343000, + "Standard_H16mr": 2.577000, + "Standard_H16r": 1.924000, + "Standard_H8": 0.875000, + "Standard_H8m": 1.172000, + "Standard_HB120-16rs_v2": 3.960000, + "Standard_HB120-16rs_v3": 3.960000, + "Standard_HB120-32rs_v2": 3.960000, + "Standard_HB120-32rs_v3": 3.960000, + "Standard_HB120-64rs_v2": 3.960000, + "Standard_HB120-64rs_v3": 3.960000, + "Standard_HB120-96rs_v2": 3.960000, + "Standard_HB120-96rs_v3": 3.960000, + "Standard_HB120rs_v2": 3.960000, + "Standard_HB120rs_v3": 3.960000, + "Standard_HB176-144rs_v4": 8.640000, + "Standard_HB176-24rs_v4": 8.640000, + "Standard_HB176-48rs_v4": 8.640000, + "Standard_HB176-96rs_v4": 8.640000, + "Standard_HB176rs_v4": 8.640000, + "Standard_HB60-15rs": 2.508000, + "Standard_HB60-30rs": 2.508000, + "Standard_HB60-45rs": 2.508000, + "Standard_HB60rs": 2.508000, + "Standard_HC44-16rs": 3.484000, + "Standard_HC44-32rs": 3.484000, + "Standard_HC44rs": 3.484000, + "Standard_HX176-144rs": 10.368000, + "Standard_HX176-24rs": 10.368000, + "Standard_HX176-48rs": 10.368000, + "Standard_HX176-96rs": 10.368000, + "Standard_HX176rs": 10.368000, + "Standard_L16s_v2": 1.498000, + "Standard_L16s_v3": 1.670000, + "Standard_L32s_v2": 2.995000, + "Standard_L32s_v3": 3.341000, + "Standard_L48s_v2": 4.493000, + "Standard_L48s_v3": 5.011000, + "Standard_L64s_v2": 5.990000, + "Standard_L64s_v3": 6.682000, + "Standard_L80s_v2": 7.488000, + "Standard_L80s_v3": 8.352000, + "Standard_L88is_v2": 8.237000, + "Standard_L8s_v2": 0.749000, + "Standard_L8s_v3": 0.835000, + "Standard_L96s_v2": 8.986000, + "Standard_M128": 16.006000, + "Standard_M128-32ms": 32.026000, + "Standard_M128-64ms": 32.026000, + "Standard_M128dms_v2": 32.028000, + "Standard_M128ds_v2": 16.008000, + "Standard_M128m": 32.026000, + "Standard_M128ms": 32.026000, + "Standard_M128ms_v2": 31.523000, + "Standard_M128s": 16.006000, + "Standard_M128s_v2": 15.503000, + "Standard_M16-4ms": 3.688000, + "Standard_M16-8ms": 3.688000, + "Standard_M16ms": 3.688000, + "Standard_M16s": 2.864000, + "Standard_M192idms_v2": 38.477000, + "Standard_M192ids_v2": 19.238000, + "Standard_M192ims_v2": 37.972000, + "Standard_M192is_v2": 18.733000, + "Standard_M208-104ms_v2": 53.544000, + "Standard_M208-104s_v2": 26.772000, + "Standard_M208-52ms_v2": 53.544000, + "Standard_M208-52s_v2": 26.772000, + "Standard_M208ms_v2": 53.544000, + "Standard_M208s_v2": 26.772000, + "Standard_M32-16ms": 7.375000, + "Standard_M32-8ms": 7.375000, + "Standard_M32dms_v2": 7.375000, + "Standard_M32ls": 3.448000, + "Standard_M32ms": 7.375000, + "Standard_M32ms_v2": 7.249000, + "Standard_M32s": 4.002000, + "Standard_M32ts": 3.248000, + "Standard_M416-104ms_v2": 118.980000, + "Standard_M416-104s_v2": 59.496000, + "Standard_M416-208ms_v2": 118.980000, + "Standard_M416-208s_v2": 59.496000, + "Standard_M416is_v2": 59.496000, + "Standard_M416ms_v2": 118.980000, + "Standard_M416s_8_v2": 79.330000, + "Standard_M416s_v2": 59.496000, + "Standard_M64": 8.003000, + "Standard_M64-16ms": 12.405000, + "Standard_M64-32ms": 12.405000, + "Standard_M64dms_v2": 12.408000, + "Standard_M64ds_v2": 8.003000, + "Standard_M64ls": 6.498000, + "Standard_M64m": 12.405000, + "Standard_M64ms": 12.405000, + "Standard_M64ms_v2": 12.156000, + "Standard_M64s": 8.003000, + "Standard_M64s_v2": 7.750000, + "Standard_M8-2ms": 1.844000, + "Standard_M8-4ms": 1.844000, + "Standard_M8ms": 1.844000, + "Standard_NC12": 2.160000, + "Standard_NC12s_v2": 4.968000, + "Standard_NC12s_v3": 6.732000, + "Standard_NC16ads_A10_v4": 2.640000, + "Standard_NC16as_T4_v3": 1.445000, + "Standard_NC24": 4.320000, + "Standard_NC24ads_A100_v4": 4.408000, + "Standard_NC24r": 4.752000, + "Standard_NC24rs_v2": 10.930000, + "Standard_NC24rs_v3": 14.810400, + "Standard_NC24s_v2": 9.936000, + "Standard_NC24s_v3": 13.464000, + "Standard_NC32ads_A10_v4": 5.280000, + "Standard_NC48ads_A100_v4": 8.815000, + "Standard_NC4as_T4_v3": 0.631000, + "Standard_NC6": 1.080000, + "Standard_NC64as_T4_v3": 5.222000, + "Standard_NC6s_v2": 2.484000, + "Standard_NC6s_v3": 3.366000, + "Standard_NC8ads_A10_v4": 1.320000, + "Standard_NC8as_T4_v3": 0.902000, + "Standard_NC96ads_A100_v4": 17.630000, + "Standard_ND12s": 4.968000, + "Standard_ND24rs": 10.930000, + "Standard_ND24s": 9.936000, + "Standard_ND40rs_v2": 26.438000, + "Standard_ND40s_v2": 14.688000, + "Standard_ND6s": 2.484000, + "Standard_ND96ams_A100_v4": 39.324000, + "Standard_ND96amsr_A100_v4": 39.324000, + "Standard_ND96asr_A100_v4": 32.630000, + "Standard_ND96asr_v4": 32.630000, + "Standard_NP10s": 1.980000, + "Standard_NP20s": 3.960000, + "Standard_NP40s": 7.920000, + "Standard_NV12": 2.300000, + "Standard_NV12ads_A10_v5": 1.090000, + "Standard_NV12s_v2": 1.368000, + "Standard_NV12s_v3": 1.368000, + "Standard_NV16as_v4": 1.118000, + "Standard_NV18ads_A10_v5": 1.920000, + "Standard_NV24": 4.599000, + "Standard_NV24s_v2": 2.736000, + "Standard_NV24s_v3": 2.736000, + "Standard_NV32as_v4": 2.237000, + "Standard_NV36adms_A10_v5": 5.424000, + "Standard_NV36ads_A10_v5": 3.840000, + "Standard_NV48s_v3": 5.472000, + "Standard_NV4as_v4": 0.280000, + "Standard_NV6": 1.150000, + "Standard_NV6ads_A10_v5": 0.545000, + "Standard_NV6s_v2": 0.684000, + "Standard_NV72ads_A10_v5": 7.824000, + "Standard_NV8as_v4": 0.559000, + "Standard_PB12s": 1.826000, + "Standard_PB24s": 3.652000, + "Standard_PB6s": 0.913000, + } + // southeastasia + initialOnDemandPrices["southeastasia"] = map[string]float64{ + "Basic_A0": 0.018000, + "Basic_A1": 0.030000, + "Basic_A2": 0.095000, + "Basic_A3": 0.232000, + "Basic_A4": 0.464000, + "DCdsv3 Type1": 7.445000, + "DCsv2 Type 1": 1.056000, + "DCsv3 Type1": 6.336000, + "Dadsv5_Type1": 7.946000, + "Dasv4_Type1": 6.325000, + "Dasv4_Type2": 6.600000, + "Dasv5_Type1": 6.653000, + "Ddsv4_Type 1": 4.963000, + "Ddsv4_Type2": 5.894000, + "Ddsv5_Type1": 7.445000, + "Dsv3_Type3": 5.272000, + "Dsv3_Type4": 6.590000, + "Dsv4_Type1": 5.280000, + "Dsv4_Type2": 6.336000, + "Dsv5_Type1": 6.600000, + "Eadsv5_Type1": 8.395000, + "Easv4_Type1": 8.026000, + "Easv4_Type2": 8.026000, + "Easv5_Type1": 7.181000, + "Ebdsv5-Type1": 7.075000, + "Ebsv5-Type1": 6.266000, + "Edsv4_Type 1": 6.125000, + "Edsv4_Type2": 7.273000, + "Edsv5_Type1": 9.187000, + "Esv3_Type3": 5.192000, + "Esv3_Type4": 7.034000, + "Esv4_Type1": 5.350000, + "Esv4_Type2": 7.022000, + "Esv5_Type1": 8.026000, + "Fsv2 Type3": 4.528000, + "Fsv2_Type2": 3.881000, + "Fsv2_Type4": 5.174000, + "Lsv2_Type1": 8.228000, + "Lsv3_Type1": 9.196000, + "Mdmsv2MedMem _Type1": 42.571000, + "Mdsv2MedMem_Type1": 21.277000, + "Ms_Type1": 21.270000, + "Msm_Type1": 42.577000, + "Msmv2MedMem Type1": 41.899000, + "Msmv2_Type1": 141.780000, + "Msv2MedMem Type1": 20.606000, + "Msv2_Type1": 70.899000, + "NVasv4_Type1": 9.329000, + "NVsv3_Type1": 6.972000, + "Standard_A0": 0.020000, + "Standard_A1": 0.060000, + "Standard_A1_v2": 0.045000, + "Standard_A2": 0.120000, + "Standard_A2_v2": 0.095000, + "Standard_A2m_v2": 0.134000, + "Standard_A3": 0.240000, + "Standard_A4": 0.480000, + "Standard_A4_v2": 0.198000, + "Standard_A4m_v2": 0.281000, + "Standard_A5": 0.270000, + "Standard_A6": 0.540000, + "Standard_A7": 1.080000, + "Standard_A8_v2": 0.417000, + "Standard_A8m_v2": 0.589000, + "Standard_B12ms": 0.634000, + "Standard_B16als_v2": 0.669000, + "Standard_B16as_v2": 0.755000, + "Standard_B16ls_v2": 0.748000, + "Standard_B16ms": 0.845000, + "Standard_B16pls_v2": 0.674000, + "Standard_B16ps_v2": 0.752000, + "Standard_B16s_v2": 0.918000, + "Standard_B1ls": 0.006600, + "Standard_B1ms": 0.026400, + "Standard_B1s": 0.013200, + "Standard_B20ms": 1.056000, + "Standard_B2als_v2": 0.056400, + "Standard_B2as_v2": 0.094400, + "Standard_B2ats_v2": 0.021000, + "Standard_B2ls_v2": 0.052800, + "Standard_B2ms": 0.106000, + "Standard_B2pls_v2": 0.042400, + "Standard_B2ps_v2": 0.084800, + "Standard_B2pts_v2": 0.010600, + "Standard_B2s": 0.052800, + "Standard_B2s_v2": 0.106000, + "Standard_B2ts_v2": 0.022400, + "Standard_B32als_v2": 1.337000, + "Standard_B32as_v2": 1.658000, + "Standard_B32ls_v2": 1.496000, + "Standard_B32s_v2": 1.837000, + "Standard_B4als_v2": 0.186000, + "Standard_B4as_v2": 0.189000, + "Standard_B4ls_v2": 0.206000, + "Standard_B4ms": 0.211000, + "Standard_B4pls_v2": 0.169000, + "Standard_B4ps_v2": 0.188000, + "Standard_B4s_v2": 0.230000, + "Standard_B8als_v2": 0.371000, + "Standard_B8as_v2": 0.378000, + "Standard_B8ls_v2": 0.374000, + "Standard_B8ms": 0.422000, + "Standard_B8pls_v2": 0.337000, + "Standard_B8ps_v2": 0.339000, + "Standard_B8s_v2": 0.422000, + "Standard_D1": 0.098000, + "Standard_D11": 0.210000, + "Standard_D11_v2": 0.191000, + "Standard_D11_v2_Promo": 0.191000, + "Standard_D12": 0.420000, + "Standard_D12_v2": 0.382000, + "Standard_D12_v2_Promo": 0.382000, + "Standard_D13": 0.840000, + "Standard_D13_v2": 0.765000, + "Standard_D13_v2_Promo": 0.765000, + "Standard_D14": 1.680000, + "Standard_D14_v2": 1.530000, + "Standard_D14_v2_Promo": 1.530000, + "Standard_D15_v2": 1.912000, + "Standard_D15i_v2": 1.912000, + "Standard_D16_v3": 1.000000, + "Standard_D16_v4": 0.960000, + "Standard_D16_v5": 0.960000, + "Standard_D16a_v4": 0.960000, + "Standard_D16ads_v5": 1.768000, + "Standard_D16as_v4": 0.960000, + "Standard_D16as_v5": 1.600000, + "Standard_D16d_v4": 1.128000, + "Standard_D16d_v5": 1.128000, + "Standard_D16ds_v4": 1.128000, + "Standard_D16ds_v5": 1.128000, + "Standard_D16lds_v5": 0.896000, + "Standard_D16ls_v5": 0.784000, + "Standard_D16pds_v5": 0.904000, + "Standard_D16plds_v5": 0.720000, + "Standard_D16pls_v5": 0.627000, + "Standard_D16ps_v5": 0.768000, + "Standard_D16s_v3": 1.000000, + "Standard_D16s_v4": 0.960000, + "Standard_D16s_v5": 0.960000, + "Standard_D1_v2": 0.078900, + "Standard_D2": 0.196000, + "Standard_D2_v2": 0.158000, + "Standard_D2_v2_Promo": 0.158000, + "Standard_D2_v3": 0.125000, + "Standard_D2_v4": 0.120000, + "Standard_D2_v5": 0.120000, + "Standard_D2a_v4": 0.120000, + "Standard_D2ads_v5": 0.129000, + "Standard_D2as_v4": 0.120000, + "Standard_D2as_v5": 0.108000, + "Standard_D2d_v4": 0.141000, + "Standard_D2d_v5": 0.141000, + "Standard_D2ds_v4": 0.141000, + "Standard_D2ds_v5": 0.141000, + "Standard_D2lds_v5": 0.112000, + "Standard_D2ls_v5": 0.098000, + "Standard_D2pds_v5": 0.113000, + "Standard_D2plds_v5": 0.090000, + "Standard_D2pls_v5": 0.078400, + "Standard_D2ps_v5": 0.096000, + "Standard_D2s_v3": 0.125000, + "Standard_D2s_v4": 0.120000, + "Standard_D2s_v5": 0.120000, + "Standard_D3": 0.392000, + "Standard_D32-16s_v3": 2.000000, + "Standard_D32-8s_v3": 2.000000, + "Standard_D32_v3": 2.000000, + "Standard_D32_v4": 1.920000, + "Standard_D32_v5": 1.920000, + "Standard_D32a_v4": 1.920000, + "Standard_D32ads_v5": 3.536000, + "Standard_D32as_v4": 1.920000, + "Standard_D32as_v5": 3.200000, + "Standard_D32d_v4": 2.256000, + "Standard_D32d_v5": 2.256000, + "Standard_D32ds_v4": 2.256000, + "Standard_D32ds_v5": 2.256000, + "Standard_D32lds_v5": 1.792000, + "Standard_D32ls_v5": 1.568000, + "Standard_D32pds_v5": 1.808000, + "Standard_D32plds_v5": 1.440000, + "Standard_D32pls_v5": 1.254000, + "Standard_D32ps_v5": 1.536000, + "Standard_D32s_v3": 2.000000, + "Standard_D32s_v4": 1.920000, + "Standard_D32s_v5": 1.920000, + "Standard_D3_v2": 0.316000, + "Standard_D3_v2_Promo": 0.316000, + "Standard_D4": 0.784000, + "Standard_D48_v3": 3.000000, + "Standard_D48_v4": 2.880000, + "Standard_D48_v5": 2.880000, + "Standard_D48a_v4": 2.880000, + "Standard_D48ads_v5": 5.304000, + "Standard_D48as_v4": 2.880000, + "Standard_D48as_v5": 4.800000, + "Standard_D48d_v4": 3.384000, + "Standard_D48d_v5": 3.384000, + "Standard_D48ds_v4": 3.384000, + "Standard_D48ds_v5": 3.384000, + "Standard_D48lds_v5": 2.688000, + "Standard_D48ls_v5": 2.352000, + "Standard_D48pds_v5": 2.712000, + "Standard_D48plds_v5": 2.160000, + "Standard_D48pls_v5": 1.882000, + "Standard_D48ps_v5": 2.304000, + "Standard_D48s_v3": 3.000000, + "Standard_D48s_v4": 2.880000, + "Standard_D48s_v5": 2.880000, + "Standard_D4_v2": 0.631000, + "Standard_D4_v2_Promo": 0.631000, + "Standard_D4_v3": 0.250000, + "Standard_D4_v4": 0.240000, + "Standard_D4_v5": 0.240000, + "Standard_D4a_v4": 0.240000, + "Standard_D4ads_v5": 0.258000, + "Standard_D4as_v4": 0.240000, + "Standard_D4as_v5": 0.216000, + "Standard_D4d_v4": 0.282000, + "Standard_D4d_v5": 0.282000, + "Standard_D4ds_v4": 0.282000, + "Standard_D4ds_v5": 0.282000, + "Standard_D4lds_v5": 0.224000, + "Standard_D4ls_v5": 0.196000, + "Standard_D4pds_v5": 0.226000, + "Standard_D4plds_v5": 0.180000, + "Standard_D4pls_v5": 0.157000, + "Standard_D4ps_v5": 0.192000, + "Standard_D4s_v3": 0.250000, + "Standard_D4s_v4": 0.240000, + "Standard_D4s_v5": 0.240000, + "Standard_D5_v2": 1.263000, + "Standard_D5_v2_Promo": 1.263000, + "Standard_D64-16s_v3": 4.000000, + "Standard_D64-32s_v3": 4.000000, + "Standard_D64_v3": 4.000000, + "Standard_D64_v4": 3.840000, + "Standard_D64_v5": 3.840000, + "Standard_D64a_v4": 3.840000, + "Standard_D64ads_v5": 4.128000, + "Standard_D64as_v4": 3.840000, + "Standard_D64as_v5": 3.456000, + "Standard_D64d_v4": 4.512000, + "Standard_D64d_v5": 4.512000, + "Standard_D64ds_v4": 4.512000, + "Standard_D64ds_v5": 4.512000, + "Standard_D64lds_v5": 3.584000, + "Standard_D64ls_v5": 3.136000, + "Standard_D64pds_v5": 3.616000, + "Standard_D64plds_v5": 2.880000, + "Standard_D64pls_v5": 2.509000, + "Standard_D64ps_v5": 3.072000, + "Standard_D64s_v3": 4.000000, + "Standard_D64s_v4": 3.840000, + "Standard_D64s_v5": 3.840000, + "Standard_D8_v3": 0.500000, + "Standard_D8_v4": 0.480000, + "Standard_D8_v5": 0.480000, + "Standard_D8a_v4": 0.480000, + "Standard_D8ads_v5": 0.884000, + "Standard_D8as_v4": 0.480000, + "Standard_D8as_v5": 0.800000, + "Standard_D8d_v4": 0.564000, + "Standard_D8d_v5": 0.564000, + "Standard_D8ds_v4": 0.564000, + "Standard_D8ds_v5": 0.564000, + "Standard_D8lds_v5": 0.448000, + "Standard_D8ls_v5": 0.392000, + "Standard_D8pds_v5": 0.452000, + "Standard_D8plds_v5": 0.360000, + "Standard_D8pls_v5": 0.314000, + "Standard_D8ps_v5": 0.384000, + "Standard_D8s_v3": 0.500000, + "Standard_D8s_v4": 0.480000, + "Standard_D8s_v5": 0.480000, + "Standard_D96_v5": 5.760000, + "Standard_D96a_v4": 5.760000, + "Standard_D96ads_v5": 10.608000, + "Standard_D96as_v4": 5.760000, + "Standard_D96as_v5": 5.184000, + "Standard_D96d_v5": 6.768000, + "Standard_D96ds_v5": 6.768000, + "Standard_D96lds_v5": 5.376000, + "Standard_D96ls_v5": 4.704000, + "Standard_D96s_v5": 5.760000, + "Standard_DC16ads_cc_v5": 1.032000, + "Standard_DC16ads_v5": 1.032000, + "Standard_DC16as_cc_v5": 0.864000, + "Standard_DC16as_v5": 0.864000, + "Standard_DC16ds_v3": 2.256000, + "Standard_DC16s_v3": 1.920000, + "Standard_DC1ds_v3": 0.141000, + "Standard_DC1s_v2": 0.120000, + "Standard_DC1s_v3": 0.120000, + "Standard_DC24ds_v3": 3.384000, + "Standard_DC24s_v3": 2.880000, + "Standard_DC2ads_v5": 0.129000, + "Standard_DC2as_v5": 0.108000, + "Standard_DC2ds_v3": 0.282000, + "Standard_DC2s_v2": 0.240000, + "Standard_DC2s_v3": 0.240000, + "Standard_DC32ads_cc_v5": 2.064000, + "Standard_DC32ads_v5": 2.064000, + "Standard_DC32as_cc_v5": 1.728000, + "Standard_DC32as_v5": 1.728000, + "Standard_DC32ds_v3": 4.512000, + "Standard_DC32s_v3": 3.840000, + "Standard_DC48ads_cc_v5": 3.096000, + "Standard_DC48ads_v5": 3.096000, + "Standard_DC48as_cc_v5": 2.592000, + "Standard_DC48as_v5": 2.592000, + "Standard_DC48ds_v3": 6.768000, + "Standard_DC48s_v3": 5.760000, + "Standard_DC4ads_cc_v5": 0.258000, + "Standard_DC4ads_v5": 0.258000, + "Standard_DC4as_cc_v5": 0.216000, + "Standard_DC4as_v5": 0.216000, + "Standard_DC4ds_v3": 0.564000, + "Standard_DC4s_v2": 0.480000, + "Standard_DC4s_v3": 0.480000, + "Standard_DC64ads_cc_v5": 4.128000, + "Standard_DC64ads_v5": 4.128000, + "Standard_DC64as_cc_v5": 3.456000, + "Standard_DC64as_v5": 3.456000, + "Standard_DC8_v2": 0.960000, + "Standard_DC8ads_cc_v5": 0.516000, + "Standard_DC8ads_v5": 0.516000, + "Standard_DC8as_cc_v5": 0.432000, + "Standard_DC8as_v5": 0.432000, + "Standard_DC8ds_v3": 1.128000, + "Standard_DC8s_v3": 0.960000, + "Standard_DC96ads_cc_v5": 6.192000, + "Standard_DC96ads_v5": 6.192000, + "Standard_DC96as_cc_v5": 5.184000, + "Standard_DC96as_v5": 5.184000, + "Standard_DS1": 0.098000, + "Standard_DS11": 0.210000, + "Standard_DS11-1_v2": 0.191000, + "Standard_DS11_v2": 0.191000, + "Standard_DS11_v2_Promo": 0.191000, + "Standard_DS12": 0.420000, + "Standard_DS12-1_v2": 0.382000, + "Standard_DS12-2_v2": 0.382000, + "Standard_DS12_v2": 0.382000, + "Standard_DS12_v2_Promo": 0.382000, + "Standard_DS13": 0.840000, + "Standard_DS13-2_v2": 0.765000, + "Standard_DS13-4_v2": 0.765000, + "Standard_DS13_v2": 0.765000, + "Standard_DS13_v2_Promo": 0.765000, + "Standard_DS14": 1.680000, + "Standard_DS14-4_v2": 1.530000, + "Standard_DS14-8_v2": 1.530000, + "Standard_DS14_v2": 1.530000, + "Standard_DS14_v2_Promo": 1.530000, + "Standard_DS15_v2": 1.912000, + "Standard_DS15i_v2": 1.912000, + "Standard_DS1_v2": 0.078900, + "Standard_DS2": 0.196000, + "Standard_DS2_v2": 0.158000, + "Standard_DS2_v2_Promo": 0.158000, + "Standard_DS3": 0.392000, + "Standard_DS3_v2": 0.316000, + "Standard_DS3_v2_Promo": 0.316000, + "Standard_DS4": 0.784000, + "Standard_DS4_v2": 0.631000, + "Standard_DS4_v2_Promo": 0.631000, + "Standard_DS5_v2": 1.263000, + "Standard_DS5_v2_Promo": 1.263000, + "Standard_E104i_v5": 8.694000, + "Standard_E104id_v5": 9.953000, + "Standard_E104ids_v5": 9.953000, + "Standard_E104is_v5": 8.694000, + "Standard_E112iads_v5": 9.794000, + "Standard_E112ias_v5": 8.378000, + "Standard_E112ibds_v5": 12.382000, + "Standard_E112ibs_v5": 10.965000, + "Standard_E16-4ads_v5": 2.008000, + "Standard_E16-4as_v4": 1.216000, + "Standard_E16-4as_v5": 1.824000, + "Standard_E16-4ds_v4": 1.392000, + "Standard_E16-4ds_v5": 1.392000, + "Standard_E16-4s_v3": 1.280000, + "Standard_E16-4s_v4": 1.216000, + "Standard_E16-4s_v5": 1.216000, + "Standard_E16-8ads_v5": 2.008000, + "Standard_E16-8as_v4": 1.216000, + "Standard_E16-8as_v5": 1.088000, + "Standard_E16-8ds_v4": 1.392000, + "Standard_E16-8ds_v5": 1.392000, + "Standard_E16-8s_v3": 1.280000, + "Standard_E16-8s_v4": 1.216000, + "Standard_E16-8s_v5": 1.216000, + "Standard_E16_v3": 1.280000, + "Standard_E16_v4": 1.216000, + "Standard_E16_v5": 1.216000, + "Standard_E16a_v4": 1.216000, + "Standard_E16ads_v5": 1.318000, + "Standard_E16as_v4": 1.216000, + "Standard_E16as_v5": 1.134000, + "Standard_E16bds_v5": 1.608000, + "Standard_E16bs_v5": 1.424000, + "Standard_E16d_v4": 1.392000, + "Standard_E16d_v5": 1.392000, + "Standard_E16ds_v4": 1.392000, + "Standard_E16ds_v5": 1.392000, + "Standard_E16pds_v5": 1.112000, + "Standard_E16ps_v5": 0.973000, + "Standard_E16s_v3": 1.280000, + "Standard_E16s_v4": 1.216000, + "Standard_E16s_v5": 1.216000, + "Standard_E20_v3": 1.600000, + "Standard_E20_v4": 1.520000, + "Standard_E20_v5": 1.520000, + "Standard_E20a_v4": 1.520000, + "Standard_E20ads_v5": 1.590000, + "Standard_E20as_v4": 1.520000, + "Standard_E20as_v5": 1.406000, + "Standard_E20d_v4": 1.740000, + "Standard_E20d_v5": 1.740000, + "Standard_E20ds_v4": 1.740000, + "Standard_E20ds_v5": 1.740000, + "Standard_E20pds_v5": 1.390000, + "Standard_E20ps_v5": 1.216000, + "Standard_E20s_v3": 1.600000, + "Standard_E20s_v4": 1.520000, + "Standard_E20s_v5": 1.520000, + "Standard_E2_v3": 0.160000, + "Standard_E2_v4": 0.152000, + "Standard_E2_v5": 0.152000, + "Standard_E2a_v4": 0.152000, + "Standard_E2ads_v5": 0.159000, + "Standard_E2as_v4": 0.152000, + "Standard_E2as_v5": 0.136000, + "Standard_E2bds_v5": 0.201000, + "Standard_E2bs_v5": 0.178000, + "Standard_E2d_v4": 0.174000, + "Standard_E2d_v5": 0.174000, + "Standard_E2ds_v4": 0.174000, + "Standard_E2ds_v5": 0.174000, + "Standard_E2pds_v5": 0.139000, + "Standard_E2ps_v5": 0.122000, + "Standard_E2s_v3": 0.160000, + "Standard_E2s_v4": 0.152000, + "Standard_E2s_v5": 0.152000, + "Standard_E32-16ads_v5": 4.016000, + "Standard_E32-16as_v4": 2.432000, + "Standard_E32-16as_v5": 3.648000, + "Standard_E32-16ds_v4": 2.784000, + "Standard_E32-16ds_v5": 2.784000, + "Standard_E32-16s_v3": 2.560000, + "Standard_E32-16s_v4": 2.432000, + "Standard_E32-16s_v5": 2.432000, + "Standard_E32-8ads_v5": 4.016000, + "Standard_E32-8as_v4": 2.432000, + "Standard_E32-8as_v5": 3.648000, + "Standard_E32-8ds_v4": 2.784000, + "Standard_E32-8ds_v5": 2.784000, + "Standard_E32-8s_v3": 2.560000, + "Standard_E32-8s_v4": 2.432000, + "Standard_E32-8s_v5": 2.432000, + "Standard_E32_v3": 2.560000, + "Standard_E32_v4": 2.432000, + "Standard_E32_v5": 2.432000, + "Standard_E32a_v4": 2.432000, + "Standard_E32ads_v5": 2.590000, + "Standard_E32as_v4": 2.432000, + "Standard_E32as_v5": 2.222000, + "Standard_E32bds_v5": 3.216000, + "Standard_E32bs_v5": 2.848000, + "Standard_E32d_v4": 2.784000, + "Standard_E32d_v5": 2.784000, + "Standard_E32ds_v4": 2.784000, + "Standard_E32ds_v5": 2.784000, + "Standard_E32pds_v5": 2.224000, + "Standard_E32ps_v5": 1.946000, + "Standard_E32s_v3": 2.560000, + "Standard_E32s_v4": 2.432000, + "Standard_E32s_v5": 2.432000, + "Standard_E4-2ads_v5": 0.318000, + "Standard_E4-2as_v4": 0.304000, + "Standard_E4-2as_v5": 0.456000, + "Standard_E4-2ds_v4": 0.348000, + "Standard_E4-2ds_v5": 0.348000, + "Standard_E4-2s_v3": 0.320000, + "Standard_E4-2s_v4": 0.304000, + "Standard_E4-2s_v5": 0.304000, + "Standard_E48_v3": 3.840000, + "Standard_E48_v4": 3.648000, + "Standard_E48_v5": 3.648000, + "Standard_E48a_v4": 3.648000, + "Standard_E48ads_v5": 3.816000, + "Standard_E48as_v4": 3.648000, + "Standard_E48as_v5": 3.264000, + "Standard_E48bds_v5": 4.824000, + "Standard_E48bs_v5": 4.272000, + "Standard_E48d_v4": 4.176000, + "Standard_E48d_v5": 4.176000, + "Standard_E48ds_v4": 4.176000, + "Standard_E48ds_v5": 4.176000, + "Standard_E48s_v3": 3.840000, + "Standard_E48s_v4": 3.648000, + "Standard_E48s_v5": 3.648000, + "Standard_E4_v3": 0.320000, + "Standard_E4_v4": 0.304000, + "Standard_E4_v5": 0.304000, + "Standard_E4a_v4": 0.304000, + "Standard_E4ads_v5": 0.318000, + "Standard_E4as_v4": 0.304000, + "Standard_E4as_v5": 0.272000, + "Standard_E4bds_v5": 0.402000, + "Standard_E4bs_v5": 0.356000, + "Standard_E4d_v4": 0.348000, + "Standard_E4d_v5": 0.348000, + "Standard_E4ds_v4": 0.348000, + "Standard_E4ds_v5": 0.348000, + "Standard_E4pds_v5": 0.278000, + "Standard_E4ps_v5": 0.243000, + "Standard_E4s_v3": 0.320000, + "Standard_E4s_v4": 0.304000, + "Standard_E4s_v5": 0.304000, + "Standard_E64-16ads_v5": 8.032000, + "Standard_E64-16as_v4": 4.864000, + "Standard_E64-16as_v5": 7.296000, + "Standard_E64-16ds_v4": 5.568000, + "Standard_E64-16ds_v5": 5.568000, + "Standard_E64-16s_v3": 4.609000, + "Standard_E64-16s_v4": 4.864000, + "Standard_E64-16s_v5": 4.864000, + "Standard_E64-32ads_v5": 8.032000, + "Standard_E64-32as_v4": 4.864000, + "Standard_E64-32as_v5": 7.296000, + "Standard_E64-32ds_v4": 5.568000, + "Standard_E64-32ds_v5": 5.568000, + "Standard_E64-32s_v3": 4.609000, + "Standard_E64-32s_v4": 4.864000, + "Standard_E64-32s_v5": 4.864000, + "Standard_E64_v3": 4.609000, + "Standard_E64_v4": 4.864000, + "Standard_E64_v5": 4.864000, + "Standard_E64a_v4": 4.864000, + "Standard_E64ads_v5": 5.134000, + "Standard_E64as_v4": 4.864000, + "Standard_E64as_v5": 4.398000, + "Standard_E64bds_v5": 6.432000, + "Standard_E64bs_v5": 5.696000, + "Standard_E64d_v4": 5.568000, + "Standard_E64d_v5": 5.568000, + "Standard_E64ds_v4": 5.568000, + "Standard_E64ds_v5": 5.568000, + "Standard_E64i_v3": 4.609000, + "Standard_E64is_v3": 4.609000, + "Standard_E64s_v3": 4.609000, + "Standard_E64s_v4": 4.864000, + "Standard_E64s_v5": 4.864000, + "Standard_E8-2ads_v5": 0.636000, + "Standard_E8-2as_v4": 0.608000, + "Standard_E8-2as_v5": 0.912000, + "Standard_E8-2ds_v4": 0.696000, + "Standard_E8-2ds_v5": 0.696000, + "Standard_E8-2s_v3": 0.640000, + "Standard_E8-2s_v4": 0.608000, + "Standard_E8-2s_v5": 0.608000, + "Standard_E8-4ads_v5": 1.004000, + "Standard_E8-4as_v4": 0.608000, + "Standard_E8-4as_v5": 0.912000, + "Standard_E8-4ds_v4": 0.696000, + "Standard_E8-4ds_v5": 0.696000, + "Standard_E8-4s_v3": 0.640000, + "Standard_E8-4s_v4": 0.608000, + "Standard_E8-4s_v5": 0.608000, + "Standard_E80ids_v4": 6.960000, + "Standard_E80is_v4": 6.080000, + "Standard_E8_v3": 0.640000, + "Standard_E8_v4": 0.608000, + "Standard_E8_v5": 0.608000, + "Standard_E8a_v4": 0.608000, + "Standard_E8ads_v5": 0.636000, + "Standard_E8as_v4": 0.608000, + "Standard_E8as_v5": 0.544000, + "Standard_E8bds_v5": 0.804000, + "Standard_E8bs_v5": 0.712000, + "Standard_E8d_v4": 0.696000, + "Standard_E8d_v5": 0.696000, + "Standard_E8ds_v4": 0.696000, + "Standard_E8ds_v5": 0.696000, + "Standard_E8pds_v5": 0.556000, + "Standard_E8ps_v5": 0.486000, + "Standard_E8s_v3": 0.640000, + "Standard_E8s_v4": 0.608000, + "Standard_E8s_v5": 0.608000, + "Standard_E96-24ads_v5": 7.632000, + "Standard_E96-24as_v4": 7.296000, + "Standard_E96-24as_v5": 10.944000, + "Standard_E96-24ds_v5": 8.352000, + "Standard_E96-24s_v5": 7.296000, + "Standard_E96-48ads_v5": 12.048000, + "Standard_E96-48as_v4": 7.296000, + "Standard_E96-48as_v5": 10.944000, + "Standard_E96-48ds_v5": 8.352000, + "Standard_E96-48s_v5": 7.296000, + "Standard_E96_v5": 7.296000, + "Standard_E96a_v4": 7.296000, + "Standard_E96ads_v5": 7.632000, + "Standard_E96as_v4": 7.296000, + "Standard_E96as_v5": 6.574000, + "Standard_E96bds_v5": 9.648000, + "Standard_E96bs_v5": 8.544000, + "Standard_E96d_v5": 8.352000, + "Standard_E96ds_v5": 8.352000, + "Standard_E96iads_v5": 8.395000, + "Standard_E96ias_v5": 7.181000, + "Standard_E96s_v5": 7.296000, + "Standard_EC16ads_cc_v5": 1.272000, + "Standard_EC16ads_v5": 1.272000, + "Standard_EC16as_cc_v5": 1.088000, + "Standard_EC16as_v5": 1.088000, + "Standard_EC20ads_cc_v5": 1.590000, + "Standard_EC20ads_v5": 1.590000, + "Standard_EC20as_cc_v5": 1.360000, + "Standard_EC20as_v5": 1.360000, + "Standard_EC2ads_v5": 0.159000, + "Standard_EC2as_v5": 0.136000, + "Standard_EC32ads_cc_v5": 2.544000, + "Standard_EC32ads_v5": 2.544000, + "Standard_EC32as_cc_v5": 2.176000, + "Standard_EC32as_v5": 2.176000, + "Standard_EC48ads_cc_v5": 3.816000, + "Standard_EC48ads_v5": 3.816000, + "Standard_EC48as_cc_v5": 3.264000, + "Standard_EC48as_v5": 3.264000, + "Standard_EC4ads_cc_v5": 0.318000, + "Standard_EC4ads_v5": 0.318000, + "Standard_EC4as_cc_v5": 0.272000, + "Standard_EC4as_v5": 0.272000, + "Standard_EC64ads_cc_v5": 5.088000, + "Standard_EC64ads_v5": 5.088000, + "Standard_EC64as_cc_v5": 4.352000, + "Standard_EC64as_v5": 4.352000, + "Standard_EC8ads_cc_v5": 0.636000, + "Standard_EC8ads_v5": 0.636000, + "Standard_EC8as_cc_v5": 0.544000, + "Standard_EC8as_v5": 0.544000, + "Standard_EC96ads_cc_v5": 7.632000, + "Standard_EC96ads_v5": 7.632000, + "Standard_EC96as_cc_v5": 6.528000, + "Standard_EC96as_v5": 6.528000, + "Standard_EC96iads_v5": 8.395000, + "Standard_EC96ias_v5": 7.181000, + "Standard_F1": 0.057700, + "Standard_F16": 0.924000, + "Standard_F16s": 0.924000, + "Standard_F16s_v2": 0.784000, + "Standard_F1s": 0.057700, + "Standard_F2": 0.115000, + "Standard_F2s": 0.115000, + "Standard_F2s_v2": 0.098000, + "Standard_F32s_v2": 1.568000, + "Standard_F4": 0.231000, + "Standard_F48s_v2": 2.352000, + "Standard_F4s": 0.231000, + "Standard_F4s_v2": 0.196000, + "Standard_F64s_v2": 3.136000, + "Standard_F72s_v2": 3.528000, + "Standard_F8": 0.462000, + "Standard_F8s": 0.462000, + "Standard_F8s_v2": 0.392000, + "Standard_FX12mds": 1.356000, + "Standard_FX24mds": 2.712000, + "Standard_FX36mds": 4.068000, + "Standard_FX48mds": 5.424000, + "Standard_FX4mds": 0.452000, + "Standard_G1": 0.660000, + "Standard_G2": 1.320000, + "Standard_G3": 2.640000, + "Standard_G4": 5.280000, + "Standard_G5": 9.390000, + "Standard_GS1": 0.660000, + "Standard_GS2": 1.320000, + "Standard_GS3": 2.640000, + "Standard_GS4": 5.280000, + "Standard_GS4-4": 5.280000, + "Standard_GS4-8": 5.280000, + "Standard_GS5": 9.390000, + "Standard_GS5-16": 9.390000, + "Standard_GS5-8": 9.390000, + "Standard_H16": 1.848000, + "Standard_H16m": 2.476000, + "Standard_H16mr": 2.724000, + "Standard_H16r": 2.033000, + "Standard_H8": 0.924000, + "Standard_H8m": 1.238000, + "Standard_HC44-16rs": 4.118000, + "Standard_HC44-32rs": 4.118000, + "Standard_HC44rs": 4.118000, + "Standard_L112ias_v3": 11.519000, + "Standard_L16as_v3": 1.496000, + "Standard_L16s": 1.496000, + "Standard_L16s_v2": 1.496000, + "Standard_L16s_v3": 1.672000, + "Standard_L32as_v3": 2.992000, + "Standard_L32s": 2.992000, + "Standard_L32s_v2": 2.992000, + "Standard_L32s_v3": 3.344000, + "Standard_L48as_v3": 4.488000, + "Standard_L48s_v2": 4.488000, + "Standard_L48s_v3": 5.016000, + "Standard_L4s": 0.374000, + "Standard_L64as_v3": 5.984000, + "Standard_L64s_v2": 5.984000, + "Standard_L64s_v3": 6.688000, + "Standard_L80as_v3": 7.480000, + "Standard_L80s_v2": 7.480000, + "Standard_L80s_v3": 8.360000, + "Standard_L88is_v2": 8.228000, + "Standard_L8as_v3": 0.748000, + "Standard_L8s": 0.748000, + "Standard_L8s_v2": 0.748000, + "Standard_L8s_v3": 0.836000, + "Standard_L96ias_v3": 9.874000, + "Standard_M128": 19.340000, + "Standard_M128-32ms": 38.698000, + "Standard_M128-64ms": 38.698000, + "Standard_M128dms_v2": 38.701000, + "Standard_M128ds_v2": 19.343000, + "Standard_M128m": 38.698000, + "Standard_M128ms": 38.698000, + "Standard_M128ms_v2": 38.090000, + "Standard_M128s": 19.340000, + "Standard_M128s_v2": 18.733000, + "Standard_M16-4ms": 4.455900, + "Standard_M16-8ms": 4.455900, + "Standard_M16ms": 4.455900, + "Standard_M16s": 3.461000, + "Standard_M192idms_v2": 46.493000, + "Standard_M192ids_v2": 23.246000, + "Standard_M192ims_v2": 45.882000, + "Standard_M192is_v2": 22.636000, + "Standard_M208-104ms_v2": 58.006000, + "Standard_M208-104s_v2": 29.003000, + "Standard_M208-52ms_v2": 58.006000, + "Standard_M208-52s_v2": 29.003000, + "Standard_M208ms_v2": 58.006000, + "Standard_M208s_v2": 29.003000, + "Standard_M32-16ms": 8.911700, + "Standard_M32-8ms": 8.911700, + "Standard_M32dms_v2": 8.912000, + "Standard_M32ls": 3.447600, + "Standard_M32ms": 8.911700, + "Standard_M32ms_v2": 8.759000, + "Standard_M32s": 4.835800, + "Standard_M32ts": 3.248000, + "Standard_M416-104ms_v2": 128.900000, + "Standard_M416-104s_v2": 64.454000, + "Standard_M416-208ms_v2": 128.900000, + "Standard_M416-208s_v2": 64.454000, + "Standard_M416is_v2": 64.454000, + "Standard_M416ms_v2": 128.900000, + "Standard_M416s_8_v2": 79.730000, + "Standard_M416s_v2": 64.454000, + "Standard_M64": 9.670000, + "Standard_M64-16ms": 14.989000, + "Standard_M64-32ms": 14.989000, + "Standard_M64dms_v2": 14.993000, + "Standard_M64ds_v2": 9.670000, + "Standard_M64ls": 6.498000, + "Standard_M64m": 14.989000, + "Standard_M64ms": 14.989000, + "Standard_M64ms_v2": 14.688000, + "Standard_M64s": 9.670000, + "Standard_M64s_v2": 9.365000, + "Standard_M8-2ms": 2.227900, + "Standard_M8-4ms": 2.227900, + "Standard_M8ms": 2.227900, + "Standard_NC12": 2.642000, + "Standard_NC12s_v2": 6.077000, + "Standard_NC12s_v3": 8.468000, + "Standard_NC16as_T4_v3": 1.685000, + "Standard_NC24": 5.283000, + "Standard_NC24ads_A100_v4": 4.775000, + "Standard_NC24r": 5.813000, + "Standard_NC24rs_v2": 13.369000, + "Standard_NC24rs_v3": 18.630000, + "Standard_NC24s_v2": 12.153000, + "Standard_NC24s_v3": 16.936000, + "Standard_NC48ads_A100_v4": 9.550000, + "Standard_NC4as_T4_v3": 0.736000, + "Standard_NC6": 1.321000, + "Standard_NC64as_T4_v3": 6.089000, + "Standard_NC6s_v2": 3.038000, + "Standard_NC6s_v3": 4.234000, + "Standard_NC8as_T4_v3": 1.052000, + "Standard_NC96ads_A100_v4": 19.100000, + "Standard_ND12s": 6.077000, + "Standard_ND24rs": 13.369000, + "Standard_ND24s": 12.153000, + "Standard_ND40rs_v2": 30.492000, + "Standard_ND40s_v2": 23.366000, + "Standard_ND6s": 3.038000, + "Standard_NP10s": 2.145000, + "Standard_NP20s": 4.290000, + "Standard_NP40s": 8.580000, + "Standard_NV12": 3.179000, + "Standard_NV12ads_A10_v5": 1.180000, + "Standard_NV12s_v3": 1.590000, + "Standard_NV16as_v4": 1.212000, + "Standard_NV18ads_A10_v5": 2.080000, + "Standard_NV24": 6.358000, + "Standard_NV24s_v3": 3.180000, + "Standard_NV32as_v4": 2.423000, + "Standard_NV36adms_A10_v5": 5.876000, + "Standard_NV36ads_A10_v5": 4.160000, + "Standard_NV48s_v3": 6.360000, + "Standard_NV4as_v4": 0.303000, + "Standard_NV6": 1.590000, + "Standard_NV6ads_A10_v5": 0.590000, + "Standard_NV72ads_A10_v5": 8.476000, + "Standard_NV8as_v4": 0.606000, + "Standard_PB12s": 2.158000, + "Standard_PB24s": 4.316000, + "Standard_PB6s": 1.079000, + } + // southindia + initialOnDemandPrices["southindia"] = map[string]float64{ + "Basic_A0": 0.016000, + "Basic_A1": 0.033000, + "Basic_A2": 0.094000, + "Basic_A3": 0.209000, + "Basic_A4": 0.418000, + "DCdsv3 Type1": 8.413000, + "DCsv3 Type1": 7.147000, + "Dadsv5_Type1": 8.946000, + "Dasv4_Type1": 7.445000, + "Dasv4_Type2": 7.445000, + "Dasv5_Type1": 7.470000, + "Ddsv4_Type 1": 5.608000, + "Ddsv4_Type2": 6.660000, + "Ddsv5_Type1": 8.413000, + "Dsv3_Type3": 4.888000, + "Dsv3_Type4": 6.110000, + "Dsv5_Type1": 7.445000, + "Eadsv5_Type1": 9.753000, + "Easv4_Type1": 9.380000, + "Easv4_Type2": 9.380000, + "Easv5_Type1": 8.413000, + "Ebdsv5-Type1": 8.289000, + "Ebsv5-Type1": 7.395000, + "Edsv5_Type1": 10.721000, + "Esv3_Type3": 5.685000, + "Esv3_Type4": 7.702000, + "Esv5_Type1": 9.380000, + "Fsv2 Type3": 4.731000, + "Fsv2_Type2": 3.703000, + "Fsv2_Type4": 5.142000, + "Lasv3_Type1": 9.678000, + "Lsv3_Type1": 10.826000, + "Mdmsv2MedMem _Type1": 44.332000, + "Mdsv2MedMem_Type1": 22.158000, + "Msmv2MedMem Type1": 43.633000, + "Msmv2_Type1": 164.690000, + "Msv2MedMem Type1": 21.459000, + "Msv2_Type1": 82.352000, + "Standard_A0": 0.018000, + "Standard_A1": 0.059000, + "Standard_A1_v2": 0.047000, + "Standard_A2": 0.118000, + "Standard_A2_v2": 0.098000, + "Standard_A2m_v2": 0.144000, + "Standard_A3": 0.236000, + "Standard_A4": 0.472000, + "Standard_A4_v2": 0.206000, + "Standard_A4m_v2": 0.301000, + "Standard_A5": 0.234000, + "Standard_A6": 0.468000, + "Standard_A7": 0.936000, + "Standard_A8_v2": 0.433000, + "Standard_A8m_v2": 0.633000, + "Standard_B12ms": 0.706000, + "Standard_B16als_v2": 0.751000, + "Standard_B16as_v2": 0.922000, + "Standard_B16ls_v2": 0.905000, + "Standard_B16ms": 0.941000, + "Standard_B16s_v2": 0.938000, + "Standard_B1ls": 0.007800, + "Standard_B1ms": 0.029400, + "Standard_B1s": 0.014500, + "Standard_B20ms": 1.177000, + "Standard_B2als_v2": 0.053000, + "Standard_B2as_v2": 0.106000, + "Standard_B2ats_v2": 0.013300, + "Standard_B2ls_v2": 0.067900, + "Standard_B2ms": 0.118000, + "Standard_B2s": 0.058000, + "Standard_B2s_v2": 0.127000, + "Standard_B2ts_v2": 0.014700, + "Standard_B32als_v2": 1.649000, + "Standard_B32as_v2": 1.844000, + "Standard_B32ls_v2": 1.662000, + "Standard_B32s_v2": 2.024000, + "Standard_B4als_v2": 0.206000, + "Standard_B4as_v2": 0.212000, + "Standard_B4ls_v2": 0.226000, + "Standard_B4ms": 0.236000, + "Standard_B4s_v2": 0.235000, + "Standard_B8als_v2": 0.376000, + "Standard_B8as_v2": 0.424000, + "Standard_B8ls_v2": 0.452000, + "Standard_B8ms": 0.471000, + "Standard_B8s_v2": 0.506000, + "Standard_D1": 0.094400, + "Standard_D11": 0.244000, + "Standard_D11_v2": 0.189000, + "Standard_D11_v2_Promo": 0.189000, + "Standard_D12": 0.489000, + "Standard_D12_v2": 0.379000, + "Standard_D12_v2_Promo": 0.379000, + "Standard_D13": 0.977000, + "Standard_D13_v2": 0.758000, + "Standard_D13_v2_Promo": 0.758000, + "Standard_D14": 1.955000, + "Standard_D14_v2": 1.516000, + "Standard_D14_v2_Promo": 1.516000, + "Standard_D15_v2": 1.895000, + "Standard_D15i_v2": 1.895000, + "Standard_D16_v3": 1.082000, + "Standard_D16_v4": 1.082000, + "Standard_D16_v5": 1.083000, + "Standard_D16a_v4": 1.083000, + "Standard_D16ads_v5": 1.898000, + "Standard_D16as_v4": 1.083000, + "Standard_D16as_v5": 1.706000, + "Standard_D16as_v5_Promo": 0.869000, + "Standard_D16d_v4": 1.274000, + "Standard_D16d_v5": 1.275000, + "Standard_D16ds_v4": 1.274000, + "Standard_D16ds_v5": 1.275000, + "Standard_D16lds_v5": 1.083000, + "Standard_D16ls_v5": 0.959000, + "Standard_D16s_v3": 1.082000, + "Standard_D16s_v4": 1.082000, + "Standard_D16s_v5": 1.083000, + "Standard_D1_v2": 0.084000, + "Standard_D2": 0.189000, + "Standard_D2_v2": 0.169000, + "Standard_D2_v2_Promo": 0.169000, + "Standard_D2_v3": 0.135000, + "Standard_D2_v4": 0.135000, + "Standard_D2_v5": 0.135000, + "Standard_D2a_v4": 0.135000, + "Standard_D2ads_v5": 0.237000, + "Standard_D2as_v4": 0.135000, + "Standard_D2as_v5": 0.213000, + "Standard_D2as_v5_Promo": 0.109000, + "Standard_D2d_v4": 0.159000, + "Standard_D2d_v5": 0.159000, + "Standard_D2ds_v4": 0.159000, + "Standard_D2ds_v5": 0.159000, + "Standard_D2lds_v5": 0.135000, + "Standard_D2ls_v5": 0.120000, + "Standard_D2s_v3": 0.135000, + "Standard_D2s_v4": 0.135000, + "Standard_D2s_v5": 0.135000, + "Standard_D3": 0.378000, + "Standard_D32-16s_v3": 2.164000, + "Standard_D32-8s_v3": 2.164000, + "Standard_D32_v3": 2.164000, + "Standard_D32_v4": 2.164000, + "Standard_D32_v5": 2.166000, + "Standard_D32a_v4": 2.166000, + "Standard_D32ads_v5": 3.796000, + "Standard_D32as_v4": 2.166000, + "Standard_D32as_v5": 1.940000, + "Standard_D32as_v5_Promo": 1.737000, + "Standard_D32d_v4": 2.547000, + "Standard_D32d_v5": 2.549000, + "Standard_D32ds_v4": 2.547000, + "Standard_D32ds_v5": 2.549000, + "Standard_D32lds_v5": 2.166000, + "Standard_D32ls_v5": 1.918000, + "Standard_D32s_v3": 2.164000, + "Standard_D32s_v4": 2.164000, + "Standard_D32s_v5": 2.166000, + "Standard_D3_v2": 0.337000, + "Standard_D3_v2_Promo": 0.337000, + "Standard_D4": 0.755000, + "Standard_D48_v3": 3.246000, + "Standard_D48_v4": 3.246000, + "Standard_D48_v5": 3.249000, + "Standard_D48a_v4": 3.249000, + "Standard_D48ads_v5": 3.486000, + "Standard_D48as_v4": 3.249000, + "Standard_D48as_v5": 2.910000, + "Standard_D48as_v5_Promo": 2.606000, + "Standard_D48d_v4": 3.821000, + "Standard_D48d_v5": 3.824000, + "Standard_D48ds_v4": 3.821000, + "Standard_D48ds_v5": 3.824000, + "Standard_D48lds_v5": 3.249000, + "Standard_D48ls_v5": 2.876000, + "Standard_D48s_v3": 3.246000, + "Standard_D48s_v4": 3.246000, + "Standard_D48s_v5": 3.249000, + "Standard_D4_v2": 0.675000, + "Standard_D4_v2_Promo": 0.675000, + "Standard_D4_v3": 0.271000, + "Standard_D4_v4": 0.271000, + "Standard_D4_v5": 0.271000, + "Standard_D4a_v4": 0.271000, + "Standard_D4ads_v5": 0.474000, + "Standard_D4as_v4": 0.271000, + "Standard_D4as_v5": 0.427000, + "Standard_D4as_v5_Promo": 0.217000, + "Standard_D4d_v4": 0.318000, + "Standard_D4d_v5": 0.319000, + "Standard_D4ds_v4": 0.318000, + "Standard_D4ds_v5": 0.319000, + "Standard_D4lds_v5": 0.271000, + "Standard_D4ls_v5": 0.240000, + "Standard_D4s_v3": 0.271000, + "Standard_D4s_v4": 0.271000, + "Standard_D4s_v5": 0.271000, + "Standard_D5_v2": 1.350000, + "Standard_D5_v2_Promo": 1.350000, + "Standard_D64-16s_v3": 4.328000, + "Standard_D64-32s_v3": 4.328000, + "Standard_D64_v3": 4.328000, + "Standard_D64_v4": 4.328000, + "Standard_D64_v5": 4.332000, + "Standard_D64a_v4": 4.332000, + "Standard_D64ads_v5": 4.647000, + "Standard_D64as_v4": 4.332000, + "Standard_D64as_v5": 3.880000, + "Standard_D64as_v5_Promo": 3.474000, + "Standard_D64d_v4": 5.095000, + "Standard_D64d_v5": 5.099000, + "Standard_D64ds_v4": 5.095000, + "Standard_D64ds_v5": 5.099000, + "Standard_D64lds_v5": 4.332000, + "Standard_D64ls_v5": 3.835000, + "Standard_D64s_v3": 4.328000, + "Standard_D64s_v4": 4.328000, + "Standard_D64s_v5": 4.332000, + "Standard_D8_v3": 0.541000, + "Standard_D8_v4": 0.541000, + "Standard_D8_v5": 0.541000, + "Standard_D8a_v4": 0.541000, + "Standard_D8ads_v5": 0.581000, + "Standard_D8as_v4": 0.541000, + "Standard_D8as_v5": 0.853000, + "Standard_D8as_v5_Promo": 0.434000, + "Standard_D8d_v4": 0.637000, + "Standard_D8d_v5": 0.637000, + "Standard_D8ds_v4": 0.637000, + "Standard_D8ds_v5": 0.637000, + "Standard_D8lds_v5": 0.541000, + "Standard_D8ls_v5": 0.479000, + "Standard_D8s_v3": 0.541000, + "Standard_D8s_v4": 0.541000, + "Standard_D8s_v5": 0.541000, + "Standard_D96_v5": 6.497000, + "Standard_D96a_v4": 6.497000, + "Standard_D96ads_v5": 6.971000, + "Standard_D96as_v4": 6.497000, + "Standard_D96as_v5": 5.820000, + "Standard_D96as_v5_Promo": 5.211000, + "Standard_D96d_v5": 7.648000, + "Standard_D96ds_v5": 7.648000, + "Standard_D96lds_v5": 6.497000, + "Standard_D96ls_v5": 5.753000, + "Standard_D96s_v5": 6.497000, + "Standard_DC16ds_v3": 2.549000, + "Standard_DC16s_v3": 2.166000, + "Standard_DC1ds_v3": 0.159000, + "Standard_DC1s_v3": 0.135000, + "Standard_DC24ds_v3": 3.824000, + "Standard_DC24s_v3": 3.249000, + "Standard_DC2ds_v3": 0.319000, + "Standard_DC2s_v3": 0.271000, + "Standard_DC32ds_v3": 5.099000, + "Standard_DC32s_v3": 4.332000, + "Standard_DC48ds_v3": 7.648000, + "Standard_DC48s_v3": 6.497000, + "Standard_DC4ds_v3": 0.637000, + "Standard_DC4s_v3": 0.541000, + "Standard_DC8ds_v3": 1.275000, + "Standard_DC8s_v3": 1.083000, + "Standard_DS1": 0.094400, + "Standard_DS11": 0.244000, + "Standard_DS11-1_v2": 0.189000, + "Standard_DS11_v2": 0.189000, + "Standard_DS11_v2_Promo": 0.189000, + "Standard_DS12": 0.489000, + "Standard_DS12-1_v2": 0.379000, + "Standard_DS12-2_v2": 0.379000, + "Standard_DS12_v2": 0.379000, + "Standard_DS12_v2_Promo": 0.379000, + "Standard_DS13": 0.977000, + "Standard_DS13-2_v2": 0.758000, + "Standard_DS13-4_v2": 0.758000, + "Standard_DS13_v2": 0.758000, + "Standard_DS13_v2_Promo": 0.758000, + "Standard_DS14": 1.955000, + "Standard_DS14-4_v2": 1.516000, + "Standard_DS14-8_v2": 1.516000, + "Standard_DS14_v2": 1.516000, + "Standard_DS14_v2_Promo": 1.516000, + "Standard_DS15_v2": 1.895000, + "Standard_DS15i_v2": 1.895000, + "Standard_DS1_v2": 0.084000, + "Standard_DS2": 0.189000, + "Standard_DS2_v2": 0.169000, + "Standard_DS2_v2_Promo": 0.169000, + "Standard_DS3": 0.378000, + "Standard_DS3_v2": 0.337000, + "Standard_DS3_v2_Promo": 0.337000, + "Standard_DS4": 0.755000, + "Standard_DS4_v2": 0.675000, + "Standard_DS4_v2_Promo": 0.675000, + "Standard_DS5_v2": 1.350000, + "Standard_DS5_v2_Promo": 1.350000, + "Standard_E104i_v5": 10.162000, + "Standard_E104id_v5": 11.614000, + "Standard_E104ids_v5": 11.614000, + "Standard_E104is_v5": 10.162000, + "Standard_E112iads_v5": 11.378000, + "Standard_E112ias_v5": 9.815000, + "Standard_E112ibds_v5": 14.505000, + "Standard_E112ibs_v5": 12.942000, + "Standard_E16-4ads_v5": 2.214000, + "Standard_E16-4as_v4": 1.421000, + "Standard_E16-4as_v5": 2.011000, + "Standard_E16-4ds_v4": 1.524000, + "Standard_E16-4ds_v5": 1.624000, + "Standard_E16-4s_v3": 1.334000, + "Standard_E16-4s_v4": 1.334000, + "Standard_E16-4s_v5": 1.421000, + "Standard_E16-8ads_v5": 1.478000, + "Standard_E16-8as_v4": 1.421000, + "Standard_E16-8as_v5": 1.275000, + "Standard_E16-8ds_v4": 1.524000, + "Standard_E16-8ds_v5": 1.624000, + "Standard_E16-8s_v3": 1.334000, + "Standard_E16-8s_v4": 1.334000, + "Standard_E16-8s_v5": 1.421000, + "Standard_E16_v3": 1.334000, + "Standard_E16_v4": 1.334000, + "Standard_E16_v5": 1.421000, + "Standard_E16a_v4": 1.421000, + "Standard_E16ads_v5": 1.524000, + "Standard_E16as_v4": 1.421000, + "Standard_E16as_v5": 1.321000, + "Standard_E16bds_v5": 1.884000, + "Standard_E16bs_v5": 1.681000, + "Standard_E16d_v4": 1.524000, + "Standard_E16d_v5": 1.624000, + "Standard_E16ds_v4": 1.524000, + "Standard_E16ds_v5": 1.624000, + "Standard_E16s_v3": 1.334000, + "Standard_E16s_v4": 1.334000, + "Standard_E16s_v5": 1.421000, + "Standard_E20_v3": 1.670000, + "Standard_E20_v4": 1.667000, + "Standard_E20_v5": 1.777000, + "Standard_E20a_v4": 1.777000, + "Standard_E20ads_v5": 1.847000, + "Standard_E20as_v4": 1.777000, + "Standard_E20as_v5": 1.639000, + "Standard_E20d_v4": 1.905000, + "Standard_E20d_v5": 2.030000, + "Standard_E20ds_v4": 1.905000, + "Standard_E20ds_v5": 2.030000, + "Standard_E20s_v3": 1.670000, + "Standard_E20s_v4": 1.667000, + "Standard_E20s_v5": 1.777000, + "Standard_E2_v3": 0.167000, + "Standard_E2_v4": 0.167000, + "Standard_E2_v5": 0.178000, + "Standard_E2a_v4": 0.178000, + "Standard_E2ads_v5": 0.231000, + "Standard_E2as_v4": 0.178000, + "Standard_E2as_v5": 0.159000, + "Standard_E2bds_v5": 0.235000, + "Standard_E2bs_v5": 0.210000, + "Standard_E2d_v4": 0.191000, + "Standard_E2d_v5": 0.203000, + "Standard_E2ds_v4": 0.191000, + "Standard_E2ds_v5": 0.203000, + "Standard_E2s_v3": 0.167000, + "Standard_E2s_v4": 0.167000, + "Standard_E2s_v5": 0.178000, + "Standard_E32-16ads_v5": 4.427000, + "Standard_E32-16as_v4": 2.843000, + "Standard_E32-16as_v5": 4.021000, + "Standard_E32-16ds_v4": 3.048000, + "Standard_E32-16ds_v5": 3.249000, + "Standard_E32-16s_v3": 2.667000, + "Standard_E32-16s_v4": 2.667000, + "Standard_E32-16s_v5": 2.843000, + "Standard_E32-8ads_v5": 4.427000, + "Standard_E32-8as_v4": 2.843000, + "Standard_E32-8as_v5": 2.549000, + "Standard_E32-8ds_v4": 3.048000, + "Standard_E32-8ds_v5": 3.249000, + "Standard_E32-8s_v3": 2.667000, + "Standard_E32-8s_v4": 2.667000, + "Standard_E32-8s_v5": 2.843000, + "Standard_E32_v3": 2.667000, + "Standard_E32_v4": 2.667000, + "Standard_E32_v5": 2.843000, + "Standard_E32a_v4": 2.843000, + "Standard_E32ads_v5": 2.955000, + "Standard_E32as_v4": 2.843000, + "Standard_E32as_v5": 2.595000, + "Standard_E32bds_v5": 3.768000, + "Standard_E32bs_v5": 3.361000, + "Standard_E32d_v4": 3.048000, + "Standard_E32d_v5": 3.249000, + "Standard_E32ds_v4": 3.048000, + "Standard_E32ds_v5": 3.249000, + "Standard_E32s_v3": 2.667000, + "Standard_E32s_v4": 2.667000, + "Standard_E32s_v5": 2.843000, + "Standard_E4-2ads_v5": 0.553000, + "Standard_E4-2as_v4": 0.355000, + "Standard_E4-2as_v5": 0.319000, + "Standard_E4-2ds_v4": 0.381000, + "Standard_E4-2ds_v5": 0.406000, + "Standard_E4-2s_v3": 0.334000, + "Standard_E4-2s_v4": 0.333000, + "Standard_E4-2s_v5": 0.355000, + "Standard_E48_v3": 4.001000, + "Standard_E48_v4": 4.001000, + "Standard_E48_v5": 4.264000, + "Standard_E48a_v4": 4.264000, + "Standard_E48ads_v5": 4.479000, + "Standard_E48as_v4": 4.264000, + "Standard_E48as_v5": 3.824000, + "Standard_E48bds_v5": 5.651000, + "Standard_E48bs_v5": 5.042000, + "Standard_E48d_v4": 4.572000, + "Standard_E48d_v5": 4.873000, + "Standard_E48ds_v4": 4.572000, + "Standard_E48ds_v5": 4.873000, + "Standard_E48s_v3": 4.001000, + "Standard_E48s_v4": 4.001000, + "Standard_E48s_v5": 4.264000, + "Standard_E4_v3": 0.334000, + "Standard_E4_v4": 0.333000, + "Standard_E4_v5": 0.355000, + "Standard_E4a_v4": 0.355000, + "Standard_E4ads_v5": 0.415000, + "Standard_E4as_v4": 0.355000, + "Standard_E4as_v5": 0.365000, + "Standard_E4bds_v5": 0.471000, + "Standard_E4bs_v5": 0.420000, + "Standard_E4d_v4": 0.381000, + "Standard_E4d_v5": 0.406000, + "Standard_E4ds_v4": 0.381000, + "Standard_E4ds_v5": 0.406000, + "Standard_E4s_v3": 0.334000, + "Standard_E4s_v4": 0.333000, + "Standard_E4s_v5": 0.355000, + "Standard_E64-16ads_v5": 8.855000, + "Standard_E64-16as_v4": 5.685000, + "Standard_E64-16as_v5": 5.099000, + "Standard_E64-16ds_v4": 6.096000, + "Standard_E64-16ds_v5": 6.497000, + "Standard_E64-16s_v3": 4.801000, + "Standard_E64-16s_v4": 5.334000, + "Standard_E64-16s_v5": 5.685000, + "Standard_E64-32ads_v5": 8.855000, + "Standard_E64-32as_v4": 5.685000, + "Standard_E64-32as_v5": 5.099000, + "Standard_E64-32ds_v4": 6.096000, + "Standard_E64-32ds_v5": 6.497000, + "Standard_E64-32s_v3": 4.801000, + "Standard_E64-32s_v4": 5.334000, + "Standard_E64-32s_v5": 5.685000, + "Standard_E64_v3": 4.801000, + "Standard_E64_v4": 5.334000, + "Standard_E64_v5": 5.685000, + "Standard_E64a_v4": 5.685000, + "Standard_E64ads_v5": 5.911000, + "Standard_E64as_v4": 5.685000, + "Standard_E64as_v5": 5.099000, + "Standard_E64bds_v5": 7.535000, + "Standard_E64bs_v5": 6.723000, + "Standard_E64d_v4": 6.096000, + "Standard_E64d_v5": 6.497000, + "Standard_E64ds_v4": 6.096000, + "Standard_E64ds_v5": 6.497000, + "Standard_E64i_v3": 4.801000, + "Standard_E64is_v3": 4.801000, + "Standard_E64s_v3": 4.801000, + "Standard_E64s_v4": 5.334000, + "Standard_E64s_v5": 5.685000, + "Standard_E8-2ads_v5": 0.739000, + "Standard_E8-2as_v4": 0.711000, + "Standard_E8-2as_v5": 0.637000, + "Standard_E8-2ds_v4": 0.762000, + "Standard_E8-2ds_v5": 0.812000, + "Standard_E8-2s_v3": 0.667000, + "Standard_E8-2s_v4": 0.667000, + "Standard_E8-2s_v5": 0.711000, + "Standard_E8-4ads_v5": 1.107000, + "Standard_E8-4as_v4": 0.711000, + "Standard_E8-4as_v5": 1.005000, + "Standard_E8-4ds_v4": 0.762000, + "Standard_E8-4ds_v5": 0.812000, + "Standard_E8-4s_v3": 0.667000, + "Standard_E8-4s_v4": 0.667000, + "Standard_E8-4s_v5": 0.711000, + "Standard_E80ids_v4": 9.155160, + "Standard_E80is_v4": 8.043840, + "Standard_E8_v3": 0.667000, + "Standard_E8_v4": 0.667000, + "Standard_E8_v5": 0.711000, + "Standard_E8a_v4": 0.711000, + "Standard_E8ads_v5": 0.785000, + "Standard_E8as_v4": 0.711000, + "Standard_E8as_v5": 0.637000, + "Standard_E8bds_v5": 0.942000, + "Standard_E8bs_v5": 0.840000, + "Standard_E8d_v4": 0.762000, + "Standard_E8d_v5": 0.812000, + "Standard_E8ds_v4": 0.762000, + "Standard_E8ds_v5": 0.812000, + "Standard_E8s_v3": 0.667000, + "Standard_E8s_v4": 0.667000, + "Standard_E8s_v5": 0.711000, + "Standard_E96-24ads_v5": 13.282000, + "Standard_E96-24as_v4": 8.528000, + "Standard_E96-24as_v5": 7.648000, + "Standard_E96-24ds_v5": 9.746000, + "Standard_E96-24s_v5": 8.528000, + "Standard_E96-48ads_v5": 8.866000, + "Standard_E96-48as_v4": 8.528000, + "Standard_E96-48as_v5": 7.648000, + "Standard_E96-48ds_v5": 9.746000, + "Standard_E96-48s_v5": 8.528000, + "Standard_E96_v5": 8.528000, + "Standard_E96a_v4": 8.528000, + "Standard_E96ads_v5": 8.866000, + "Standard_E96as_v4": 8.528000, + "Standard_E96as_v5": 7.648000, + "Standard_E96bds_v5": 11.303000, + "Standard_E96bs_v5": 10.084000, + "Standard_E96d_v5": 9.746000, + "Standard_E96ds_v5": 9.746000, + "Standard_E96iads_v5": 9.753000, + "Standard_E96ias_v4": 8.528000, + "Standard_E96ias_v5": 8.413000, + "Standard_E96s_v5": 8.528000, + "Standard_F1": 0.060000, + "Standard_F16": 0.966000, + "Standard_F16s": 0.966000, + "Standard_F16s_v2": 0.748000, + "Standard_F1s": 0.060000, + "Standard_F2": 0.121000, + "Standard_F2s": 0.121000, + "Standard_F2s_v2": 0.093500, + "Standard_F32s_v2": 1.496000, + "Standard_F4": 0.242000, + "Standard_F48s_v2": 2.244000, + "Standard_F4s": 0.242000, + "Standard_F4s_v2": 0.187000, + "Standard_F64s_v2": 2.992000, + "Standard_F72s_v2": 3.366000, + "Standard_F8": 0.483000, + "Standard_F8s": 0.483000, + "Standard_F8s_v2": 0.374000, + "Standard_L16as_v3": 1.760000, + "Standard_L16s_v3": 1.968000, + "Standard_L32as_v3": 3.519000, + "Standard_L32s_v3": 3.937000, + "Standard_L48as_v3": 5.279000, + "Standard_L48s_v3": 5.905000, + "Standard_L64as_v3": 7.039000, + "Standard_L64s_v3": 7.873000, + "Standard_L80as_v3": 8.798000, + "Standard_L80s_v3": 9.842000, + "Standard_L8as_v3": 0.880000, + "Standard_L8s_v3": 0.984000, + "Standard_M128": 20.207000, + "Standard_M128-32ms": 40.440000, + "Standard_M128-64ms": 40.440000, + "Standard_M128dms_v2": 40.302000, + "Standard_M128ds_v2": 20.143000, + "Standard_M128m": 40.440000, + "Standard_M128ms": 40.440000, + "Standard_M128ms_v2": 39.666000, + "Standard_M128s": 20.207000, + "Standard_M128s_v2": 19.508000, + "Standard_M16-4ms": 4.610000, + "Standard_M16-8ms": 4.610000, + "Standard_M16ms": 4.610000, + "Standard_M16s": 3.616000, + "Standard_M192idms_v2": 48.416000, + "Standard_M192ids_v2": 24.208000, + "Standard_M192ims_v2": 47.781000, + "Standard_M192is_v2": 23.573000, + "Standard_M208-104ms_v2": 67.376000, + "Standard_M208-104s_v2": 33.688000, + "Standard_M208-52ms_v2": 67.376000, + "Standard_M208-52s_v2": 33.688000, + "Standard_M208ms_v2": 67.376000, + "Standard_M208s_v2": 33.688000, + "Standard_M32-16ms": 9.311200, + "Standard_M32-8ms": 9.311200, + "Standard_M32dms_v2": 9.280000, + "Standard_M32ls": 3.619980, + "Standard_M32ms": 9.311200, + "Standard_M32ms_v2": 9.122000, + "Standard_M32s": 5.052500, + "Standard_M32ts": 3.410820, + "Standard_M416-104ms_v2": 149.720000, + "Standard_M416-104s_v2": 74.866000, + "Standard_M416-208ms_v2": 149.720000, + "Standard_M416-208s_v2": 74.866000, + "Standard_M416is_v2": 74.866000, + "Standard_M416ms_v2": 149.720000, + "Standard_M416s_8_v2": 99.830000, + "Standard_M416s_v2": 74.866000, + "Standard_M64": 10.104000, + "Standard_M64-16ms": 15.664000, + "Standard_M64-32ms": 15.664000, + "Standard_M64dms_v2": 15.613000, + "Standard_M64ds_v2": 10.070000, + "Standard_M64ls": 6.822900, + "Standard_M64m": 15.664000, + "Standard_M64ms": 15.664000, + "Standard_M64ms_v2": 15.296000, + "Standard_M64s": 10.104000, + "Standard_M64s_v2": 9.752000, + "Standard_M8-2ms": 2.305000, + "Standard_M8-4ms": 2.305000, + "Standard_M8ms": 2.305000, + "Standard_NC16as_T4_v3": 1.697000, + "Standard_NC4as_T4_v3": 0.742000, + "Standard_NC64as_T4_v3": 6.123000, + "Standard_NC8as_T4_v3": 1.059000, + "Standard_ND96ams_A100_v4": 49.483000, + "Standard_ND96amsr_A100_v4": 49.483000, + "Standard_SQLG7_AMD_IaaS": 11.370000, + "Standard_SQLG7_AMD_NVME": 11.370000, + } + // swedencentral + initialOnDemandPrices["swedencentral"] = map[string]float64{ + "Dadsv5_Type1": 6.776000, + "Dasv4_Type1": 5.610000, + "Dasv4_Type2": 5.610000, + "Dasv5_Type1": 5.667000, + "Ddsv4_Type 1": 4.224000, + "Ddsv4_Type2": 5.016000, + "Ddsv5_Type1": 6.336000, + "Dsv3_Type3": 4.488000, + "Dsv3_Type4": 5.386000, + "Dsv4_Type1": 4.488000, + "Dsv4_Type2": 5.386000, + "Dsv5_Type1": 5.610000, + "Eadsv5_Type1": 7.339000, + "Easv4_Type1": 7.075000, + "Easv4_Type2": 7.075000, + "Easv5_Type1": 6.389000, + "Ebdsv5-Type1": 6.248000, + "Ebsv5-Type1": 5.614000, + "Edsv4_Type 1": 5.350000, + "Edsv4_Type2": 6.354000, + "Edsv5_Type1": 8.026000, + "Esv3_Type3": 4.717000, + "Esv3_Type4": 6.191000, + "Esv4_Type1": 4.717000, + "Esv4_Type2": 6.191000, + "Esv5_Type1": 7.075000, + "Lsv2_Type1": 7.172000, + "Lsv3_Type1": 8.008000, + "Mdmsv2MedMem _Type1": 41.103000, + "Mdsv2MedMem_Type1": 20.544000, + "Msmv2MedMem Type1": 40.455000, + "Msmv2_Type1": 130.880000, + "Msv2MedMem Type1": 19.896000, + "Msv2_Type1": 65.446000, + "Standard_A1_v2": 0.041000, + "Standard_A2_v2": 0.086100, + "Standard_A2m_v2": 0.113000, + "Standard_A4_v2": 0.182000, + "Standard_A4m_v2": 0.226000, + "Standard_A8_v2": 0.380000, + "Standard_A8m_v2": 0.453000, + "Standard_B12ms": 0.518000, + "Standard_B16als_v2": 0.624000, + "Standard_B16as_v2": 0.622000, + "Standard_B16ls_v2": 0.686000, + "Standard_B16ms": 0.691000, + "Standard_B16pls_v2": 0.561000, + "Standard_B16ps_v2": 0.624000, + "Standard_B16s_v2": 0.765000, + "Standard_B1ls": 0.005400, + "Standard_B1ms": 0.021600, + "Standard_B1s": 0.010800, + "Standard_B20ms": 0.864000, + "Standard_B2als_v2": 0.048100, + "Standard_B2as_v2": 0.087000, + "Standard_B2ats_v2": 0.018900, + "Standard_B2ls_v2": 0.052400, + "Standard_B2ms": 0.086400, + "Standard_B2pls_v2": 0.043600, + "Standard_B2ps_v2": 0.078000, + "Standard_B2pts_v2": 0.017800, + "Standard_B2s": 0.043200, + "Standard_B2s_v2": 0.095600, + "Standard_B2ts_v2": 0.010800, + "Standard_B32als_v2": 1.249000, + "Standard_B32as_v2": 1.244000, + "Standard_B32ls_v2": 1.371000, + "Standard_B32s_v2": 1.382000, + "Standard_B4als_v2": 0.138000, + "Standard_B4as_v2": 0.156000, + "Standard_B4ls_v2": 0.153000, + "Standard_B4ms": 0.173000, + "Standard_B4pls_v2": 0.140000, + "Standard_B4ps_v2": 0.156000, + "Standard_B4s_v2": 0.191000, + "Standard_B8als_v2": 0.275000, + "Standard_B8as_v2": 0.311000, + "Standard_B8ls_v2": 0.343000, + "Standard_B8ms": 0.346000, + "Standard_B8pls_v2": 0.280000, + "Standard_B8ps_v2": 0.312000, + "Standard_B8s_v2": 0.346000, + "Standard_D11_v2": 0.171000, + "Standard_D11_v2_Promo": 0.171000, + "Standard_D12_v2": 0.345000, + "Standard_D12_v2_Promo": 0.345000, + "Standard_D13_v2": 0.691000, + "Standard_D13_v2_Promo": 0.691000, + "Standard_D14_v2": 1.381000, + "Standard_D14_v2_Promo": 1.381000, + "Standard_D15_v2": 1.726000, + "Standard_D15i_v2": 1.726000, + "Standard_D16_v3": 0.816000, + "Standard_D16_v4": 0.816000, + "Standard_D16_v5": 0.816000, + "Standard_D16a_v4": 0.816000, + "Standard_D16ads_v5": 1.616000, + "Standard_D16as_v4": 0.816000, + "Standard_D16as_v5": 0.736000, + "Standard_D16d_v4": 0.960000, + "Standard_D16d_v5": 0.960000, + "Standard_D16ds_v4": 0.960000, + "Standard_D16ds_v5": 0.960000, + "Standard_D16lds_v5": 0.832000, + "Standard_D16ls_v5": 0.728000, + "Standard_D16pds_v5": 0.768000, + "Standard_D16plds_v5": 0.666000, + "Standard_D16pls_v5": 0.584000, + "Standard_D16ps_v5": 0.656000, + "Standard_D16s_v3": 0.816000, + "Standard_D16s_v4": 0.816000, + "Standard_D16s_v5": 0.816000, + "Standard_D1_v2": 0.068000, + "Standard_D2_v2": 0.136000, + "Standard_D2_v2_Promo": 0.136000, + "Standard_D2_v3": 0.102000, + "Standard_D2_v4": 0.102000, + "Standard_D2_v5": 0.102000, + "Standard_D2a_v4": 0.102000, + "Standard_D2ads_v5": 0.202000, + "Standard_D2as_v4": 0.102000, + "Standard_D2as_v5": 0.184000, + "Standard_D2d_v4": 0.120000, + "Standard_D2d_v5": 0.120000, + "Standard_D2ds_v4": 0.120000, + "Standard_D2ds_v5": 0.120000, + "Standard_D2lds_v5": 0.104000, + "Standard_D2ls_v5": 0.091000, + "Standard_D2pds_v5": 0.096000, + "Standard_D2plds_v5": 0.083200, + "Standard_D2pls_v5": 0.073000, + "Standard_D2ps_v5": 0.082000, + "Standard_D2s_v3": 0.102000, + "Standard_D2s_v4": 0.102000, + "Standard_D2s_v5": 0.102000, + "Standard_D32-16s_v3": 1.632000, + "Standard_D32-8s_v3": 1.632000, + "Standard_D32_v3": 1.632000, + "Standard_D32_v4": 1.632000, + "Standard_D32_v5": 1.632000, + "Standard_D32a_v4": 1.632000, + "Standard_D32ads_v5": 1.760000, + "Standard_D32as_v4": 1.632000, + "Standard_D32as_v5": 1.472000, + "Standard_D32d_v4": 1.920000, + "Standard_D32d_v5": 1.920000, + "Standard_D32ds_v4": 1.920000, + "Standard_D32ds_v5": 1.920000, + "Standard_D32lds_v5": 1.664000, + "Standard_D32ls_v5": 1.456000, + "Standard_D32pds_v5": 1.536000, + "Standard_D32plds_v5": 1.331000, + "Standard_D32pls_v5": 1.168000, + "Standard_D32ps_v5": 1.312000, + "Standard_D32s_v3": 1.632000, + "Standard_D32s_v4": 1.632000, + "Standard_D32s_v5": 1.632000, + "Standard_D3_v2": 0.272000, + "Standard_D3_v2_Promo": 0.272000, + "Standard_D48_v3": 2.448000, + "Standard_D48_v4": 2.448000, + "Standard_D48_v5": 2.448000, + "Standard_D48a_v4": 2.448000, + "Standard_D48ads_v5": 2.640000, + "Standard_D48as_v4": 2.448000, + "Standard_D48as_v5": 4.416000, + "Standard_D48d_v4": 2.880000, + "Standard_D48d_v5": 2.880000, + "Standard_D48ds_v4": 2.880000, + "Standard_D48ds_v5": 2.880000, + "Standard_D48lds_v5": 2.496000, + "Standard_D48ls_v5": 2.184000, + "Standard_D48pds_v5": 2.304000, + "Standard_D48plds_v5": 1.997000, + "Standard_D48pls_v5": 1.752000, + "Standard_D48ps_v5": 1.968000, + "Standard_D48s_v3": 2.448000, + "Standard_D48s_v4": 2.448000, + "Standard_D48s_v5": 2.448000, + "Standard_D4_v2": 0.544000, + "Standard_D4_v2_Promo": 0.544000, + "Standard_D4_v3": 0.204000, + "Standard_D4_v4": 0.204000, + "Standard_D4_v5": 0.204000, + "Standard_D4a_v4": 0.204000, + "Standard_D4ads_v5": 0.404000, + "Standard_D4as_v4": 0.204000, + "Standard_D4as_v5": 0.184000, + "Standard_D4d_v4": 0.240000, + "Standard_D4d_v5": 0.240000, + "Standard_D4ds_v4": 0.240000, + "Standard_D4ds_v5": 0.240000, + "Standard_D4lds_v5": 0.208000, + "Standard_D4ls_v5": 0.182000, + "Standard_D4pds_v5": 0.192000, + "Standard_D4plds_v5": 0.166000, + "Standard_D4pls_v5": 0.146000, + "Standard_D4ps_v5": 0.164000, + "Standard_D4s_v3": 0.204000, + "Standard_D4s_v4": 0.204000, + "Standard_D4s_v5": 0.204000, + "Standard_D5_v2": 1.088000, + "Standard_D5_v2_Promo": 1.088000, + "Standard_D64-16s_v3": 3.264000, + "Standard_D64-32s_v3": 3.264000, + "Standard_D64_v3": 3.264000, + "Standard_D64_v4": 3.264000, + "Standard_D64_v5": 3.264000, + "Standard_D64a_v4": 3.264000, + "Standard_D64ads_v5": 6.464000, + "Standard_D64as_v4": 3.264000, + "Standard_D64as_v5": 5.888000, + "Standard_D64d_v4": 3.840000, + "Standard_D64d_v5": 3.840000, + "Standard_D64ds_v4": 3.840000, + "Standard_D64ds_v5": 3.840000, + "Standard_D64lds_v5": 3.328000, + "Standard_D64ls_v5": 2.912000, + "Standard_D64pds_v5": 3.072000, + "Standard_D64plds_v5": 2.662000, + "Standard_D64pls_v5": 2.336000, + "Standard_D64ps_v5": 2.624000, + "Standard_D64s_v3": 3.264000, + "Standard_D64s_v4": 3.264000, + "Standard_D64s_v5": 3.264000, + "Standard_D8_v3": 0.408000, + "Standard_D8_v4": 0.408000, + "Standard_D8_v5": 0.408000, + "Standard_D8a_v4": 0.408000, + "Standard_D8ads_v5": 0.808000, + "Standard_D8as_v4": 0.408000, + "Standard_D8as_v5": 0.368000, + "Standard_D8d_v4": 0.480000, + "Standard_D8d_v5": 0.480000, + "Standard_D8ds_v4": 0.480000, + "Standard_D8ds_v5": 0.480000, + "Standard_D8lds_v5": 0.416000, + "Standard_D8ls_v5": 0.364000, + "Standard_D8pds_v5": 0.384000, + "Standard_D8plds_v5": 0.333000, + "Standard_D8pls_v5": 0.292000, + "Standard_D8ps_v5": 0.328000, + "Standard_D8s_v3": 0.408000, + "Standard_D8s_v4": 0.408000, + "Standard_D8s_v5": 0.408000, + "Standard_D96_v5": 4.896000, + "Standard_D96a_v4": 4.896000, + "Standard_D96ads_v5": 9.696000, + "Standard_D96as_v4": 4.896000, + "Standard_D96as_v5": 4.416000, + "Standard_D96d_v5": 5.760000, + "Standard_D96ds_v5": 5.760000, + "Standard_D96lds_v5": 4.992000, + "Standard_D96ls_v5": 4.368000, + "Standard_D96s_v5": 4.896000, + "Standard_DC2s": 0.198000, + "Standard_DC4s": 0.395000, + "Standard_DS11-1_v2": 0.171000, + "Standard_DS11_v2": 0.171000, + "Standard_DS11_v2_Promo": 0.171000, + "Standard_DS12-1_v2": 0.345000, + "Standard_DS12-2_v2": 0.345000, + "Standard_DS12_v2": 0.345000, + "Standard_DS12_v2_Promo": 0.345000, + "Standard_DS13-2_v2": 0.691000, + "Standard_DS13-4_v2": 0.691000, + "Standard_DS13_v2": 0.691000, + "Standard_DS13_v2_Promo": 0.691000, + "Standard_DS14-4_v2": 1.381000, + "Standard_DS14-8_v2": 1.381000, + "Standard_DS14_v2": 1.381000, + "Standard_DS14_v2_Promo": 1.381000, + "Standard_DS15_v2": 1.726000, + "Standard_DS15i_v2": 1.726000, + "Standard_DS1_v2": 0.068000, + "Standard_DS2_v2": 0.136000, + "Standard_DS2_v2_Promo": 0.136000, + "Standard_DS3_v2": 0.272000, + "Standard_DS3_v2_Promo": 0.272000, + "Standard_DS4_v2": 0.544000, + "Standard_DS4_v2_Promo": 0.544000, + "Standard_DS5_v2": 1.088000, + "Standard_DS5_v2_Promo": 1.088000, + "Standard_E104i_v5": 7.665000, + "Standard_E104id_v5": 8.694000, + "Standard_E104ids_v5": 8.694000, + "Standard_E104is_v5": 7.665000, + "Standard_E112iads_v5": 8.562000, + "Standard_E112ias_v5": 7.454000, + "Standard_E112ibds_v5": 10.934000, + "Standard_E112ibs_v5": 9.825000, + "Standard_E16-4ads_v5": 1.848000, + "Standard_E16-4as_v4": 1.072000, + "Standard_E16-4as_v5": 0.968000, + "Standard_E16-4ds_v4": 1.216000, + "Standard_E16-4ds_v5": 1.216000, + "Standard_E16-4s_v3": 1.072000, + "Standard_E16-4s_v4": 1.072000, + "Standard_E16-4s_v5": 1.072000, + "Standard_E16-8ads_v5": 1.848000, + "Standard_E16-8as_v4": 1.072000, + "Standard_E16-8as_v5": 1.704000, + "Standard_E16-8ds_v4": 1.216000, + "Standard_E16-8ds_v5": 1.216000, + "Standard_E16-8s_v3": 1.072000, + "Standard_E16-8s_v4": 1.072000, + "Standard_E16-8s_v5": 1.072000, + "Standard_E16_v3": 1.072000, + "Standard_E16_v4": 1.072000, + "Standard_E16_v5": 1.072000, + "Standard_E16a_v4": 1.072000, + "Standard_E16ads_v5": 1.112000, + "Standard_E16as_v4": 1.072000, + "Standard_E16as_v5": 0.968000, + "Standard_E16bds_v5": 1.420000, + "Standard_E16bs_v5": 1.276000, + "Standard_E16d_v4": 1.216000, + "Standard_E16d_v5": 1.216000, + "Standard_E16ds_v4": 1.216000, + "Standard_E16ds_v5": 1.216000, + "Standard_E16pds_v5": 0.973000, + "Standard_E16ps_v5": 0.856000, + "Standard_E16s_v3": 1.072000, + "Standard_E16s_v4": 1.072000, + "Standard_E16s_v5": 1.072000, + "Standard_E20_v3": 1.340000, + "Standard_E20_v4": 1.340000, + "Standard_E20_v5": 1.340000, + "Standard_E20a_v4": 1.340000, + "Standard_E20ads_v5": 1.436000, + "Standard_E20as_v4": 1.340000, + "Standard_E20as_v5": 1.210000, + "Standard_E20d_v4": 1.520000, + "Standard_E20d_v5": 1.520000, + "Standard_E20ds_v4": 1.520000, + "Standard_E20ds_v5": 1.520000, + "Standard_E20pds_v5": 1.216000, + "Standard_E20ps_v5": 1.070000, + "Standard_E20s_v3": 1.340000, + "Standard_E20s_v4": 1.340000, + "Standard_E20s_v5": 1.340000, + "Standard_E2_v3": 0.134000, + "Standard_E2_v4": 0.134000, + "Standard_E2_v5": 0.134000, + "Standard_E2a_v4": 0.134000, + "Standard_E2ads_v5": 0.185000, + "Standard_E2as_v4": 0.134000, + "Standard_E2as_v5": 0.167000, + "Standard_E2bds_v5": 0.178000, + "Standard_E2bs_v5": 0.160000, + "Standard_E2d_v4": 0.152000, + "Standard_E2d_v5": 0.152000, + "Standard_E2ds_v4": 0.152000, + "Standard_E2ds_v5": 0.152000, + "Standard_E2pds_v5": 0.122000, + "Standard_E2ps_v5": 0.107000, + "Standard_E2s_v3": 0.134000, + "Standard_E2s_v4": 0.134000, + "Standard_E2s_v5": 0.134000, + "Standard_E32-16ads_v5": 3.696000, + "Standard_E32-16as_v4": 2.144000, + "Standard_E32-16as_v5": 1.936000, + "Standard_E32-16ds_v4": 2.432000, + "Standard_E32-16ds_v5": 2.432000, + "Standard_E32-16s_v3": 2.144000, + "Standard_E32-16s_v4": 2.144000, + "Standard_E32-16s_v5": 2.144000, + "Standard_E32-8ads_v5": 2.224000, + "Standard_E32-8as_v4": 2.144000, + "Standard_E32-8as_v5": 1.936000, + "Standard_E32-8ds_v4": 2.432000, + "Standard_E32-8ds_v5": 2.432000, + "Standard_E32-8s_v3": 2.144000, + "Standard_E32-8s_v4": 2.144000, + "Standard_E32-8s_v5": 2.144000, + "Standard_E32_v3": 2.144000, + "Standard_E32_v4": 2.144000, + "Standard_E32_v5": 2.144000, + "Standard_E32a_v4": 2.144000, + "Standard_E32ads_v5": 2.224000, + "Standard_E32as_v4": 2.144000, + "Standard_E32as_v5": 1.936000, + "Standard_E32bds_v5": 2.840000, + "Standard_E32bs_v5": 2.552000, + "Standard_E32d_v4": 2.432000, + "Standard_E32d_v5": 2.432000, + "Standard_E32ds_v4": 2.432000, + "Standard_E32ds_v5": 2.432000, + "Standard_E32pds_v5": 1.946000, + "Standard_E32ps_v5": 1.712000, + "Standard_E32s_v3": 2.144000, + "Standard_E32s_v4": 2.144000, + "Standard_E32s_v5": 2.144000, + "Standard_E4-2ads_v5": 0.278000, + "Standard_E4-2as_v4": 0.268000, + "Standard_E4-2as_v5": 0.426000, + "Standard_E4-2ds_v4": 0.304000, + "Standard_E4-2ds_v5": 0.304000, + "Standard_E4-2s_v3": 0.268000, + "Standard_E4-2s_v4": 0.268000, + "Standard_E4-2s_v5": 0.268000, + "Standard_E48_v3": 3.216000, + "Standard_E48_v4": 3.216000, + "Standard_E48_v5": 3.216000, + "Standard_E48a_v4": 3.216000, + "Standard_E48ads_v5": 3.382000, + "Standard_E48as_v4": 3.216000, + "Standard_E48as_v5": 2.904000, + "Standard_E48bds_v5": 4.260000, + "Standard_E48bs_v5": 3.828000, + "Standard_E48d_v4": 3.648000, + "Standard_E48d_v5": 3.648000, + "Standard_E48ds_v4": 3.648000, + "Standard_E48ds_v5": 3.648000, + "Standard_E48s_v3": 3.216000, + "Standard_E48s_v4": 3.216000, + "Standard_E48s_v5": 3.216000, + "Standard_E4_v3": 0.268000, + "Standard_E4_v4": 0.268000, + "Standard_E4_v5": 0.268000, + "Standard_E4a_v4": 0.268000, + "Standard_E4ads_v5": 0.278000, + "Standard_E4as_v4": 0.268000, + "Standard_E4as_v5": 0.288000, + "Standard_E4bds_v5": 0.355000, + "Standard_E4bs_v5": 0.319000, + "Standard_E4d_v4": 0.304000, + "Standard_E4d_v5": 0.304000, + "Standard_E4ds_v4": 0.304000, + "Standard_E4ds_v5": 0.304000, + "Standard_E4pds_v5": 0.243000, + "Standard_E4ps_v5": 0.214000, + "Standard_E4s_v3": 0.268000, + "Standard_E4s_v4": 0.268000, + "Standard_E4s_v5": 0.268000, + "Standard_E64-16ads_v5": 7.392000, + "Standard_E64-16as_v4": 4.288000, + "Standard_E64-16as_v5": 6.816000, + "Standard_E64-16ds_v4": 4.864000, + "Standard_E64-16ds_v5": 4.864000, + "Standard_E64-16s_v3": 4.288000, + "Standard_E64-16s_v4": 4.288000, + "Standard_E64-16s_v5": 4.288000, + "Standard_E64-32ads_v5": 7.392000, + "Standard_E64-32as_v4": 4.288000, + "Standard_E64-32as_v5": 6.816000, + "Standard_E64-32ds_v4": 4.864000, + "Standard_E64-32ds_v5": 4.864000, + "Standard_E64-32s_v3": 4.288000, + "Standard_E64-32s_v4": 4.288000, + "Standard_E64-32s_v5": 4.288000, + "Standard_E64_v3": 4.288000, + "Standard_E64_v4": 4.288000, + "Standard_E64_v5": 4.288000, + "Standard_E64a_v4": 4.288000, + "Standard_E64ads_v5": 4.448000, + "Standard_E64as_v4": 4.288000, + "Standard_E64as_v5": 3.872000, + "Standard_E64bds_v5": 5.680000, + "Standard_E64bs_v5": 5.104000, + "Standard_E64d_v4": 4.864000, + "Standard_E64d_v5": 4.864000, + "Standard_E64ds_v4": 4.864000, + "Standard_E64ds_v5": 4.864000, + "Standard_E64i_v3": 4.288000, + "Standard_E64i_v4_SPECIAL": 4.288000, + "Standard_E64id_v4_SPECIAL": 4.864000, + "Standard_E64ids_v4_SPECIAL": 4.864000, + "Standard_E64is_v3": 4.288000, + "Standard_E64is_v4_SPECIAL": 4.288000, + "Standard_E64s_v3": 4.288000, + "Standard_E64s_v4": 4.288000, + "Standard_E64s_v5": 4.288000, + "Standard_E8-2ads_v5": 0.556000, + "Standard_E8-2as_v4": 0.536000, + "Standard_E8-2as_v5": 0.484000, + "Standard_E8-2ds_v4": 0.608000, + "Standard_E8-2ds_v5": 0.608000, + "Standard_E8-2s_v3": 0.536000, + "Standard_E8-2s_v4": 0.536000, + "Standard_E8-2s_v5": 0.536000, + "Standard_E8-4ads_v5": 0.556000, + "Standard_E8-4as_v4": 0.536000, + "Standard_E8-4as_v5": 0.484000, + "Standard_E8-4ds_v4": 0.608000, + "Standard_E8-4ds_v5": 0.608000, + "Standard_E8-4s_v3": 0.536000, + "Standard_E8-4s_v4": 0.536000, + "Standard_E8-4s_v5": 0.536000, + "Standard_E80ids_v4": 6.080000, + "Standard_E80is_v4": 5.360000, + "Standard_E8_v3": 0.536000, + "Standard_E8_v4": 0.536000, + "Standard_E8_v5": 0.536000, + "Standard_E8a_v4": 0.536000, + "Standard_E8ads_v5": 0.602000, + "Standard_E8as_v4": 0.536000, + "Standard_E8as_v5": 0.530000, + "Standard_E8bds_v5": 0.710000, + "Standard_E8bs_v5": 0.638000, + "Standard_E8d_v4": 0.608000, + "Standard_E8d_v5": 0.608000, + "Standard_E8ds_v4": 0.608000, + "Standard_E8ds_v5": 0.608000, + "Standard_E8pds_v5": 0.486000, + "Standard_E8ps_v5": 0.428000, + "Standard_E8s_v3": 0.536000, + "Standard_E8s_v4": 0.536000, + "Standard_E8s_v5": 0.536000, + "Standard_E96-24ads_v5": 6.672000, + "Standard_E96-24as_v4": 6.432000, + "Standard_E96-24as_v5": 10.224000, + "Standard_E96-24ds_v5": 7.296000, + "Standard_E96-24s_v5": 6.432000, + "Standard_E96-48ads_v5": 6.672000, + "Standard_E96-48as_v4": 6.432000, + "Standard_E96-48as_v5": 5.808000, + "Standard_E96-48ds_v5": 7.296000, + "Standard_E96-48s_v5": 6.432000, + "Standard_E96_v5": 6.432000, + "Standard_E96a_v4": 6.432000, + "Standard_E96ads_v5": 6.718000, + "Standard_E96as_v4": 6.432000, + "Standard_E96as_v5": 5.854000, + "Standard_E96bds_v5": 8.520000, + "Standard_E96bs_v5": 7.656000, + "Standard_E96d_v5": 7.296000, + "Standard_E96ds_v5": 7.296000, + "Standard_E96iads_v5": 7.339000, + "Standard_E96ias_v4": 6.432000, + "Standard_E96ias_v5": 6.389000, + "Standard_E96s_v5": 6.432000, + "Standard_F1": 0.056800, + "Standard_F16": 0.909000, + "Standard_F16s": 0.909000, + "Standard_F16s_v2": 0.776000, + "Standard_F1s": 0.056800, + "Standard_F2": 0.114000, + "Standard_F2s": 0.114000, + "Standard_F2s_v2": 0.097000, + "Standard_F32s_v2": 1.552000, + "Standard_F4": 0.227000, + "Standard_F48s_v2": 2.328000, + "Standard_F4s": 0.227000, + "Standard_F4s_v2": 0.194000, + "Standard_F64s_v2": 3.104000, + "Standard_F72s_v2": 3.492000, + "Standard_F8": 0.455000, + "Standard_F8s": 0.455000, + "Standard_F8s_v2": 0.388000, + "Standard_HB176-144rs_v4": 9.360000, + "Standard_HB176-24rs_v4": 9.360000, + "Standard_HB176-48rs_v4": 9.360000, + "Standard_HB176-96rs_v4": 9.360000, + "Standard_HB176rs_v4": 9.360000, + "Standard_HX176-144rs": 11.232000, + "Standard_HX176-24rs": 11.232000, + "Standard_HX176-48rs": 11.232000, + "Standard_HX176-96rs": 11.232000, + "Standard_HX176rs": 11.232000, + "Standard_L112ias_v3": 10.041000, + "Standard_L16as_v3": 1.304000, + "Standard_L16s_v2": 1.304000, + "Standard_L16s_v3": 1.456000, + "Standard_L32as_v3": 2.608000, + "Standard_L32s_v2": 2.608000, + "Standard_L32s_v3": 2.912000, + "Standard_L48as_v3": 3.912000, + "Standard_L48s_v2": 3.912000, + "Standard_L48s_v3": 4.368000, + "Standard_L64as_v3": 5.216000, + "Standard_L64s_v2": 5.216000, + "Standard_L64s_v3": 5.824000, + "Standard_L80as_v3": 6.520000, + "Standard_L80s_v2": 6.520000, + "Standard_L80s_v3": 7.280000, + "Standard_L88is_v2": 7.172000, + "Standard_L8as_v3": 0.652000, + "Standard_L8s_v2": 0.652000, + "Standard_L8s_v3": 0.728000, + "Standard_L96ias_v3": 8.606000, + "Standard_M128": 18.673000, + "Standard_M128-32ms": 37.363000, + "Standard_M128-64ms": 37.363000, + "Standard_M128dms_v2": 37.366000, + "Standard_M128ds_v2": 18.676000, + "Standard_M128m": 37.363000, + "Standard_M128ms": 37.363000, + "Standard_M128ms_v2": 36.777000, + "Standard_M128s": 18.673000, + "Standard_M128s_v2": 18.087000, + "Standard_M16-4ms": 4.302000, + "Standard_M16-8ms": 4.302000, + "Standard_M16ms": 4.302000, + "Standard_M16s": 3.342000, + "Standard_M192idms_v2": 44.889000, + "Standard_M192ids_v2": 22.445000, + "Standard_M192ims_v2": 44.300000, + "Standard_M192is_v2": 21.856000, + "Standard_M208-104ms_v2": 53.544000, + "Standard_M208-104s_v2": 26.772000, + "Standard_M208-52ms_v2": 53.544000, + "Standard_M208-52s_v2": 26.772000, + "Standard_M208ms_v2": 53.544000, + "Standard_M208s_v2": 26.772000, + "Standard_M32-16ms": 8.604000, + "Standard_M32-8ms": 8.604000, + "Standard_M32dms_v2": 8.604000, + "Standard_M32ls": 4.022000, + "Standard_M32ms": 8.604000, + "Standard_M32ms_v2": 8.457000, + "Standard_M32s": 4.669000, + "Standard_M32ts": 3.790000, + "Standard_M416-104ms_v2": 118.980000, + "Standard_M416-104s_v2": 59.496000, + "Standard_M416-208ms_v2": 118.980000, + "Standard_M416-208s_v2": 59.496000, + "Standard_M416is_v2": 59.496000, + "Standard_M416ms_v2": 118.980000, + "Standard_M416s_8_v2": 79.750000, + "Standard_M416s_v2": 59.496000, + "Standard_M64": 9.337000, + "Standard_M64-16ms": 14.471800, + "Standard_M64-32ms": 14.471800, + "Standard_M64dms_v2": 14.476000, + "Standard_M64ds_v2": 9.337000, + "Standard_M64ls": 7.581000, + "Standard_M64m": 14.471800, + "Standard_M64ms": 14.471800, + "Standard_M64ms_v2": 14.181000, + "Standard_M64s": 9.337000, + "Standard_M64s_v2": 9.042000, + "Standard_M8-2ms": 2.151100, + "Standard_M8-4ms": 2.151100, + "Standard_M8ms": 2.151100, + "Standard_ND40s_v2": 12.240000, + "Standard_ND96ams_A100_v4": 42.601000, + "Standard_ND96amsr_A100_v4": 42.601000, + "Standard_NV12ads_A10_v5": 1.180000, + "Standard_NV18ads_A10_v5": 2.080000, + "Standard_NV36adms_A10_v5": 5.876000, + "Standard_NV36ads_A10_v5": 4.160000, + "Standard_NV6ads_A10_v5": 0.590000, + "Standard_NV72ads_A10_v5": 8.476000, + "Standard_SQLG7_NVME": 7.904000, + } + // swedensouth + initialOnDemandPrices["swedensouth"] = map[string]float64{ + "Dadsv5_Type1": 8.809000, + "Dasv4_Type1": 7.293000, + "Dasv4_Type2": 7.293000, + "Dasv5_Type1": 7.367000, + "Ddsv5_Type1": 8.237000, + "Dsv3_Type3": 5.834000, + "Dsv3_Type4": 7.001000, + "Dsv4_Type1": 5.834000, + "Dsv5_Type1": 7.293000, + "Eadsv5_Type1": 9.541000, + "Easv4_Type1": 9.198000, + "Easv4_Type2": 9.198000, + "Easv5_Type1": 8.305000, + "Ebdsv5-Type1": 8.122000, + "Ebsv5-Type1": 7.299000, + "Edsv4_Type 1": 6.956000, + "Esv3_Type3": 6.132000, + "Esv3_Type4": 8.048000, + "Lasv3_Type1": 9.324000, + "Lsv3_Type1": 10.410000, + "Standard_A1_v2": 0.053300, + "Standard_A2_v2": 0.112000, + "Standard_A2m_v2": 0.147000, + "Standard_A4_v2": 0.237000, + "Standard_A4m_v2": 0.294000, + "Standard_A8_v2": 0.495000, + "Standard_A8m_v2": 0.588000, + "Standard_B12ms": 0.673000, + "Standard_B16als_v2": 0.716000, + "Standard_B16as_v2": 0.809000, + "Standard_B16ls_v2": 0.869000, + "Standard_B16ms": 0.898000, + "Standard_B16s_v2": 0.899000, + "Standard_B1ls": 0.007020, + "Standard_B1ms": 0.028100, + "Standard_B1s": 0.014000, + "Standard_B20ms": 1.123000, + "Standard_B2als_v2": 0.050500, + "Standard_B2as_v2": 0.101000, + "Standard_B2ats_v2": 0.012600, + "Standard_B2ls_v2": 0.056200, + "Standard_B2ms": 0.112000, + "Standard_B2s": 0.056200, + "Standard_B2s_v2": 0.122000, + "Standard_B2ts_v2": 0.014000, + "Standard_B32als_v2": 1.432000, + "Standard_B32as_v2": 1.617000, + "Standard_B32ls_v2": 1.739000, + "Standard_B32s_v2": 1.944000, + "Standard_B4als_v2": 0.179000, + "Standard_B4as_v2": 0.221000, + "Standard_B4ls_v2": 0.199000, + "Standard_B4ms": 0.225000, + "Standard_B4s_v2": 0.243000, + "Standard_B8als_v2": 0.358000, + "Standard_B8as_v2": 0.404000, + "Standard_B8ls_v2": 0.435000, + "Standard_B8ms": 0.449000, + "Standard_B8s_v2": 0.486000, + "Standard_D11_v2": 0.223000, + "Standard_D11_v2_Promo": 0.223000, + "Standard_D12_v2": 0.449000, + "Standard_D12_v2_Promo": 0.449000, + "Standard_D13_v2": 0.898000, + "Standard_D13_v2_Promo": 0.898000, + "Standard_D14_v2": 1.795000, + "Standard_D14_v2_Promo": 1.795000, + "Standard_D15_v2": 2.244000, + "Standard_D15i_v2": 2.244000, + "Standard_D16_v3": 1.061000, + "Standard_D16_v4": 1.061000, + "Standard_D16_v5": 1.061000, + "Standard_D16a_v4": 1.061000, + "Standard_D16ads_v5": 1.880000, + "Standard_D16as_v4": 1.061000, + "Standard_D16as_v5": 0.957000, + "Standard_D16as_v5_Promo": 0.853000, + "Standard_D16d_v4": 1.248000, + "Standard_D16d_v5": 1.248000, + "Standard_D16ds_v5": 1.248000, + "Standard_D16lds_v5": 1.082000, + "Standard_D16ls_v5": 0.946000, + "Standard_D16s_v3": 1.061000, + "Standard_D16s_v4": 1.061000, + "Standard_D16s_v5": 1.061000, + "Standard_D1_v2": 0.088400, + "Standard_D2_v2": 0.177000, + "Standard_D2_v2_Promo": 0.177000, + "Standard_D2_v3": 0.133000, + "Standard_D2_v4": 0.133000, + "Standard_D2_v5": 0.133000, + "Standard_D2a_v4": 0.133000, + "Standard_D2ads_v5": 0.143000, + "Standard_D2as_v4": 0.133000, + "Standard_D2as_v5": 0.120000, + "Standard_D2as_v5_Promo": 0.107000, + "Standard_D2d_v4": 0.156000, + "Standard_D2d_v5": 0.156000, + "Standard_D2ds_v5": 0.156000, + "Standard_D2lds_v5": 0.135000, + "Standard_D2ls_v5": 0.118000, + "Standard_D2s_v3": 0.133000, + "Standard_D2s_v4": 0.133000, + "Standard_D2s_v5": 0.133000, + "Standard_D32-16s_v3": 2.122000, + "Standard_D32-8s_v3": 2.122000, + "Standard_D32_v3": 2.122000, + "Standard_D32_v4": 2.122000, + "Standard_D32_v5": 2.122000, + "Standard_D32a_v4": 2.122000, + "Standard_D32ads_v5": 3.760000, + "Standard_D32as_v4": 2.122000, + "Standard_D32as_v5": 3.386000, + "Standard_D32as_v5_Promo": 1.706000, + "Standard_D32d_v4": 2.496000, + "Standard_D32d_v5": 2.496000, + "Standard_D32ds_v5": 2.496000, + "Standard_D32lds_v5": 2.163000, + "Standard_D32ls_v5": 1.893000, + "Standard_D32s_v3": 2.122000, + "Standard_D32s_v4": 2.122000, + "Standard_D32s_v5": 2.122000, + "Standard_D3_v2": 0.353000, + "Standard_D3_v2_Promo": 0.353000, + "Standard_D48_v3": 3.182000, + "Standard_D48_v4": 3.182000, + "Standard_D48_v5": 3.182000, + "Standard_D48a_v4": 3.182000, + "Standard_D48ads_v5": 5.640000, + "Standard_D48as_v4": 3.182000, + "Standard_D48as_v5": 2.870000, + "Standard_D48as_v5_Promo": 2.558000, + "Standard_D48d_v4": 3.744000, + "Standard_D48d_v5": 3.744000, + "Standard_D48ds_v5": 3.744000, + "Standard_D48lds_v5": 3.245000, + "Standard_D48ls_v5": 2.839000, + "Standard_D48s_v3": 3.182000, + "Standard_D48s_v4": 3.182000, + "Standard_D48s_v5": 3.182000, + "Standard_D4_v2": 0.707000, + "Standard_D4_v2_Promo": 0.707000, + "Standard_D4_v3": 0.265000, + "Standard_D4_v4": 0.265000, + "Standard_D4_v5": 0.265000, + "Standard_D4a_v4": 0.265000, + "Standard_D4ads_v5": 0.286000, + "Standard_D4as_v4": 0.265000, + "Standard_D4as_v5": 0.423000, + "Standard_D4as_v5_Promo": 0.213000, + "Standard_D4d_v4": 0.312000, + "Standard_D4d_v5": 0.312000, + "Standard_D4ds_v5": 0.312000, + "Standard_D4lds_v5": 0.270000, + "Standard_D4ls_v5": 0.237000, + "Standard_D4s_v3": 0.265000, + "Standard_D4s_v4": 0.265000, + "Standard_D4s_v5": 0.265000, + "Standard_D5_v2": 1.414000, + "Standard_D5_v2_Promo": 1.414000, + "Standard_D64-16s_v3": 4.243000, + "Standard_D64-32s_v3": 4.243000, + "Standard_D64_v3": 4.243000, + "Standard_D64_v4": 4.243000, + "Standard_D64_v5": 4.243000, + "Standard_D64a_v4": 4.243000, + "Standard_D64ads_v5": 4.576000, + "Standard_D64as_v4": 4.243000, + "Standard_D64as_v5": 6.771000, + "Standard_D64as_v5_Promo": 3.411000, + "Standard_D64d_v4": 4.992000, + "Standard_D64d_v5": 4.992000, + "Standard_D64ds_v5": 4.992000, + "Standard_D64lds_v5": 4.326000, + "Standard_D64ls_v5": 3.786000, + "Standard_D64s_v3": 4.243000, + "Standard_D64s_v4": 4.243000, + "Standard_D64s_v5": 4.243000, + "Standard_D8_v3": 0.530000, + "Standard_D8_v4": 0.530000, + "Standard_D8_v5": 0.530000, + "Standard_D8a_v4": 0.530000, + "Standard_D8ads_v5": 0.940000, + "Standard_D8as_v4": 0.530000, + "Standard_D8as_v5": 0.846000, + "Standard_D8as_v5_Promo": 0.426000, + "Standard_D8d_v4": 0.624000, + "Standard_D8d_v5": 0.624000, + "Standard_D8ds_v5": 0.624000, + "Standard_D8lds_v5": 0.541000, + "Standard_D8ls_v5": 0.473000, + "Standard_D8s_v3": 0.530000, + "Standard_D8s_v4": 0.530000, + "Standard_D8s_v5": 0.530000, + "Standard_D96_v5": 6.365000, + "Standard_D96a_v4": 6.365000, + "Standard_D96ads_v5": 11.280000, + "Standard_D96as_v4": 6.365000, + "Standard_D96as_v5": 5.741000, + "Standard_D96as_v5_Promo": 5.117000, + "Standard_D96d_v5": 7.488000, + "Standard_D96ds_v5": 7.488000, + "Standard_D96lds_v5": 6.490000, + "Standard_D96ls_v5": 5.678000, + "Standard_D96s_v5": 6.365000, + "Standard_DS11-1_v2": 0.223000, + "Standard_DS11_v2": 0.223000, + "Standard_DS11_v2_Promo": 0.223000, + "Standard_DS12-1_v2": 0.449000, + "Standard_DS12-2_v2": 0.449000, + "Standard_DS12_v2": 0.449000, + "Standard_DS12_v2_Promo": 0.449000, + "Standard_DS13-2_v2": 0.898000, + "Standard_DS13-4_v2": 0.898000, + "Standard_DS13_v2": 0.898000, + "Standard_DS13_v2_Promo": 0.898000, + "Standard_DS14-4_v2": 1.795000, + "Standard_DS14-8_v2": 1.795000, + "Standard_DS14_v2": 1.795000, + "Standard_DS14_v2_Promo": 1.795000, + "Standard_DS15_v2": 2.244000, + "Standard_DS15i_v2": 2.244000, + "Standard_DS1_v2": 0.088400, + "Standard_DS2_v2": 0.177000, + "Standard_DS2_v2_Promo": 0.177000, + "Standard_DS3_v2": 0.353000, + "Standard_DS3_v2_Promo": 0.353000, + "Standard_DS4_v2": 0.707000, + "Standard_DS4_v2_Promo": 0.707000, + "Standard_DS5_v2": 1.414000, + "Standard_DS5_v2_Promo": 1.414000, + "Standard_E104i_v5": 9.964000, + "Standard_E104id_v5": 11.303000, + "Standard_E104ids_v5": 11.303000, + "Standard_E104is_v5": 9.964000, + "Standard_E112iads_v5": 11.131000, + "Standard_E112ias_v5": 9.690000, + "Standard_E112ibds_v5": 14.214000, + "Standard_E112ibs_v5": 12.773000, + "Standard_E16-4ads_v5": 2.182000, + "Standard_E16-4as_v4": 1.394000, + "Standard_E16-4as_v5": 1.258000, + "Standard_E16-4ds_v4": 1.581000, + "Standard_E16-4ds_v5": 1.581000, + "Standard_E16-4s_v3": 1.394000, + "Standard_E16-4s_v5": 1.394000, + "Standard_E16-8ads_v5": 1.446000, + "Standard_E16-8as_v4": 1.394000, + "Standard_E16-8as_v5": 1.258000, + "Standard_E16-8ds_v4": 1.581000, + "Standard_E16-8ds_v5": 1.581000, + "Standard_E16-8s_v3": 1.394000, + "Standard_E16-8s_v5": 1.394000, + "Standard_E16_v3": 1.394000, + "Standard_E16_v5": 1.394000, + "Standard_E16a_v4": 1.394000, + "Standard_E16ads_v5": 1.446000, + "Standard_E16as_v4": 1.394000, + "Standard_E16as_v5": 1.304000, + "Standard_E16bds_v5": 1.846000, + "Standard_E16bs_v5": 1.659000, + "Standard_E16d_v4": 1.581000, + "Standard_E16d_v5": 1.581000, + "Standard_E16ds_v4": 1.581000, + "Standard_E16ds_v5": 1.581000, + "Standard_E16s_v3": 1.394000, + "Standard_E16s_v5": 1.394000, + "Standard_E20_v3": 1.742000, + "Standard_E20_v5": 1.742000, + "Standard_E20a_v4": 1.742000, + "Standard_E20ads_v5": 1.807000, + "Standard_E20as_v4": 1.742000, + "Standard_E20as_v5": 1.573000, + "Standard_E20d_v4": 1.976000, + "Standard_E20d_v5": 1.976000, + "Standard_E20ds_v4": 1.976000, + "Standard_E20ds_v5": 1.976000, + "Standard_E20s_v3": 1.742000, + "Standard_E20s_v5": 1.742000, + "Standard_E2_v3": 0.174000, + "Standard_E2_v5": 0.174000, + "Standard_E2a_v4": 0.174000, + "Standard_E2ads_v5": 0.227000, + "Standard_E2as_v4": 0.174000, + "Standard_E2as_v5": 0.203000, + "Standard_E2bds_v5": 0.231000, + "Standard_E2bs_v5": 0.207000, + "Standard_E2d_v4": 0.198000, + "Standard_E2d_v5": 0.198000, + "Standard_E2ds_v4": 0.198000, + "Standard_E2ds_v5": 0.198000, + "Standard_E2s_v3": 0.174000, + "Standard_E2s_v5": 0.174000, + "Standard_E32-16ads_v5": 4.363000, + "Standard_E32-16as_v4": 2.787000, + "Standard_E32-16as_v5": 2.517000, + "Standard_E32-16ds_v4": 3.162000, + "Standard_E32-16ds_v5": 3.162000, + "Standard_E32-16s_v3": 2.787000, + "Standard_E32-16s_v5": 2.787000, + "Standard_E32-8ads_v5": 4.363000, + "Standard_E32-8as_v4": 2.787000, + "Standard_E32-8as_v5": 2.517000, + "Standard_E32-8ds_v4": 3.162000, + "Standard_E32-8ds_v5": 3.162000, + "Standard_E32-8s_v3": 2.787000, + "Standard_E32-8s_v5": 2.787000, + "Standard_E32_v3": 2.787000, + "Standard_E32_v5": 2.787000, + "Standard_E32a_v4": 2.787000, + "Standard_E32ads_v5": 2.937000, + "Standard_E32as_v4": 2.787000, + "Standard_E32as_v5": 2.563000, + "Standard_E32bds_v5": 3.692000, + "Standard_E32bs_v5": 3.318000, + "Standard_E32d_v4": 3.162000, + "Standard_E32d_v5": 3.162000, + "Standard_E32ds_v4": 3.162000, + "Standard_E32ds_v5": 3.162000, + "Standard_E32s_v3": 2.787000, + "Standard_E32s_v5": 2.787000, + "Standard_E4-2ads_v5": 0.361000, + "Standard_E4-2as_v4": 0.348000, + "Standard_E4-2as_v5": 0.499000, + "Standard_E4-2ds_v4": 0.395000, + "Standard_E4-2ds_v5": 0.395000, + "Standard_E4-2s_v3": 0.348000, + "Standard_E4-2s_v5": 0.348000, + "Standard_E48_v3": 4.181000, + "Standard_E48_v5": 4.181000, + "Standard_E48a_v4": 4.181000, + "Standard_E48ads_v5": 4.337000, + "Standard_E48as_v4": 4.181000, + "Standard_E48as_v5": 3.821000, + "Standard_E48bds_v5": 5.538000, + "Standard_E48bs_v5": 4.976000, + "Standard_E48d_v4": 4.742000, + "Standard_E48d_v5": 4.742000, + "Standard_E48ds_v4": 4.742000, + "Standard_E48ds_v5": 4.742000, + "Standard_E48s_v3": 4.181000, + "Standard_E48s_v5": 4.181000, + "Standard_E4_v3": 0.348000, + "Standard_E4_v5": 0.348000, + "Standard_E4a_v4": 0.348000, + "Standard_E4ads_v5": 0.407000, + "Standard_E4as_v4": 0.348000, + "Standard_E4as_v5": 0.361000, + "Standard_E4bds_v5": 0.462000, + "Standard_E4bs_v5": 0.415000, + "Standard_E4d_v4": 0.395000, + "Standard_E4d_v5": 0.395000, + "Standard_E4ds_v4": 0.395000, + "Standard_E4ds_v5": 0.395000, + "Standard_E4s_v3": 0.348000, + "Standard_E4s_v5": 0.348000, + "Standard_E64-16ads_v5": 5.782000, + "Standard_E64-16as_v4": 5.574000, + "Standard_E64-16as_v5": 5.034000, + "Standard_E64-16ds_v4": 6.323000, + "Standard_E64-16ds_v5": 6.323000, + "Standard_E64-16s_v3": 5.574000, + "Standard_E64-16s_v5": 5.574000, + "Standard_E64-32ads_v5": 5.782000, + "Standard_E64-32as_v4": 5.574000, + "Standard_E64-32as_v5": 7.978000, + "Standard_E64-32ds_v4": 6.323000, + "Standard_E64-32ds_v5": 6.323000, + "Standard_E64-32s_v3": 5.574000, + "Standard_E64-32s_v5": 5.574000, + "Standard_E64_v3": 5.574000, + "Standard_E64_v5": 5.574000, + "Standard_E64a_v4": 5.574000, + "Standard_E64ads_v5": 5.782000, + "Standard_E64as_v4": 5.574000, + "Standard_E64as_v5": 5.080000, + "Standard_E64bds_v5": 7.384000, + "Standard_E64bs_v5": 6.635000, + "Standard_E64d_v4": 6.323000, + "Standard_E64d_v5": 6.323000, + "Standard_E64ds_v4": 6.323000, + "Standard_E64ds_v5": 6.323000, + "Standard_E64i_v3": 5.574000, + "Standard_E64is_v3": 5.574000, + "Standard_E64s_v3": 5.574000, + "Standard_E64s_v5": 5.574000, + "Standard_E8-2ads_v5": 1.091000, + "Standard_E8-2as_v4": 0.697000, + "Standard_E8-2as_v5": 0.629000, + "Standard_E8-2ds_v4": 0.790000, + "Standard_E8-2ds_v5": 0.790000, + "Standard_E8-2s_v3": 0.697000, + "Standard_E8-2s_v5": 0.697000, + "Standard_E8-4ads_v5": 1.091000, + "Standard_E8-4as_v4": 0.697000, + "Standard_E8-4as_v5": 0.997000, + "Standard_E8-4ds_v4": 0.790000, + "Standard_E8-4ds_v5": 0.790000, + "Standard_E8-4s_v3": 0.697000, + "Standard_E8-4s_v5": 0.697000, + "Standard_E80ids_v4": 7.904000, + "Standard_E8_v3": 0.697000, + "Standard_E8_v5": 0.697000, + "Standard_E8a_v4": 0.697000, + "Standard_E8ads_v5": 0.723000, + "Standard_E8as_v4": 0.697000, + "Standard_E8as_v5": 0.629000, + "Standard_E8bds_v5": 0.923000, + "Standard_E8bs_v5": 0.829000, + "Standard_E8d_v4": 0.790000, + "Standard_E8d_v5": 0.790000, + "Standard_E8ds_v4": 0.790000, + "Standard_E8ds_v5": 0.790000, + "Standard_E8s_v3": 0.697000, + "Standard_E8s_v5": 0.697000, + "Standard_E96-24ads_v5": 13.090000, + "Standard_E96-24as_v4": 8.362000, + "Standard_E96-24as_v5": 11.966000, + "Standard_E96-24ds_v5": 9.485000, + "Standard_E96-24s_v5": 8.362000, + "Standard_E96-48ads_v5": 13.090000, + "Standard_E96-48as_v4": 8.362000, + "Standard_E96-48as_v5": 7.550000, + "Standard_E96-48ds_v5": 9.485000, + "Standard_E96-48s_v5": 8.362000, + "Standard_E96_v5": 8.362000, + "Standard_E96a_v4": 8.362000, + "Standard_E96ads_v5": 8.720000, + "Standard_E96as_v4": 8.362000, + "Standard_E96as_v5": 7.550000, + "Standard_E96bds_v5": 11.076000, + "Standard_E96bs_v5": 9.953000, + "Standard_E96d_v5": 9.485000, + "Standard_E96ds_v5": 9.485000, + "Standard_E96iads_v5": 9.541000, + "Standard_E96ias_v4": 8.362000, + "Standard_E96ias_v5": 8.305000, + "Standard_E96s_v5": 8.362000, + "Standard_F1": 0.073900, + "Standard_F16": 1.182000, + "Standard_F16s": 1.182000, + "Standard_F16s_v2": 0.952000, + "Standard_F1s": 0.073900, + "Standard_F2": 0.148000, + "Standard_F2s": 0.148000, + "Standard_F2s_v2": 0.119000, + "Standard_F32s_v2": 1.904000, + "Standard_F4": 0.295000, + "Standard_F48s_v2": 2.856000, + "Standard_F4s": 0.295000, + "Standard_F4s_v2": 0.238000, + "Standard_F64s_v2": 3.808000, + "Standard_F72s_v2": 4.284000, + "Standard_F8": 0.591000, + "Standard_F8s": 0.591000, + "Standard_F8s_v2": 0.476000, + "Standard_L16as_v3": 1.695000, + "Standard_L16s_v3": 1.893000, + "Standard_L32as_v3": 3.390000, + "Standard_L32s_v3": 3.786000, + "Standard_L48as_v3": 5.086000, + "Standard_L48s_v3": 5.678000, + "Standard_L64as_v3": 6.781000, + "Standard_L64s_v3": 7.571000, + "Standard_L80as_v3": 8.476000, + "Standard_L80s_v3": 9.464000, + "Standard_L8as_v3": 0.848000, + "Standard_L8s_v3": 0.946000, + } + // switzerlandnorth + initialOnDemandPrices["switzerlandnorth"] = map[string]float64{ + "DCdsv3 Type1": 8.950000, + "DCsv3 Type1": 7.603000, + "Dadsv5_Type1": 9.073000, + "Dasv4_Type1": 7.550000, + "Dasv4_Type2": 7.550000, + "Dasv5_Type1": 7.576000, + "Ddsv4_Type 1": 4.452800, + "Ddsv4_Type2": 7.085000, + "Ddsv5_Type1": 7.899000, + "Dsv3_Type3": 5.566000, + "Dsv3_Type4": 6.866000, + "Dsv4_Type1": 5.566000, + "Dsv4_Type2": 7.920000, + "Dsv5_Type1": 6.958000, + "Easv4_Type1": 9.514000, + "Easv4_Type2": 9.514000, + "Easv5_Type1": 8.532000, + "Ebdsv5-Type1": 8.406000, + "Ebsv5-Type1": 7.500000, + "Edsv4_Type 1": 6.699000, + "Edsv4_Type2": 9.029000, + "Edsv5_Type1": 10.032000, + "Esv3_Type3": 5.586000, + "Esv3_Type4": 7.568000, + "Esv4_Type1": 5.886100, + "Esv4_Type2": 8.732000, + "Esv5_Type1": 8.818000, + "Fsv2 Type3": 4.929100, + "Fsv2_Type2": 4.225100, + "Fsv2_Type4": 6.078000, + "Lasv3_Type1": 8.998000, + "Lsv3_Type1": 10.980000, + "Mdmsv2MedMem _Type1": 45.213300, + "Mdsv2MedMem_Type1": 22.598400, + "Msmv2MedMem Type1": 44.500500, + "Msmv2_Type1": 170.140000, + "Msv2MedMem Type1": 21.885600, + "Msv2_Type1": 85.079000, + "NDamsrA100v4_Type1": 51.547000, + "NVsv3_Type1": 6.897000, + "Standard_A1_v2": 0.045100, + "Standard_A2_v2": 0.095700, + "Standard_A2m_v2": 0.136400, + "Standard_A4_v2": 0.197000, + "Standard_A4m_v2": 0.286000, + "Standard_A8_v2": 0.394000, + "Standard_A8m_v2": 0.595000, + "Standard_B12ms": 0.633600, + "Standard_B16als_v2": 0.673000, + "Standard_B16as_v2": 0.834000, + "Standard_B16ls_v2": 0.822000, + "Standard_B16ms": 0.844800, + "Standard_B16s_v2": 0.845000, + "Standard_B1ls": 0.006600, + "Standard_B1ms": 0.026400, + "Standard_B1s": 0.013200, + "Standard_B20ms": 1.056000, + "Standard_B2als_v2": 0.047500, + "Standard_B2as_v2": 0.095000, + "Standard_B2ats_v2": 0.011900, + "Standard_B2ls_v2": 0.052800, + "Standard_B2ms": 0.105600, + "Standard_B2s": 0.052800, + "Standard_B2s_v2": 0.106000, + "Standard_B2ts_v2": 0.022400, + "Standard_B32als_v2": 1.494000, + "Standard_B32as_v2": 1.668000, + "Standard_B32ls_v2": 1.643000, + "Standard_B32s_v2": 1.837000, + "Standard_B4als_v2": 0.168000, + "Standard_B4as_v2": 0.208000, + "Standard_B4ls_v2": 0.206000, + "Standard_B4ms": 0.211200, + "Standard_B4s_v2": 0.230000, + "Standard_B8als_v2": 0.374000, + "Standard_B8as_v2": 0.417000, + "Standard_B8ls_v2": 0.411000, + "Standard_B8ms": 0.422400, + "Standard_B8s_v2": 0.422000, + "Standard_D11_v2": 0.209000, + "Standard_D12_v2": 0.416900, + "Standard_D13_v2": 0.834900, + "Standard_D14_v2": 1.669800, + "Standard_D15_v2": 2.086700, + "Standard_D15i_v2": 2.086700, + "Standard_D16_v3": 1.056000, + "Standard_D16_v4": 1.012000, + "Standard_D16_v5": 1.012000, + "Standard_D16a_v4": 1.098000, + "Standard_D16ads_v5": 1.914000, + "Standard_D16as_v4": 1.098000, + "Standard_D16as_v5": 0.984000, + "Standard_D16d_v4": 1.196800, + "Standard_D16d_v5": 1.197000, + "Standard_D16ds_v4": 1.196800, + "Standard_D16ds_v5": 1.197000, + "Standard_D16lds_v5": 1.098000, + "Standard_D16ls_v5": 0.972000, + "Standard_D16s_v3": 1.056000, + "Standard_D16s_v4": 1.012000, + "Standard_D16s_v5": 1.012000, + "Standard_D1_v2": 0.074690, + "Standard_D2_v2": 0.149600, + "Standard_D2_v3": 0.132000, + "Standard_D2_v4": 0.126500, + "Standard_D2_v5": 0.127000, + "Standard_D2a_v4": 0.137000, + "Standard_D2ads_v5": 0.239000, + "Standard_D2as_v4": 0.137000, + "Standard_D2as_v5": 0.123000, + "Standard_D2d_v4": 0.149600, + "Standard_D2d_v5": 0.150000, + "Standard_D2ds_v4": 0.149600, + "Standard_D2ds_v5": 0.150000, + "Standard_D2lds_v5": 0.137000, + "Standard_D2ls_v5": 0.122000, + "Standard_D2s_v3": 0.132000, + "Standard_D2s_v4": 0.126500, + "Standard_D2s_v5": 0.127000, + "Standard_D32-16s_v3": 2.112000, + "Standard_D32-8s_v3": 2.112000, + "Standard_D32_v3": 2.112000, + "Standard_D32_v4": 2.024000, + "Standard_D32_v5": 2.024000, + "Standard_D32a_v4": 2.196000, + "Standard_D32ads_v5": 2.357000, + "Standard_D32as_v4": 2.196000, + "Standard_D32as_v5": 1.968000, + "Standard_D32d_v4": 2.393600, + "Standard_D32d_v5": 2.394000, + "Standard_D32ds_v4": 2.393600, + "Standard_D32ds_v5": 2.394000, + "Standard_D32lds_v5": 2.196000, + "Standard_D32ls_v5": 1.945000, + "Standard_D32s_v3": 2.112000, + "Standard_D32s_v4": 2.024000, + "Standard_D32s_v5": 2.024000, + "Standard_D3_v2": 0.299200, + "Standard_D48_v3": 3.168000, + "Standard_D48_v4": 3.036000, + "Standard_D48_v5": 3.036000, + "Standard_D48a_v4": 3.295000, + "Standard_D48ads_v5": 3.535000, + "Standard_D48as_v4": 3.295000, + "Standard_D48as_v5": 5.160000, + "Standard_D48d_v4": 3.590400, + "Standard_D48d_v5": 3.590000, + "Standard_D48ds_v4": 3.590400, + "Standard_D48ds_v5": 3.590000, + "Standard_D48lds_v5": 3.295000, + "Standard_D48ls_v5": 2.917000, + "Standard_D48s_v3": 3.168000, + "Standard_D48s_v4": 3.036000, + "Standard_D48s_v5": 3.036000, + "Standard_D4_v2": 0.598400, + "Standard_D4_v3": 0.264000, + "Standard_D4_v4": 0.253000, + "Standard_D4_v5": 0.253000, + "Standard_D4a_v4": 0.275000, + "Standard_D4ads_v5": 0.295000, + "Standard_D4as_v4": 0.275000, + "Standard_D4as_v5": 0.246000, + "Standard_D4d_v4": 0.299200, + "Standard_D4d_v5": 0.299000, + "Standard_D4ds_v4": 0.299200, + "Standard_D4ds_v5": 0.299000, + "Standard_D4lds_v5": 0.275000, + "Standard_D4ls_v5": 0.243000, + "Standard_D4s_v3": 0.264000, + "Standard_D4s_v4": 0.253000, + "Standard_D4s_v5": 0.253000, + "Standard_D5_v2": 1.195700, + "Standard_D64-16s_v3": 4.224000, + "Standard_D64-32s_v3": 4.224000, + "Standard_D64_v3": 4.224000, + "Standard_D64_v4": 4.048000, + "Standard_D64_v5": 4.048000, + "Standard_D64a_v4": 4.393000, + "Standard_D64ads_v5": 4.713000, + "Standard_D64as_v4": 4.393000, + "Standard_D64as_v5": 3.935000, + "Standard_D64d_v4": 4.787200, + "Standard_D64d_v5": 4.787000, + "Standard_D64ds_v4": 4.787200, + "Standard_D64ds_v5": 4.787000, + "Standard_D64lds_v5": 4.393000, + "Standard_D64ls_v5": 3.890000, + "Standard_D64s_v3": 4.224000, + "Standard_D64s_v4": 4.048000, + "Standard_D64s_v5": 4.048000, + "Standard_D8_v3": 0.528000, + "Standard_D8_v4": 0.506000, + "Standard_D8_v5": 0.506000, + "Standard_D8a_v4": 0.549000, + "Standard_D8ads_v5": 0.589000, + "Standard_D8as_v4": 0.549000, + "Standard_D8as_v5": 0.492000, + "Standard_D8d_v4": 0.598400, + "Standard_D8d_v5": 0.598000, + "Standard_D8ds_v4": 0.598400, + "Standard_D8ds_v5": 0.598000, + "Standard_D8lds_v5": 0.549000, + "Standard_D8ls_v5": 0.486000, + "Standard_D8s_v3": 0.528000, + "Standard_D8s_v4": 0.506000, + "Standard_D8s_v5": 0.506000, + "Standard_D96_v5": 6.072000, + "Standard_D96a_v4": 6.589000, + "Standard_D96ads_v5": 7.070000, + "Standard_D96as_v4": 6.589000, + "Standard_D96as_v5": 5.903000, + "Standard_D96d_v5": 7.181000, + "Standard_D96ds_v5": 7.181000, + "Standard_D96lds_v5": 6.589000, + "Standard_D96ls_v5": 5.834000, + "Standard_D96s_v5": 6.072000, + "Standard_DC16ads_v5": 1.178000, + "Standard_DC16as_v5": 0.984000, + "Standard_DC16ds_v3": 2.712000, + "Standard_DC16s_v3": 2.304000, + "Standard_DC1ds_v3": 0.170000, + "Standard_DC1s_v3": 0.144000, + "Standard_DC24ds_v3": 4.068000, + "Standard_DC24s_v3": 3.456000, + "Standard_DC2ads_v5": 0.147000, + "Standard_DC2as_v5": 0.123000, + "Standard_DC2ds_v3": 0.339000, + "Standard_DC2s_v3": 0.288000, + "Standard_DC32ads_v5": 2.357000, + "Standard_DC32as_v5": 1.968000, + "Standard_DC32ds_v3": 5.424000, + "Standard_DC32s_v3": 4.608000, + "Standard_DC48ads_v5": 3.535000, + "Standard_DC48as_v5": 2.952000, + "Standard_DC48ds_v3": 8.136000, + "Standard_DC48s_v3": 6.912000, + "Standard_DC4ads_v5": 0.295000, + "Standard_DC4as_v5": 0.246000, + "Standard_DC4ds_v3": 0.678000, + "Standard_DC4s_v3": 0.576000, + "Standard_DC64ads_v5": 4.713000, + "Standard_DC64as_v5": 3.935000, + "Standard_DC8ads_v5": 0.589000, + "Standard_DC8as_v5": 0.492000, + "Standard_DC8ds_v3": 1.356000, + "Standard_DC8s_v3": 1.152000, + "Standard_DC96ads_v5": 7.070000, + "Standard_DC96as_v5": 5.903000, + "Standard_DS11-1_v2": 0.209000, + "Standard_DS11_v2": 0.209000, + "Standard_DS11_v2_Promo": 0.282125, + "Standard_DS12-1_v2": 0.416900, + "Standard_DS12-2_v2": 0.416900, + "Standard_DS12_v2": 0.416900, + "Standard_DS12_v2_Promo": 0.565775, + "Standard_DS13-2_v2": 0.834900, + "Standard_DS13-4_v2": 0.834900, + "Standard_DS13_v2": 0.834900, + "Standard_DS13_v2_Promo": 1.130025, + "Standard_DS14-4_v2": 1.669800, + "Standard_DS14-8_v2": 1.669800, + "Standard_DS14_v2": 1.669800, + "Standard_DS14_v2_Promo": 2.260050, + "Standard_DS15_v2": 2.086700, + "Standard_DS15i_v2": 2.086700, + "Standard_DS1_v2": 0.074690, + "Standard_DS2_v2": 0.149600, + "Standard_DS2_v2_Promo": 0.208050, + "Standard_DS3_v2": 0.299200, + "Standard_DS3_v2_Promo": 0.343000, + "Standard_DS4_v2": 0.598400, + "Standard_DS5_v2": 1.195700, + "Standard_E104i_v5": 9.550000, + "Standard_E104id_v5": 10.900000, + "Standard_E104ids_v5": 10.868000, + "Standard_E104is_v5": 9.552000, + "Standard_E112iads_v5": 11.540000, + "Standard_E112ias_v5": 9.954000, + "Standard_E112ibds_v5": 14.711000, + "Standard_E112ibs_v5": 13.125000, + "Standard_E16-4ads_v5": 2.235000, + "Standard_E16-4as_v4": 1.441000, + "Standard_E16-4as_v5": 1.293000, + "Standard_E16-4ds_v4": 1.522400, + "Standard_E16-4ds_v5": 1.520000, + "Standard_E16-4s_v3": 1.408000, + "Standard_E16-4s_v4": 1.337600, + "Standard_E16-4s_v5": 1.336000, + "Standard_E16-8ads_v5": 2.235000, + "Standard_E16-8as_v4": 1.441000, + "Standard_E16-8as_v5": 1.293000, + "Standard_E16-8ds_v4": 1.522400, + "Standard_E16-8ds_v5": 1.520000, + "Standard_E16-8s_v3": 1.408000, + "Standard_E16-8s_v4": 1.337600, + "Standard_E16-8s_v5": 1.336000, + "Standard_E16_v3": 1.408000, + "Standard_E16_v4": 1.337600, + "Standard_E16_v5": 1.338000, + "Standard_E16a_v4": 1.441000, + "Standard_E16ads_v5": 1.545000, + "Standard_E16as_v4": 1.441000, + "Standard_E16as_v5": 1.339000, + "Standard_E16bds_v5": 1.910000, + "Standard_E16bs_v5": 1.705000, + "Standard_E16d_v4": 1.522400, + "Standard_E16d_v5": 1.522000, + "Standard_E16ds_v4": 1.522400, + "Standard_E16ds_v5": 1.520000, + "Standard_E16s_v3": 1.408000, + "Standard_E16s_v4": 1.337600, + "Standard_E16s_v5": 1.336000, + "Standard_E20_v3": 1.760000, + "Standard_E20_v4": 1.672000, + "Standard_E20_v5": 1.672000, + "Standard_E20a_v4": 1.802000, + "Standard_E20ads_v5": 1.919000, + "Standard_E20as_v4": 1.802000, + "Standard_E20as_v5": 1.662000, + "Standard_E20d_v4": 1.903000, + "Standard_E20d_v5": 1.903000, + "Standard_E20ds_v4": 1.903000, + "Standard_E20ds_v5": 1.900000, + "Standard_E20s_v3": 1.760000, + "Standard_E20s_v4": 1.672000, + "Standard_E20s_v5": 1.670000, + "Standard_E2_v3": 0.176000, + "Standard_E2_v4": 0.167200, + "Standard_E2_v5": 0.167000, + "Standard_E2a_v4": 0.180000, + "Standard_E2ads_v5": 0.187000, + "Standard_E2as_v4": 0.180000, + "Standard_E2as_v5": 0.208000, + "Standard_E2bds_v5": 0.239000, + "Standard_E2bs_v5": 0.213000, + "Standard_E2d_v4": 0.190300, + "Standard_E2d_v5": 0.190000, + "Standard_E2ds_v4": 0.190300, + "Standard_E2ds_v5": 0.190000, + "Standard_E2s_v3": 0.176000, + "Standard_E2s_v4": 0.167200, + "Standard_E2s_v5": 0.167000, + "Standard_E32-16ads_v5": 2.997000, + "Standard_E32-16as_v4": 2.883000, + "Standard_E32-16as_v5": 2.585000, + "Standard_E32-16ds_v4": 3.044800, + "Standard_E32-16ds_v5": 3.040000, + "Standard_E32-16s_v3": 2.816000, + "Standard_E32-16s_v4": 2.675200, + "Standard_E32-16s_v5": 2.672000, + "Standard_E32-8ads_v5": 4.469000, + "Standard_E32-8as_v4": 2.883000, + "Standard_E32-8as_v5": 2.585000, + "Standard_E32-8ds_v4": 3.044800, + "Standard_E32-8ds_v5": 3.040000, + "Standard_E32-8s_v3": 2.816000, + "Standard_E32-8s_v4": 2.675200, + "Standard_E32-8s_v5": 2.672000, + "Standard_E32_v3": 2.816000, + "Standard_E32_v4": 2.675200, + "Standard_E32_v5": 2.675000, + "Standard_E32a_v4": 2.883000, + "Standard_E32ads_v5": 3.043000, + "Standard_E32as_v4": 2.883000, + "Standard_E32as_v5": 2.585000, + "Standard_E32bds_v5": 3.821000, + "Standard_E32bs_v5": 3.409000, + "Standard_E32d_v4": 3.044800, + "Standard_E32d_v5": 3.045000, + "Standard_E32ds_v4": 3.044800, + "Standard_E32ds_v5": 3.040000, + "Standard_E32s_v3": 2.816000, + "Standard_E32s_v4": 2.675200, + "Standard_E32s_v5": 2.672000, + "Standard_E4-2ads_v5": 0.559000, + "Standard_E4-2as_v4": 0.360000, + "Standard_E4-2as_v5": 0.323000, + "Standard_E4-2ds_v4": 0.380600, + "Standard_E4-2ds_v5": 0.380000, + "Standard_E4-2s_v3": 0.352000, + "Standard_E4-2s_v4": 0.334400, + "Standard_E4-2s_v5": 0.334000, + "Standard_E48_v3": 4.011700, + "Standard_E48_v4": 4.012800, + "Standard_E48_v5": 4.013000, + "Standard_E48a_v4": 4.324000, + "Standard_E48ads_v5": 4.542000, + "Standard_E48as_v4": 4.324000, + "Standard_E48as_v5": 3.924000, + "Standard_E48bds_v5": 5.731000, + "Standard_E48bs_v5": 5.114000, + "Standard_E48d_v4": 4.567200, + "Standard_E48d_v5": 4.567000, + "Standard_E48ds_v4": 4.567200, + "Standard_E48ds_v5": 4.560000, + "Standard_E48s_v3": 4.011700, + "Standard_E48s_v4": 4.012800, + "Standard_E48s_v5": 4.008000, + "Standard_E4_v3": 0.352000, + "Standard_E4_v4": 0.334400, + "Standard_E4_v5": 0.334000, + "Standard_E4a_v4": 0.360000, + "Standard_E4ads_v5": 0.375000, + "Standard_E4as_v4": 0.360000, + "Standard_E4as_v5": 0.323000, + "Standard_E4bds_v5": 0.478000, + "Standard_E4bs_v5": 0.426000, + "Standard_E4d_v4": 0.380600, + "Standard_E4d_v5": 0.381000, + "Standard_E4ds_v4": 0.380600, + "Standard_E4ds_v5": 0.380000, + "Standard_E4s_v3": 0.352000, + "Standard_E4s_v4": 0.334400, + "Standard_E4s_v5": 0.334000, + "Standard_E64-16ads_v5": 8.939000, + "Standard_E64-16as_v4": 5.766000, + "Standard_E64-16as_v5": 5.171000, + "Standard_E64-16ds_v4": 6.089600, + "Standard_E64-16ds_v5": 6.080000, + "Standard_E64-16s_v3": 4.813600, + "Standard_E64-16s_v4": 5.350400, + "Standard_E64-16s_v5": 5.344000, + "Standard_E64-32ads_v5": 5.995000, + "Standard_E64-32as_v4": 5.766000, + "Standard_E64-32as_v5": 5.171000, + "Standard_E64-32ds_v4": 6.089600, + "Standard_E64-32ds_v5": 6.080000, + "Standard_E64-32s_v3": 4.813600, + "Standard_E64-32s_v4": 5.350400, + "Standard_E64-32s_v5": 5.344000, + "Standard_E64_v3": 4.813600, + "Standard_E64_v4": 5.350400, + "Standard_E64_v5": 5.350000, + "Standard_E64a_v4": 5.766000, + "Standard_E64ads_v5": 6.041000, + "Standard_E64as_v4": 5.766000, + "Standard_E64as_v5": 5.171000, + "Standard_E64bds_v5": 7.642000, + "Standard_E64bs_v5": 6.818000, + "Standard_E64d_v4": 6.089600, + "Standard_E64d_v5": 6.090000, + "Standard_E64ds_v4": 6.089600, + "Standard_E64ds_v5": 6.080000, + "Standard_E64i_v3": 4.813600, + "Standard_E64is_v3": 4.813600, + "Standard_E64s_v3": 4.813600, + "Standard_E64s_v4": 5.350400, + "Standard_E64s_v5": 5.344000, + "Standard_E8-2ads_v5": 1.117000, + "Standard_E8-2as_v4": 0.721000, + "Standard_E8-2as_v5": 1.014000, + "Standard_E8-2ds_v4": 0.761200, + "Standard_E8-2ds_v5": 0.760000, + "Standard_E8-2s_v3": 0.704000, + "Standard_E8-2s_v4": 0.668800, + "Standard_E8-2s_v5": 0.668000, + "Standard_E8-4ads_v5": 1.117000, + "Standard_E8-4as_v4": 0.721000, + "Standard_E8-4as_v5": 1.014000, + "Standard_E8-4ds_v4": 0.761200, + "Standard_E8-4ds_v5": 0.760000, + "Standard_E8-4s_v3": 0.704000, + "Standard_E8-4s_v4": 0.668800, + "Standard_E8-4s_v5": 0.668000, + "Standard_E80ids_v4": 7.612000, + "Standard_E80is_v4": 6.688000, + "Standard_E8_v3": 0.704000, + "Standard_E8_v4": 0.668800, + "Standard_E8_v5": 0.669000, + "Standard_E8a_v4": 0.721000, + "Standard_E8ads_v5": 0.795000, + "Standard_E8as_v4": 0.721000, + "Standard_E8as_v5": 0.692000, + "Standard_E8bds_v5": 0.955000, + "Standard_E8bs_v5": 0.852000, + "Standard_E8d_v4": 0.761200, + "Standard_E8d_v5": 0.761000, + "Standard_E8ds_v4": 0.761200, + "Standard_E8ds_v5": 0.760000, + "Standard_E8s_v3": 0.704000, + "Standard_E8s_v4": 0.668800, + "Standard_E8s_v5": 0.668000, + "Standard_E96-24ads_v5": 13.408000, + "Standard_E96-24as_v4": 8.649000, + "Standard_E96-24as_v5": 12.172000, + "Standard_E96-24ds_v5": 9.120000, + "Standard_E96-24s_v5": 8.016000, + "Standard_E96-48ads_v5": 8.992000, + "Standard_E96-48as_v4": 8.649000, + "Standard_E96-48as_v5": 7.756000, + "Standard_E96-48ds_v5": 9.120000, + "Standard_E96-48s_v5": 8.016000, + "Standard_E96_v5": 8.026000, + "Standard_E96a_v4": 8.649000, + "Standard_E96ads_v5": 9.038000, + "Standard_E96as_v4": 8.649000, + "Standard_E96as_v5": 7.756000, + "Standard_E96bds_v5": 11.463000, + "Standard_E96bs_v5": 10.227000, + "Standard_E96d_v5": 9.134000, + "Standard_E96ds_v5": 9.120000, + "Standard_E96iads_v5": 9.891000, + "Standard_E96ias_v4": 8.649000, + "Standard_E96ias_v5": 8.532000, + "Standard_E96s_v5": 8.016000, + "Standard_EC16ads_v5": 1.499000, + "Standard_EC16as_v5": 1.293000, + "Standard_EC20ads_v5": 1.873000, + "Standard_EC20as_v5": 1.616000, + "Standard_EC2ads_v5": 0.187000, + "Standard_EC2as_v5": 0.162000, + "Standard_EC32ads_v5": 2.997000, + "Standard_EC32as_v5": 2.585000, + "Standard_EC48ads_v5": 4.496000, + "Standard_EC48as_v5": 3.878000, + "Standard_EC4ads_v5": 0.375000, + "Standard_EC4as_v5": 0.323000, + "Standard_EC64ads_v5": 5.995000, + "Standard_EC64as_v5": 5.171000, + "Standard_EC8ads_v5": 0.749000, + "Standard_EC8as_v5": 0.646000, + "Standard_EC96ads_v5": 8.992000, + "Standard_EC96as_v5": 7.756000, + "Standard_EC96iads_v5": 9.891000, + "Standard_EC96ias_v5": 8.532000, + "Standard_F1": 0.062480, + "Standard_F16": 0.999900, + "Standard_F16s": 0.999900, + "Standard_F16s_v2": 0.853600, + "Standard_F1s": 0.062480, + "Standard_F2": 0.125400, + "Standard_F2s": 0.125400, + "Standard_F2s_v2": 0.106700, + "Standard_F32s_v2": 1.707200, + "Standard_F4": 0.249700, + "Standard_F48s_v2": 2.560800, + "Standard_F4s": 0.249700, + "Standard_F4s_v2": 0.213400, + "Standard_F64s_v2": 3.414400, + "Standard_F72s_v2": 3.841200, + "Standard_F8": 0.499400, + "Standard_F8s": 0.499400, + "Standard_F8s_v2": 0.426800, + "Standard_L16as_v3": 1.636000, + "Standard_L16s_v3": 1.996000, + "Standard_L32as_v3": 3.272000, + "Standard_L32s_v3": 3.993000, + "Standard_L48as_v3": 4.908000, + "Standard_L48s_v3": 5.989000, + "Standard_L64as_v3": 6.544000, + "Standard_L64s_v3": 7.985000, + "Standard_L80as_v3": 8.180000, + "Standard_L80s_v3": 9.981000, + "Standard_L8as_v3": 0.818000, + "Standard_L8s_v3": 0.998000, + "Standard_M128": 20.540300, + "Standard_M128-32ms": 41.099520, + "Standard_M128-64ms": 41.099520, + "Standard_M128dms_v2": 41.102600, + "Standard_M128ds_v2": 20.543600, + "Standard_M128m": 41.099520, + "Standard_M128ms": 41.099520, + "Standard_M128ms_v2": 40.454700, + "Standard_M128s": 20.540300, + "Standard_M128s_v2": 19.895700, + "Standard_M16-4ms": 4.732420, + "Standard_M16-8ms": 4.732420, + "Standard_M16ms": 4.732420, + "Standard_M16s": 3.675980, + "Standard_M192idms_v2": 49.377900, + "Standard_M192ids_v2": 24.689500, + "Standard_M192ims_v2": 48.730000, + "Standard_M192is_v2": 24.041600, + "Standard_M208-104ms_v2": 69.607000, + "Standard_M208-104s_v2": 34.804000, + "Standard_M208-52ms_v2": 69.607000, + "Standard_M208-52s_v2": 34.804000, + "Standard_M208ms_v2": 69.607000, + "Standard_M208s_v2": 34.804000, + "Standard_M32-16ms": 9.464840, + "Standard_M32-8ms": 9.464840, + "Standard_M32dms_v2": 9.464400, + "Standard_M32ls": 3.792360, + "Standard_M32ms": 9.464840, + "Standard_M32ms_v2": 9.302700, + "Standard_M32s": 5.135900, + "Standard_M32ts": 3.572800, + "Standard_M416-104ms_v2": 154.670000, + "Standard_M416-104s_v2": 77.345000, + "Standard_M416-208ms_v2": 154.670000, + "Standard_M416-208s_v2": 77.345000, + "Standard_M416is_v2": 77.345000, + "Standard_M416ms_v2": 154.670000, + "Standard_M416s_8_v2": 94.540000, + "Standard_M416s_v2": 77.345000, + "Standard_M64": 10.270700, + "Standard_M64-16ms": 15.919200, + "Standard_M64-32ms": 15.919200, + "Standard_M64dms_v2": 15.923600, + "Standard_M64ds_v2": 10.270700, + "Standard_M64ls": 7.147800, + "Standard_M64m": 15.919200, + "Standard_M64ms": 15.919200, + "Standard_M64ms_v2": 15.599100, + "Standard_M64s": 10.270700, + "Standard_M64s_v2": 9.946200, + "Standard_M8-2ms": 2.366210, + "Standard_M8-4ms": 2.366210, + "Standard_M8ms": 2.366210, + "Standard_NC12s_v3": 8.411000, + "Standard_NC24rs_v3": 18.503000, + "Standard_NC24s_v3": 16.821000, + "Standard_NC6s_v3": 4.205000, + "Standard_ND96ams_A100_v4": 46.861000, + "Standard_ND96amsr_A100_v4": 46.861000, + "Standard_NV12s_v3": 1.569000, + "Standard_NV24s_v3": 3.135000, + "Standard_NV48s_v3": 6.270000, + } + // switzerlandwest + initialOnDemandPrices["switzerlandwest"] = map[string]float64{ + "Dadsv5_Type1": 11.795000, + "Dasv4_Type1": 9.816000, + "Dasv4_Type2": 9.816000, + "Dasv5_Type1": 9.848000, + "Ddsv5_Type1": 10.276000, + "Dsv3_Type3": 5.915000, + "Dsv3_Type4": 7.394000, + "Dsv5_Type1": 9.035000, + "Eadsv5_Type1": 12.858000, + "Easv4_Type1": 12.368000, + "Easv4_Type2": 12.368000, + "Easv5_Type1": 11.092000, + "Ebdsv5-Type1": 10.928000, + "Ebsv5-Type1": 9.750000, + "Edsv5_Type1": 13.306000, + "Esv3_Type3": 6.016000, + "Esv3_Type4": 8.150000, + "Esv5_Type1": 11.669000, + "Fsv2_Type2": 4.549510, + "Lasv3_Type1": 11.697000, + "Lsv3_Type1": 14.273000, + "Ms_Type1": 32.037000, + "Msm_Type1": 64.129000, + "Msmv2_Type1": 221.400000, + "Msv2_Type1": 110.710000, + "Standard_A1_v2": 0.058670, + "Standard_A2_v2": 0.124490, + "Standard_A2m_v2": 0.176680, + "Standard_A4_v2": 0.256000, + "Standard_A4m_v2": 0.371420, + "Standard_A8_v2": 0.512000, + "Standard_A8m_v2": 0.774000, + "Standard_B12ms": 0.823130, + "Standard_B16als_v2": 0.875000, + "Standard_B16as_v2": 0.988000, + "Standard_B16ls_v2": 0.973000, + "Standard_B16ms": 1.097690, + "Standard_B16s_v2": 1.172000, + "Standard_B1ls": 0.008580, + "Standard_B1ms": 0.034280, + "Standard_B1s": 0.017140, + "Standard_B20ms": 1.371980, + "Standard_B2als_v2": 0.061800, + "Standard_B2as_v2": 0.133000, + "Standard_B2ats_v2": 0.024600, + "Standard_B2ls_v2": 0.068600, + "Standard_B2ms": 0.137740, + "Standard_B2s": 0.068660, + "Standard_B2s_v2": 0.146000, + "Standard_B2ts_v2": 0.026400, + "Standard_B32als_v2": 1.750000, + "Standard_B32as_v2": 2.124000, + "Standard_B32ls_v2": 2.092000, + "Standard_B32s_v2": 2.196000, + "Standard_B4als_v2": 0.237000, + "Standard_B4as_v2": 0.266000, + "Standard_B4ls_v2": 0.243000, + "Standard_B4ms": 0.275480, + "Standard_B4s_v2": 0.275000, + "Standard_B8als_v2": 0.475000, + "Standard_B8as_v2": 0.531000, + "Standard_B8ls_v2": 0.523000, + "Standard_B8ms": 0.548850, + "Standard_B8s_v2": 0.549000, + "Standard_D11_v2": 0.271330, + "Standard_D12_v2": 0.542150, + "Standard_D13_v2": 1.085740, + "Standard_D14_v2": 2.170560, + "Standard_D15_v2": 2.712710, + "Standard_D15i_v2": 2.712710, + "Standard_D16_v3": 1.373170, + "Standard_D16_v4": 1.315950, + "Standard_D16_v5": 1.314000, + "Standard_D16a_v4": 1.428000, + "Standard_D16ads_v5": 1.532000, + "Standard_D16as_v4": 1.428000, + "Standard_D16as_v5": 1.279000, + "Standard_D16as_v5_Promo": 1.145000, + "Standard_D16d_v4": 1.556020, + "Standard_D16d_v5": 1.557000, + "Standard_D16ds_v4": 1.556020, + "Standard_D16ds_v5": 1.557000, + "Standard_D16lds_v5": 1.428000, + "Standard_D16ls_v5": 1.264000, + "Standard_D16s_v3": 1.373170, + "Standard_D16s_v4": 1.315950, + "Standard_D16s_v5": 1.314000, + "Standard_D1_v2": 0.097020, + "Standard_D2_v2": 0.194570, + "Standard_D2_v3": 0.171420, + "Standard_D2_v4": 0.164270, + "Standard_D2_v5": 0.164000, + "Standard_D2a_v4": 0.178000, + "Standard_D2ads_v5": 0.191000, + "Standard_D2as_v4": 0.178000, + "Standard_D2as_v5": 0.252000, + "Standard_D2as_v5_Promo": 0.143000, + "Standard_D2d_v4": 0.193600, + "Standard_D2d_v5": 0.194000, + "Standard_D2ds_v4": 0.193600, + "Standard_D2ds_v5": 0.194000, + "Standard_D2lds_v5": 0.178000, + "Standard_D2ls_v5": 0.158000, + "Standard_D2s_v3": 0.171420, + "Standard_D2s_v4": 0.164270, + "Standard_D2s_v5": 0.164000, + "Standard_D32-16s_v3": 2.745420, + "Standard_D32-8s_v3": 2.745420, + "Standard_D32_v3": 2.745420, + "Standard_D32_v4": 2.631020, + "Standard_D32_v5": 2.628000, + "Standard_D32a_v4": 2.855000, + "Standard_D32ads_v5": 3.064000, + "Standard_D32as_v4": 2.855000, + "Standard_D32as_v5": 4.030000, + "Standard_D32as_v5_Promo": 2.290000, + "Standard_D32d_v4": 3.112030, + "Standard_D32d_v5": 3.114000, + "Standard_D32ds_v4": 3.112030, + "Standard_D32ds_v5": 3.114000, + "Standard_D32lds_v5": 2.855000, + "Standard_D32ls_v5": 2.528000, + "Standard_D32s_v3": 2.745420, + "Standard_D32s_v4": 2.631020, + "Standard_D32s_v5": 2.628000, + "Standard_D3_v2": 0.389140, + "Standard_D48_v3": 4.118580, + "Standard_D48_v4": 3.946980, + "Standard_D48_v5": 3.943000, + "Standard_D48a_v4": 4.283000, + "Standard_D48ads_v5": 6.803000, + "Standard_D48as_v4": 4.283000, + "Standard_D48as_v5": 6.045000, + "Standard_D48as_v5_Promo": 3.435000, + "Standard_D48d_v4": 4.667170, + "Standard_D48d_v5": 4.671000, + "Standard_D48ds_v4": 4.667170, + "Standard_D48ds_v5": 4.671000, + "Standard_D48lds_v5": 4.283000, + "Standard_D48ls_v5": 3.792000, + "Standard_D48s_v3": 4.118580, + "Standard_D48s_v4": 3.946980, + "Standard_D48s_v5": 3.943000, + "Standard_D4_v2": 0.778010, + "Standard_D4_v3": 0.342830, + "Standard_D4_v4": 0.328550, + "Standard_D4_v5": 0.329000, + "Standard_D4a_v4": 0.357000, + "Standard_D4ads_v5": 0.383000, + "Standard_D4as_v4": 0.357000, + "Standard_D4as_v5": 0.504000, + "Standard_D4as_v5_Promo": 0.286000, + "Standard_D4d_v4": 0.389220, + "Standard_D4d_v5": 0.389000, + "Standard_D4ds_v4": 0.389220, + "Standard_D4ds_v5": 0.389000, + "Standard_D4lds_v5": 0.357000, + "Standard_D4ls_v5": 0.316000, + "Standard_D4s_v3": 0.342830, + "Standard_D4s_v4": 0.328550, + "Standard_D4s_v5": 0.329000, + "Standard_D5_v2": 1.553680, + "Standard_D64-16s_v3": 5.490830, + "Standard_D64-32s_v3": 5.490830, + "Standard_D64_v3": 5.490830, + "Standard_D64_v4": 5.262050, + "Standard_D64_v5": 5.257000, + "Standard_D64a_v4": 5.711000, + "Standard_D64ads_v5": 6.127000, + "Standard_D64as_v4": 5.711000, + "Standard_D64as_v5": 8.060000, + "Standard_D64as_v5_Promo": 4.581000, + "Standard_D64d_v4": 6.223180, + "Standard_D64d_v5": 6.228000, + "Standard_D64ds_v4": 6.223180, + "Standard_D64ds_v5": 6.228000, + "Standard_D64lds_v5": 5.711000, + "Standard_D64ls_v5": 5.057000, + "Standard_D64s_v3": 5.490830, + "Standard_D64s_v4": 5.262050, + "Standard_D64s_v5": 5.257000, + "Standard_D8_v3": 0.686580, + "Standard_D8_v4": 0.657980, + "Standard_D8_v5": 0.657000, + "Standard_D8a_v4": 0.714000, + "Standard_D8ads_v5": 0.766000, + "Standard_D8as_v4": 0.714000, + "Standard_D8as_v5": 1.007000, + "Standard_D8as_v5_Promo": 0.573000, + "Standard_D8d_v4": 0.777570, + "Standard_D8d_v5": 0.778000, + "Standard_D8ds_v4": 0.777570, + "Standard_D8ds_v5": 0.778000, + "Standard_D8lds_v5": 0.714000, + "Standard_D8ls_v5": 0.632000, + "Standard_D8s_v3": 0.686580, + "Standard_D8s_v4": 0.657980, + "Standard_D8s_v5": 0.657000, + "Standard_D96_v5": 7.885000, + "Standard_D96a_v4": 8.566000, + "Standard_D96ads_v5": 13.607000, + "Standard_D96as_v4": 8.566000, + "Standard_D96as_v5": 12.090000, + "Standard_D96as_v5_Promo": 6.871000, + "Standard_D96d_v5": 9.341000, + "Standard_D96ds_v5": 9.341000, + "Standard_D96lds_v5": 8.566000, + "Standard_D96ls_v5": 7.585000, + "Standard_D96s_v5": 7.885000, + "Standard_DS11-1_v2": 0.271330, + "Standard_DS11_v2": 0.271330, + "Standard_DS12-1_v2": 0.542150, + "Standard_DS12-2_v2": 0.542150, + "Standard_DS12_v2": 0.542150, + "Standard_DS13-2_v2": 1.085740, + "Standard_DS13-4_v2": 1.085740, + "Standard_DS13_v2": 1.085740, + "Standard_DS14-4_v2": 2.170560, + "Standard_DS14-8_v2": 2.170560, + "Standard_DS14_v2": 2.170560, + "Standard_DS15_v2": 2.712710, + "Standard_DS15i_v2": 2.712710, + "Standard_DS1_v2": 0.097020, + "Standard_DS2_v2": 0.194570, + "Standard_DS3_v2": 0.389140, + "Standard_DS4_v2": 0.778010, + "Standard_DS5_v2": 1.553680, + "Standard_E104i_v5": 12.418000, + "Standard_E104id_v5": 14.128400, + "Standard_E104ids_v5": 14.414000, + "Standard_E104is_v5": 12.641000, + "Standard_E112iads_v5": 15.001000, + "Standard_E112ias_v5": 12.940000, + "Standard_E112ibds_v5": 19.124000, + "Standard_E112ibs_v5": 17.063000, + "Standard_E16-4ads_v5": 1.948000, + "Standard_E16-4as_v4": 1.874000, + "Standard_E16-4as_v5": 2.417000, + "Standard_E16-4ds_v4": 2.010480, + "Standard_E16-4ds_v5": 2.016000, + "Standard_E16-4s_v3": 1.830580, + "Standard_E16-4s_v4": 1.766660, + "Standard_E16-4s_v5": 1.768000, + "Standard_E16-8ads_v5": 1.948000, + "Standard_E16-8as_v4": 1.874000, + "Standard_E16-8as_v5": 1.681000, + "Standard_E16-8ds_v4": 2.010480, + "Standard_E16-8ds_v5": 2.016000, + "Standard_E16-8s_v3": 1.830580, + "Standard_E16-8s_v4": 1.766660, + "Standard_E16-8s_v5": 1.768000, + "Standard_E16_v3": 1.830580, + "Standard_E16_v4": 1.766660, + "Standard_E16_v5": 1.769000, + "Standard_E16a_v4": 1.874000, + "Standard_E16ads_v5": 1.994000, + "Standard_E16as_v4": 1.874000, + "Standard_E16as_v5": 1.681000, + "Standard_E16bds_v5": 2.484000, + "Standard_E16bs_v5": 2.216000, + "Standard_E16d_v4": 2.010480, + "Standard_E16d_v5": 2.016000, + "Standard_E16ds_v4": 2.010480, + "Standard_E16ds_v5": 2.016000, + "Standard_E16s_v3": 1.830580, + "Standard_E16s_v4": 1.766660, + "Standard_E16s_v5": 1.768000, + "Standard_E20_v3": 2.288000, + "Standard_E20_v4": 2.208100, + "Standard_E20_v5": 2.212000, + "Standard_E20a_v4": 2.342000, + "Standard_E20ads_v5": 2.481000, + "Standard_E20as_v4": 2.342000, + "Standard_E20as_v5": 2.101000, + "Standard_E20d_v4": 2.513550, + "Standard_E20d_v5": 2.520000, + "Standard_E20ds_v4": 2.513550, + "Standard_E20ds_v5": 2.520000, + "Standard_E20s_v3": 2.288000, + "Standard_E20s_v4": 2.208100, + "Standard_E20s_v5": 2.210000, + "Standard_E2_v3": 0.229170, + "Standard_E2_v4": 0.221160, + "Standard_E2_v5": 0.221000, + "Standard_E2a_v4": 0.234000, + "Standard_E2ads_v5": 0.244000, + "Standard_E2as_v4": 0.234000, + "Standard_E2as_v5": 0.256000, + "Standard_E2bds_v5": 0.310000, + "Standard_E2bs_v5": 0.277000, + "Standard_E2d_v4": 0.251090, + "Standard_E2d_v5": 0.252000, + "Standard_E2ds_v4": 0.251090, + "Standard_E2ds_v5": 0.252000, + "Standard_E2s_v3": 0.229170, + "Standard_E2s_v4": 0.221160, + "Standard_E2s_v5": 0.221000, + "Standard_E32-16ads_v5": 3.896000, + "Standard_E32-16as_v4": 3.748000, + "Standard_E32-16as_v5": 4.833000, + "Standard_E32-16ds_v4": 4.020970, + "Standard_E32-16ds_v5": 4.032000, + "Standard_E32-16s_v3": 3.661170, + "Standard_E32-16s_v4": 3.533320, + "Standard_E32-16s_v5": 3.536000, + "Standard_E32-8ads_v5": 5.368000, + "Standard_E32-8as_v4": 3.748000, + "Standard_E32-8as_v5": 3.361000, + "Standard_E32-8ds_v4": 4.020970, + "Standard_E32-8ds_v5": 4.032000, + "Standard_E32-8s_v3": 3.661170, + "Standard_E32-8s_v4": 3.533320, + "Standard_E32-8s_v5": 3.536000, + "Standard_E32_v3": 3.661170, + "Standard_E32_v4": 3.533320, + "Standard_E32_v5": 3.539000, + "Standard_E32a_v4": 3.748000, + "Standard_E32ads_v5": 3.942000, + "Standard_E32as_v4": 3.748000, + "Standard_E32as_v5": 3.407000, + "Standard_E32bds_v5": 4.967000, + "Standard_E32bs_v5": 4.432000, + "Standard_E32d_v4": 4.020970, + "Standard_E32d_v5": 4.032000, + "Standard_E32ds_v4": 4.020970, + "Standard_E32ds_v5": 4.032000, + "Standard_E32s_v3": 3.661170, + "Standard_E32s_v4": 3.533320, + "Standard_E32s_v5": 3.536000, + "Standard_E4-2ads_v5": 0.671000, + "Standard_E4-2as_v4": 0.468000, + "Standard_E4-2as_v5": 0.604000, + "Standard_E4-2ds_v4": 0.503060, + "Standard_E4-2ds_v5": 0.504000, + "Standard_E4-2s_v3": 0.457420, + "Standard_E4-2s_v4": 0.441440, + "Standard_E4-2s_v5": 0.442000, + "Standard_E48_v3": 5.215730, + "Standard_E48_v4": 5.299970, + "Standard_E48_v5": 5.308000, + "Standard_E48a_v4": 5.622000, + "Standard_E48ads_v5": 5.891000, + "Standard_E48as_v4": 5.622000, + "Standard_E48as_v5": 5.042000, + "Standard_E48bds_v5": 7.451000, + "Standard_E48bs_v5": 6.648000, + "Standard_E48d_v4": 6.031450, + "Standard_E48d_v5": 6.047000, + "Standard_E48ds_v4": 6.031450, + "Standard_E48ds_v5": 6.048000, + "Standard_E48s_v3": 5.215730, + "Standard_E48s_v4": 5.299970, + "Standard_E48s_v5": 5.304000, + "Standard_E4_v3": 0.457420, + "Standard_E4_v4": 0.441440, + "Standard_E4_v5": 0.442000, + "Standard_E4a_v4": 0.468000, + "Standard_E4ads_v5": 0.533000, + "Standard_E4as_v4": 0.468000, + "Standard_E4as_v5": 0.466000, + "Standard_E4bds_v5": 0.621000, + "Standard_E4bs_v5": 0.554000, + "Standard_E4d_v4": 0.503060, + "Standard_E4d_v5": 0.504000, + "Standard_E4ds_v4": 0.503060, + "Standard_E4ds_v5": 0.504000, + "Standard_E4s_v3": 0.457420, + "Standard_E4s_v4": 0.441440, + "Standard_E4s_v5": 0.442000, + "Standard_E64-16ads_v5": 10.737000, + "Standard_E64-16as_v4": 7.495000, + "Standard_E64-16as_v5": 9.666000, + "Standard_E64-16ds_v4": 8.041940, + "Standard_E64-16ds_v5": 8.064000, + "Standard_E64-16s_v3": 6.256810, + "Standard_E64-16s_v4": 7.065750, + "Standard_E64-16s_v5": 7.072000, + "Standard_E64-32ads_v5": 10.737000, + "Standard_E64-32as_v4": 7.495000, + "Standard_E64-32as_v5": 9.666000, + "Standard_E64-32ds_v4": 8.041940, + "Standard_E64-32ds_v5": 8.064000, + "Standard_E64-32s_v3": 6.256810, + "Standard_E64-32s_v4": 7.065750, + "Standard_E64-32s_v5": 7.072000, + "Standard_E64_v3": 6.256810, + "Standard_E64_v4": 7.065750, + "Standard_E64_v5": 7.077000, + "Standard_E64a_v4": 7.495000, + "Standard_E64ads_v5": 7.839000, + "Standard_E64as_v4": 7.495000, + "Standard_E64as_v5": 6.768000, + "Standard_E64bds_v5": 9.935000, + "Standard_E64bs_v5": 8.864000, + "Standard_E64d_v4": 8.041940, + "Standard_E64d_v5": 8.063000, + "Standard_E64ds_v4": 8.041940, + "Standard_E64ds_v5": 8.064000, + "Standard_E64i_v3": 6.256810, + "Standard_E64is_v3": 6.256810, + "Standard_E64s_v3": 6.256810, + "Standard_E64s_v4": 7.065750, + "Standard_E64s_v5": 7.072000, + "Standard_E8-2ads_v5": 0.974000, + "Standard_E8-2as_v4": 0.937000, + "Standard_E8-2as_v5": 1.208000, + "Standard_E8-2ds_v4": 1.005240, + "Standard_E8-2ds_v5": 1.008000, + "Standard_E8-2s_v3": 0.914830, + "Standard_E8-2s_v4": 0.882890, + "Standard_E8-2s_v5": 0.884000, + "Standard_E8-4ads_v5": 1.342000, + "Standard_E8-4as_v4": 0.937000, + "Standard_E8-4as_v5": 1.208000, + "Standard_E8-4ds_v4": 1.005240, + "Standard_E8-4ds_v5": 1.008000, + "Standard_E8-4s_v3": 0.914830, + "Standard_E8-4s_v4": 0.882890, + "Standard_E8-4s_v5": 0.884000, + "Standard_E80ids_v4": 10.053300, + "Standard_E80is_v4": 8.832410, + "Standard_E8_v3": 0.914830, + "Standard_E8_v4": 0.882890, + "Standard_E8_v5": 0.885000, + "Standard_E8a_v4": 0.937000, + "Standard_E8ads_v5": 1.020000, + "Standard_E8as_v4": 0.937000, + "Standard_E8as_v5": 0.840000, + "Standard_E8bds_v5": 1.242000, + "Standard_E8bs_v5": 1.108000, + "Standard_E8d_v4": 1.005240, + "Standard_E8d_v5": 1.008000, + "Standard_E8ds_v4": 1.005240, + "Standard_E8ds_v5": 1.008000, + "Standard_E8s_v3": 0.914830, + "Standard_E8s_v4": 0.882890, + "Standard_E8s_v5": 0.884000, + "Standard_E96-24ads_v5": 11.689000, + "Standard_E96-24as_v4": 11.243000, + "Standard_E96-24as_v5": 10.083000, + "Standard_E96-24ds_v5": 12.096000, + "Standard_E96-24s_v5": 10.608000, + "Standard_E96-48ads_v5": 16.105000, + "Standard_E96-48as_v4": 11.243000, + "Standard_E96-48as_v5": 14.499000, + "Standard_E96-48ds_v5": 12.096000, + "Standard_E96-48s_v5": 10.608000, + "Standard_E96_v5": 10.616000, + "Standard_E96a_v4": 11.243000, + "Standard_E96ads_v5": 11.735000, + "Standard_E96as_v4": 11.243000, + "Standard_E96as_v5": 10.129000, + "Standard_E96bds_v5": 14.902000, + "Standard_E96bs_v5": 13.296000, + "Standard_E96d_v5": 12.095000, + "Standard_E96ds_v5": 12.096000, + "Standard_E96iads_v5": 12.858000, + "Standard_E96ias_v4": 11.243000, + "Standard_E96ias_v5": 11.092000, + "Standard_E96s_v5": 10.608000, + "Standard_F1": 0.081170, + "Standard_F16": 1.300600, + "Standard_F16s": 1.300600, + "Standard_F16s_v2": 1.110320, + "Standard_F1s": 0.081170, + "Standard_F2": 0.162930, + "Standard_F2s": 0.162930, + "Standard_F2s_v2": 0.138890, + "Standard_F32s_v2": 2.219730, + "Standard_F4": 0.324430, + "Standard_F48s_v2": 3.330050, + "Standard_F4s": 0.324430, + "Standard_F4s_v2": 0.277510, + "Standard_F64s_v2": 4.438260, + "Standard_F72s_v2": 4.994480, + "Standard_F8": 0.649130, + "Standard_F8s": 0.649130, + "Standard_F8s_v2": 0.554110, + "Standard_L16as_v3": 2.127000, + "Standard_L16s_v3": 2.595000, + "Standard_L32as_v3": 4.254000, + "Standard_L32s_v3": 5.190000, + "Standard_L48as_v3": 6.380000, + "Standard_L48s_v3": 7.786000, + "Standard_L64as_v3": 8.507000, + "Standard_L64s_v3": 10.381000, + "Standard_L80as_v3": 10.634000, + "Standard_L80s_v3": 12.976000, + "Standard_L8as_v3": 1.063000, + "Standard_L8s_v3": 1.298000, + "Standard_M128": 26.702020, + "Standard_M128-32ms": 53.429560, + "Standard_M128-64ms": 53.429560, + "Standard_M128m": 53.429560, + "Standard_M128ms": 53.429560, + "Standard_M128s": 26.702020, + "Standard_M16-4ms": 6.152240, + "Standard_M16-8ms": 6.152240, + "Standard_M16ms": 6.152240, + "Standard_M16s": 4.778770, + "Standard_M208-104ms_v2": 90.579000, + "Standard_M208-104s_v2": 45.289000, + "Standard_M208-52ms_v2": 90.579000, + "Standard_M208-52s_v2": 45.289000, + "Standard_M208ms_v2": 90.579000, + "Standard_M208s_v2": 45.289000, + "Standard_M32-16ms": 12.304750, + "Standard_M32-8ms": 12.304750, + "Standard_M32ls": 4.929980, + "Standard_M32ms": 12.304750, + "Standard_M32s": 6.676760, + "Standard_M32ts": 4.644270, + "Standard_M416-104ms_v2": 201.270000, + "Standard_M416-104s_v2": 100.650000, + "Standard_M416-208ms_v2": 201.270000, + "Standard_M416-208s_v2": 100.650000, + "Standard_M416is_v2": 100.650000, + "Standard_M416ms_v2": 201.270000, + "Standard_M416s_8_v2": 122.900000, + "Standard_M416s_v2": 100.650000, + "Standard_M64": 13.351730, + "Standard_M64-16ms": 20.695140, + "Standard_M64-32ms": 20.695140, + "Standard_M64ls": 9.291770, + "Standard_M64m": 20.695140, + "Standard_M64ms": 20.695140, + "Standard_M64s": 13.351730, + "Standard_M8-2ms": 3.075800, + "Standard_M8-4ms": 3.075800, + "Standard_M8ms": 3.075800, + "Standard_ND96ams_A100_v4": 60.919000, + "Standard_ND96amsr_A100_v4": 60.919000, + } + // uaecentral + initialOnDemandPrices["uaecentral"] = map[string]float64{ + "DCdsv3 Type1": 5.118000, + "DCsv3 Type1": 4.350000, + "Ddsv4_Type 1": 6.464000, + "Ddsv4_Type2": 7.676000, + "Ddsv5_Type1": 9.514000, + "Dsv3_Type3": 6.590000, + "Dsv3_Type4": 8.238000, + "Dsv4_Type1": 6.864000, + "Dsv4_Type2": 8.580000, + "Dsv5_Type1": 8.416000, + "Ebdsv5-Type1": 9.427000, + "Ebsv5-Type1": 8.466000, + "Edsv4_Type 1": 7.907000, + "Edsv4_Type2": 9.390000, + "Edsv5_Type1": 12.081000, + "Esv3_Type3": 6.703000, + "Esv3_Type4": 9.082000, + "Esv4_Type1": 6.919000, + "Esv4_Type2": 9.081000, + "Esv5_Type1": 10.646000, + "Fsv2_Type2": 5.251000, + "Lasv3_Type1": 10.811000, + "Lsv3_Type1": 12.069000, + "Mdmsv2MedMem _Type1": 45.800000, + "Mdsv2MedMem_Type1": 22.892000, + "Ms_Type1": 22.883000, + "Msm_Type1": 45.807000, + "Msmv2MedMem Type1": 45.078000, + "Msmv2_Type1": 170.140000, + "Msv2MedMem Type1": 22.169000, + "Msv2_Type1": 85.079000, + "Standard_A1_v2": 0.053300, + "Standard_A2_v2": 0.113000, + "Standard_A2m_v2": 0.154000, + "Standard_A4_v2": 0.238000, + "Standard_A4m_v2": 0.324000, + "Standard_A8_v2": 0.498000, + "Standard_A8m_v2": 0.682000, + "Standard_B12ms": 0.749000, + "Standard_B16als_v2": 0.905000, + "Standard_B16as_v2": 1.012000, + "Standard_B16ls_v2": 0.997000, + "Standard_B16ms": 0.998000, + "Standard_B16s_v2": 1.043000, + "Standard_B1ls": 0.007800, + "Standard_B1ms": 0.031200, + "Standard_B1s": 0.015600, + "Standard_B20ms": 1.248000, + "Standard_B2als_v2": 0.067900, + "Standard_B2as_v2": 0.127000, + "Standard_B2ats_v2": 0.023900, + "Standard_B2ls_v2": 0.065200, + "Standard_B2ms": 0.125000, + "Standard_B2s": 0.062400, + "Standard_B2s_v2": 0.130000, + "Standard_B2ts_v2": 0.025500, + "Standard_B32als_v2": 1.663000, + "Standard_B32as_v2": 2.025000, + "Standard_B32ls_v2": 1.995000, + "Standard_B32s_v2": 2.233000, + "Standard_B4als_v2": 0.226000, + "Standard_B4as_v2": 0.235000, + "Standard_B4ls_v2": 0.249000, + "Standard_B4ms": 0.250000, + "Standard_B4s_v2": 0.279000, + "Standard_B8als_v2": 0.453000, + "Standard_B8as_v2": 0.469000, + "Standard_B8ls_v2": 0.462000, + "Standard_B8ms": 0.499000, + "Standard_B8s_v2": 0.558000, + "Standard_D1": 0.105000, + "Standard_D11": 0.271000, + "Standard_D11_v2": 0.233000, + "Standard_D12": 0.541000, + "Standard_D12_v2": 0.466000, + "Standard_D13": 1.082000, + "Standard_D13_v2": 0.933000, + "Standard_D14": 2.164000, + "Standard_D14_v2": 1.866000, + "Standard_D15_v2": 2.332000, + "Standard_D15i_v2": 2.332000, + "Standard_D16_v3": 1.248000, + "Standard_D16_v4": 1.248000, + "Standard_D16_v5": 1.224000, + "Standard_D16ads_v5": 2.056000, + "Standard_D16as_v5": 1.838000, + "Standard_D16d_v4": 1.469000, + "Standard_D16d_v5": 1.441000, + "Standard_D16ds_v4": 1.469000, + "Standard_D16ds_v5": 1.441000, + "Standard_D16lds_v5": 1.248000, + "Standard_D16ls_v5": 1.098000, + "Standard_D16s_v3": 1.248000, + "Standard_D16s_v4": 1.248000, + "Standard_D16s_v5": 1.224000, + "Standard_D1_v2": 0.089300, + "Standard_D2": 0.209000, + "Standard_D2_v2": 0.178000, + "Standard_D2_v3": 0.156000, + "Standard_D2_v4": 0.156000, + "Standard_D2_v5": 0.153000, + "Standard_D2ads_v5": 0.257000, + "Standard_D2as_v5": 0.230000, + "Standard_D2d_v4": 0.184000, + "Standard_D2d_v5": 0.180000, + "Standard_D2ds_v4": 0.184000, + "Standard_D2ds_v5": 0.180000, + "Standard_D2lds_v5": 0.156000, + "Standard_D2ls_v5": 0.137000, + "Standard_D2s_v3": 0.156000, + "Standard_D2s_v4": 0.156000, + "Standard_D2s_v5": 0.153000, + "Standard_D3": 0.418000, + "Standard_D32-16s_v3": 2.496000, + "Standard_D32-8s_v3": 2.496000, + "Standard_D32_v3": 2.496000, + "Standard_D32_v4": 2.496000, + "Standard_D32_v5": 2.448000, + "Standard_D32ads_v5": 4.112000, + "Standard_D32as_v5": 3.677000, + "Standard_D32d_v4": 2.938000, + "Standard_D32d_v5": 2.883000, + "Standard_D32ds_v4": 2.938000, + "Standard_D32ds_v5": 2.883000, + "Standard_D32lds_v5": 2.496000, + "Standard_D32ls_v5": 2.196000, + "Standard_D32s_v3": 2.496000, + "Standard_D32s_v4": 2.496000, + "Standard_D32s_v5": 2.448000, + "Standard_D3_v2": 0.357000, + "Standard_D4": 0.836000, + "Standard_D48_v3": 3.744000, + "Standard_D48_v4": 3.744000, + "Standard_D48_v5": 3.672000, + "Standard_D48ads_v5": 6.167000, + "Standard_D48as_v5": 5.515000, + "Standard_D48d_v4": 4.407000, + "Standard_D48d_v5": 4.324000, + "Standard_D48ds_v4": 4.407000, + "Standard_D48ds_v5": 4.324000, + "Standard_D48lds_v5": 3.744000, + "Standard_D48ls_v5": 3.295000, + "Standard_D48s_v3": 3.744000, + "Standard_D48s_v4": 3.744000, + "Standard_D48s_v5": 3.672000, + "Standard_D4_v2": 0.714000, + "Standard_D4_v3": 0.312000, + "Standard_D4_v4": 0.312000, + "Standard_D4_v5": 0.306000, + "Standard_D4ads_v5": 0.514000, + "Standard_D4as_v5": 0.460000, + "Standard_D4d_v4": 0.367000, + "Standard_D4d_v5": 0.360000, + "Standard_D4ds_v4": 0.367000, + "Standard_D4ds_v5": 0.360000, + "Standard_D4lds_v5": 0.312000, + "Standard_D4ls_v5": 0.275000, + "Standard_D4s_v3": 0.312000, + "Standard_D4s_v4": 0.312000, + "Standard_D4s_v5": 0.306000, + "Standard_D5_v2": 1.429000, + "Standard_D64-16s_v3": 4.992000, + "Standard_D64-32s_v3": 4.992000, + "Standard_D64_v3": 4.992000, + "Standard_D64_v4": 4.992000, + "Standard_D64_v5": 4.896000, + "Standard_D64ads_v5": 8.223000, + "Standard_D64as_v5": 7.354000, + "Standard_D64d_v4": 5.876000, + "Standard_D64d_v5": 5.766000, + "Standard_D64ds_v4": 5.876000, + "Standard_D64ds_v5": 5.766000, + "Standard_D64lds_v5": 4.992000, + "Standard_D64ls_v5": 4.393000, + "Standard_D64s_v3": 4.992000, + "Standard_D64s_v4": 4.992000, + "Standard_D64s_v5": 4.896000, + "Standard_D8_v3": 0.624000, + "Standard_D8_v4": 0.624000, + "Standard_D8_v5": 0.612000, + "Standard_D8ads_v5": 1.028000, + "Standard_D8as_v5": 0.919000, + "Standard_D8d_v4": 0.734000, + "Standard_D8d_v5": 0.721000, + "Standard_D8ds_v4": 0.734000, + "Standard_D8ds_v5": 0.721000, + "Standard_D8lds_v5": 0.624000, + "Standard_D8ls_v5": 0.549000, + "Standard_D8s_v3": 0.624000, + "Standard_D8s_v4": 0.624000, + "Standard_D8s_v5": 0.612000, + "Standard_D96_v5": 7.344000, + "Standard_D96ads_v5": 12.335000, + "Standard_D96as_v5": 11.030000, + "Standard_D96d_v5": 8.649000, + "Standard_D96ds_v5": 8.649000, + "Standard_D96lds_v5": 7.488000, + "Standard_D96ls_v5": 6.589000, + "Standard_D96s_v5": 7.344000, + "Standard_DC16ds_v3": 1.551000, + "Standard_DC16s_v3": 1.318000, + "Standard_DC1ds_v3": 0.096900, + "Standard_DC1s_v3": 0.082400, + "Standard_DC24ds_v3": 2.326000, + "Standard_DC24s_v3": 1.977000, + "Standard_DC2ds_v3": 0.194000, + "Standard_DC2s_v3": 0.165000, + "Standard_DC32ds_v3": 3.102000, + "Standard_DC32s_v3": 2.636000, + "Standard_DC48ds_v3": 4.653000, + "Standard_DC48s_v3": 3.954000, + "Standard_DC4ds_v3": 0.388000, + "Standard_DC4s_v3": 0.330000, + "Standard_DC8ds_v3": 0.775000, + "Standard_DC8s_v3": 0.659000, + "Standard_DS1": 0.105000, + "Standard_DS11": 0.271000, + "Standard_DS11-1_v2": 0.233000, + "Standard_DS11_v2": 0.233000, + "Standard_DS12": 0.541000, + "Standard_DS12-1_v2": 0.466000, + "Standard_DS12-2_v2": 0.466000, + "Standard_DS12_v2": 0.466000, + "Standard_DS13": 1.082000, + "Standard_DS13-2_v2": 0.933000, + "Standard_DS13-4_v2": 0.933000, + "Standard_DS13_v2": 0.933000, + "Standard_DS14": 2.164000, + "Standard_DS14-4_v2": 1.866000, + "Standard_DS14-8_v2": 1.866000, + "Standard_DS14_v2": 1.866000, + "Standard_DS15_v2": 2.332000, + "Standard_DS15i_v2": 2.332000, + "Standard_DS1_v2": 0.089300, + "Standard_DS2": 0.209000, + "Standard_DS2_v2": 0.178000, + "Standard_DS3": 0.418000, + "Standard_DS3_v2": 0.357000, + "Standard_DS4": 0.836000, + "Standard_DS4_v2": 0.714000, + "Standard_DS5_v2": 1.429000, + "Standard_E104i_v5": 11.533000, + "Standard_E104id_v5": 13.087000, + "Standard_E104ids_v5": 13.087000, + "Standard_E104is_v5": 11.533000, + "Standard_E112ibds_v5": 16.496000, + "Standard_E112ibs_v5": 14.815000, + "Standard_E16-4ds_v4": 1.797000, + "Standard_E16-4ds_v5": 1.830000, + "Standard_E16-4s_v3": 1.572000, + "Standard_E16-4s_v4": 1.572000, + "Standard_E16-4s_v5": 1.613000, + "Standard_E16-8ds_v4": 1.797000, + "Standard_E16-8ds_v5": 1.830000, + "Standard_E16-8s_v3": 1.572000, + "Standard_E16-8s_v4": 1.572000, + "Standard_E16-8s_v5": 1.613000, + "Standard_E16_v3": 1.572000, + "Standard_E16_v4": 1.572000, + "Standard_E16_v5": 1.613000, + "Standard_E16bds_v5": 2.142000, + "Standard_E16bs_v5": 1.924000, + "Standard_E16d_v4": 1.797000, + "Standard_E16d_v5": 1.830000, + "Standard_E16ds_v4": 1.797000, + "Standard_E16ds_v5": 1.830000, + "Standard_E16s_v3": 1.572000, + "Standard_E16s_v4": 1.572000, + "Standard_E16s_v5": 1.613000, + "Standard_E20_v3": 2.080000, + "Standard_E20_v4": 1.966000, + "Standard_E20_v5": 2.016000, + "Standard_E20d_v4": 2.246000, + "Standard_E20d_v5": 2.288000, + "Standard_E20ds_v4": 2.246000, + "Standard_E20ds_v5": 2.288000, + "Standard_E20s_v3": 2.080000, + "Standard_E20s_v4": 1.966000, + "Standard_E20s_v5": 2.016000, + "Standard_E2_v3": 0.196000, + "Standard_E2_v4": 0.197000, + "Standard_E2_v5": 0.202000, + "Standard_E2bds_v5": 0.268000, + "Standard_E2bs_v5": 0.241000, + "Standard_E2d_v4": 0.225000, + "Standard_E2d_v5": 0.229000, + "Standard_E2ds_v4": 0.225000, + "Standard_E2ds_v5": 0.229000, + "Standard_E2s_v3": 0.196000, + "Standard_E2s_v4": 0.197000, + "Standard_E2s_v5": 0.202000, + "Standard_E32-16ds_v4": 3.594000, + "Standard_E32-16ds_v5": 3.661000, + "Standard_E32-16s_v3": 3.145000, + "Standard_E32-16s_v4": 3.145000, + "Standard_E32-16s_v5": 3.226000, + "Standard_E32-8ds_v4": 3.594000, + "Standard_E32-8ds_v5": 3.661000, + "Standard_E32-8s_v3": 3.145000, + "Standard_E32-8s_v4": 3.145000, + "Standard_E32-8s_v5": 3.226000, + "Standard_E32_v3": 3.145000, + "Standard_E32_v4": 3.145000, + "Standard_E32_v5": 3.226000, + "Standard_E32bds_v5": 4.285000, + "Standard_E32bs_v5": 3.848000, + "Standard_E32d_v4": 3.594000, + "Standard_E32d_v5": 3.661000, + "Standard_E32ds_v4": 3.594000, + "Standard_E32ds_v5": 3.661000, + "Standard_E32s_v3": 3.145000, + "Standard_E32s_v4": 3.145000, + "Standard_E32s_v5": 3.226000, + "Standard_E4-2ds_v4": 0.449000, + "Standard_E4-2ds_v5": 0.458000, + "Standard_E4-2s_v3": 0.393000, + "Standard_E4-2s_v4": 0.393000, + "Standard_E4-2s_v5": 0.403000, + "Standard_E48_v3": 4.717000, + "Standard_E48_v4": 4.717000, + "Standard_E48_v5": 4.839000, + "Standard_E48bds_v5": 6.427000, + "Standard_E48bs_v5": 5.772000, + "Standard_E48d_v4": 5.391000, + "Standard_E48d_v5": 5.491000, + "Standard_E48ds_v4": 5.391000, + "Standard_E48ds_v5": 5.491000, + "Standard_E48s_v3": 4.717000, + "Standard_E48s_v4": 4.717000, + "Standard_E48s_v5": 4.839000, + "Standard_E4_v3": 0.393000, + "Standard_E4_v4": 0.393000, + "Standard_E4_v5": 0.403000, + "Standard_E4bds_v5": 0.536000, + "Standard_E4bs_v5": 0.481000, + "Standard_E4d_v4": 0.449000, + "Standard_E4d_v5": 0.458000, + "Standard_E4ds_v4": 0.449000, + "Standard_E4ds_v5": 0.458000, + "Standard_E4s_v3": 0.393000, + "Standard_E4s_v4": 0.393000, + "Standard_E4s_v5": 0.403000, + "Standard_E64-16ds_v4": 7.188000, + "Standard_E64-16ds_v5": 7.322000, + "Standard_E64-16s_v3": 5.661000, + "Standard_E64-16s_v4": 6.290000, + "Standard_E64-16s_v5": 6.452000, + "Standard_E64-32ds_v4": 7.188000, + "Standard_E64-32ds_v5": 7.322000, + "Standard_E64-32s_v3": 5.661000, + "Standard_E64-32s_v4": 6.290000, + "Standard_E64-32s_v5": 6.452000, + "Standard_E64_v3": 5.661000, + "Standard_E64_v4": 6.290000, + "Standard_E64_v5": 6.452000, + "Standard_E64bds_v5": 8.570000, + "Standard_E64bs_v5": 7.696000, + "Standard_E64d_v4": 7.188000, + "Standard_E64d_v5": 7.322000, + "Standard_E64ds_v4": 7.188000, + "Standard_E64ds_v5": 7.322000, + "Standard_E64i_v3": 5.689000, + "Standard_E64is_v3": 5.689000, + "Standard_E64s_v3": 5.661000, + "Standard_E64s_v4": 6.290000, + "Standard_E64s_v5": 6.452000, + "Standard_E8-2ds_v4": 0.899000, + "Standard_E8-2ds_v5": 0.915000, + "Standard_E8-2s_v3": 0.786000, + "Standard_E8-2s_v4": 0.786000, + "Standard_E8-2s_v5": 0.807000, + "Standard_E8-4ds_v4": 0.899000, + "Standard_E8-4ds_v5": 0.915000, + "Standard_E8-4s_v3": 0.786000, + "Standard_E8-4s_v4": 0.786000, + "Standard_E8-4s_v5": 0.807000, + "Standard_E80ids_v4": 8.986000, + "Standard_E80is_v4": 7.862000, + "Standard_E8_v3": 0.786000, + "Standard_E8_v4": 0.786000, + "Standard_E8_v5": 0.807000, + "Standard_E8bds_v5": 1.071000, + "Standard_E8bs_v5": 0.962000, + "Standard_E8d_v4": 0.899000, + "Standard_E8d_v5": 0.915000, + "Standard_E8ds_v4": 0.899000, + "Standard_E8ds_v5": 0.915000, + "Standard_E8s_v3": 0.786000, + "Standard_E8s_v4": 0.786000, + "Standard_E8s_v5": 0.807000, + "Standard_E96-24ds_v5": 10.982000, + "Standard_E96-24s_v5": 9.678000, + "Standard_E96-48ds_v5": 10.982000, + "Standard_E96-48s_v5": 9.678000, + "Standard_E96_v5": 9.678000, + "Standard_E96bds_v5": 12.854000, + "Standard_E96bs_v5": 11.544000, + "Standard_E96d_v5": 10.982000, + "Standard_E96ds_v5": 10.982000, + "Standard_E96s_v5": 9.678000, + "Standard_F1": 0.073800, + "Standard_F16": 1.182000, + "Standard_F16s": 1.182000, + "Standard_F16s_v2": 1.009000, + "Standard_F1s": 0.073800, + "Standard_F2": 0.148000, + "Standard_F2s": 0.148000, + "Standard_F2s_v2": 0.126000, + "Standard_F32s_v2": 2.018000, + "Standard_F4": 0.295000, + "Standard_F48s_v2": 3.026000, + "Standard_F4s": 0.295000, + "Standard_F4s_v2": 0.252000, + "Standard_F64s_v2": 4.035000, + "Standard_F72s_v2": 4.540000, + "Standard_F8": 0.590000, + "Standard_F8s": 0.590000, + "Standard_F8s_v2": 0.504000, + "Standard_L16as_v3": 1.966000, + "Standard_L16s_v3": 2.194000, + "Standard_L32as_v3": 3.931000, + "Standard_L32s_v3": 4.389000, + "Standard_L48as_v3": 5.897000, + "Standard_L48s_v3": 6.583000, + "Standard_L64as_v3": 7.862000, + "Standard_L64s_v3": 8.778000, + "Standard_L80as_v3": 9.828000, + "Standard_L80s_v3": 10.972000, + "Standard_L8as_v3": 0.983000, + "Standard_L8s_v3": 1.097000, + "Standard_M128": 20.807000, + "Standard_M128-32ms": 41.633000, + "Standard_M128-64ms": 41.633000, + "Standard_M128dms_v2": 41.636000, + "Standard_M128ds_v2": 20.810000, + "Standard_M128m": 41.633000, + "Standard_M128ms": 41.633000, + "Standard_M128ms_v2": 40.980000, + "Standard_M128s": 20.807000, + "Standard_M128s_v2": 20.154000, + "Standard_M16-4ms": 4.794000, + "Standard_M16-8ms": 4.794000, + "Standard_M16ms": 4.794000, + "Standard_M16s": 3.724000, + "Standard_M192idms_v2": 50.020000, + "Standard_M192ids_v2": 25.010000, + "Standard_M192ims_v2": 49.363000, + "Standard_M192is_v2": 24.353000, + "Standard_M208-104ms_v2": 69.607000, + "Standard_M208-104s_v2": 34.804000, + "Standard_M208-52ms_v2": 69.607000, + "Standard_M208-52s_v2": 34.804000, + "Standard_M208ms_v2": 69.607000, + "Standard_M208s_v2": 34.804000, + "Standard_M32-16ms": 9.588000, + "Standard_M32-8ms": 9.588000, + "Standard_M32dms_v2": 9.588000, + "Standard_M32ls": 4.481880, + "Standard_M32ms": 9.588000, + "Standard_M32ms_v2": 9.424000, + "Standard_M32s": 5.202565, + "Standard_M32ts": 4.222920, + "Standard_M416-104ms_v2": 154.670000, + "Standard_M416-104s_v2": 77.345000, + "Standard_M416-208ms_v2": 154.670000, + "Standard_M416-208s_v2": 77.345000, + "Standard_M416is_v2": 77.345000, + "Standard_M416ms_v2": 154.670000, + "Standard_M416s_8_v2": 122.900000, + "Standard_M416s_v2": 77.345000, + "Standard_M64": 10.404000, + "Standard_M64-16ms": 16.125612, + "Standard_M64-32ms": 16.125612, + "Standard_M64dms_v2": 16.130000, + "Standard_M64ds_v2": 10.404000, + "Standard_M64ls": 8.447400, + "Standard_M64m": 16.125612, + "Standard_M64ms": 16.125612, + "Standard_M64ms_v2": 15.802000, + "Standard_M64s": 10.404000, + "Standard_M64s_v2": 10.075000, + "Standard_M8-2ms": 2.397000, + "Standard_M8-4ms": 2.397000, + "Standard_M8ms": 2.397000, + "Standard_ND96ams_A100_v4": 60.919000, + "Standard_ND96amsr_A100_v4": 60.919000, + } + // uaenorth + initialOnDemandPrices["uaenorth"] = map[string]float64{ + "DCdsv3 Type1": 3.937000, + "DCsv3 Type1": 3.346000, + "Dadsv5_Type1": 7.817000, + "Dasv4_Type1": 6.474000, + "Dasv4_Type2": 6.474000, + "Dasv5_Type1": 6.530000, + "Ddsv4_Type 1": 4.875000, + "Ddsv4_Type2": 5.789000, + "Ddsv5_Type1": 7.318000, + "Dsv3_Type3": 5.069000, + "Dsv3_Type4": 6.336000, + "Dsv4_Type1": 5.170000, + "Dsv4_Type2": 6.217000, + "Dsv5_Type1": 6.474000, + "Eadsv5_Type1": 8.496000, + "Easv4_Type1": 8.189000, + "Easv4_Type2": 8.189000, + "Easv5_Type1": 7.392000, + "Ebdsv5-Type1": 7.251000, + "Ebsv5-Type1": 6.512000, + "Edsv4_Type 1": 6.195000, + "Edsv4_Type2": 7.942000, + "Edsv5_Type1": 9.293000, + "Esv3_Type3": 5.156000, + "Esv3_Type4": 6.986000, + "Esv4_Type1": 5.456000, + "Esv4_Type2": 7.161000, + "Esv5_Type1": 8.189000, + "Fsv2 Type3": 4.874000, + "Fsv2_Type2": 4.039000, + "Fsv2_Type4": 5.386000, + "Lasv3_Type1": 8.316000, + "Lsv3_Type1": 9.284000, + "Mdmsv2MedMem _Type1": 38.754000, + "Mdsv2MedMem_Type1": 19.370000, + "Ms_Type1": 19.363000, + "Msm_Type1": 38.759000, + "Msmv2MedMem Type1": 38.143000, + "Msmv2_Type1": 143.970000, + "Msv2MedMem Type1": 18.759000, + "Msv2_Type1": 71.990000, + "NVsv3_Type1": 6.120000, + "Standard_A1_v2": 0.043200, + "Standard_A2_v2": 0.091200, + "Standard_A2m_v2": 0.119000, + "Standard_A4_v2": 0.191000, + "Standard_A4m_v2": 0.250000, + "Standard_A8_v2": 0.400000, + "Standard_A8m_v2": 0.524000, + "Standard_B12ms": 0.599000, + "Standard_B16als_v2": 0.713000, + "Standard_B16as_v2": 0.796000, + "Standard_B16ls_v2": 0.784000, + "Standard_B16ms": 0.799000, + "Standard_B16s_v2": 0.802000, + "Standard_B1ls": 0.006240, + "Standard_B1ms": 0.025000, + "Standard_B1s": 0.012500, + "Standard_B20ms": 0.998000, + "Standard_B2als_v2": 0.054300, + "Standard_B2as_v2": 0.099500, + "Standard_B2ats_v2": 0.011300, + "Standard_B2ls_v2": 0.050200, + "Standard_B2ms": 0.099800, + "Standard_B2s": 0.049900, + "Standard_B2s_v2": 0.110000, + "Standard_B2ts_v2": 0.012500, + "Standard_B32als_v2": 1.279000, + "Standard_B32as_v2": 1.444000, + "Standard_B32ls_v2": 1.421000, + "Standard_B32s_v2": 1.752000, + "Standard_B4als_v2": 0.160000, + "Standard_B4as_v2": 0.199000, + "Standard_B4ls_v2": 0.196000, + "Standard_B4ms": 0.200000, + "Standard_B4s_v2": 0.219000, + "Standard_B8als_v2": 0.320000, + "Standard_B8as_v2": 0.398000, + "Standard_B8ls_v2": 0.355000, + "Standard_B8ms": 0.399000, + "Standard_B8s_v2": 0.401000, + "Standard_D1": 0.080400, + "Standard_D11": 0.208000, + "Standard_D11_v2": 0.179000, + "Standard_D12": 0.416000, + "Standard_D12_v2": 0.359000, + "Standard_D13": 0.832000, + "Standard_D13_v2": 0.718000, + "Standard_D14": 1.665000, + "Standard_D14_v2": 1.435000, + "Standard_D15_v2": 1.794000, + "Standard_D15i_v2": 1.794000, + "Standard_D16_v3": 0.960000, + "Standard_D16_v4": 0.942000, + "Standard_D16_v5": 0.942000, + "Standard_D16a_v4": 0.942000, + "Standard_D16ads_v5": 1.015000, + "Standard_D16as_v4": 0.942000, + "Standard_D16as_v5": 1.584000, + "Standard_D16d_v4": 1.109000, + "Standard_D16d_v5": 1.109000, + "Standard_D16ds_v4": 1.109000, + "Standard_D16ds_v5": 1.109000, + "Standard_D16lds_v5": 0.960000, + "Standard_D16ls_v5": 0.845000, + "Standard_D16s_v3": 0.960000, + "Standard_D16s_v4": 0.942000, + "Standard_D16s_v5": 0.942000, + "Standard_D1_v2": 0.068700, + "Standard_D2": 0.161000, + "Standard_D2_v2": 0.137000, + "Standard_D2_v3": 0.120000, + "Standard_D2_v4": 0.118000, + "Standard_D2_v5": 0.118000, + "Standard_D2a_v4": 0.118000, + "Standard_D2ads_v5": 0.219000, + "Standard_D2as_v4": 0.118000, + "Standard_D2as_v5": 0.198000, + "Standard_D2d_v4": 0.139000, + "Standard_D2d_v5": 0.139000, + "Standard_D2ds_v4": 0.139000, + "Standard_D2ds_v5": 0.139000, + "Standard_D2lds_v5": 0.120000, + "Standard_D2ls_v5": 0.106000, + "Standard_D2s_v3": 0.120000, + "Standard_D2s_v4": 0.118000, + "Standard_D2s_v5": 0.118000, + "Standard_D3": 0.322000, + "Standard_D32-16s_v3": 1.920000, + "Standard_D32-8s_v3": 1.920000, + "Standard_D32_v3": 1.920000, + "Standard_D32_v4": 1.883000, + "Standard_D32_v5": 1.883000, + "Standard_D32a_v4": 1.883000, + "Standard_D32ads_v5": 3.502000, + "Standard_D32as_v4": 1.883000, + "Standard_D32as_v5": 1.696000, + "Standard_D32d_v4": 2.218000, + "Standard_D32d_v5": 2.218000, + "Standard_D32ds_v4": 2.218000, + "Standard_D32ds_v5": 2.218000, + "Standard_D32lds_v5": 1.920000, + "Standard_D32ls_v5": 1.690000, + "Standard_D32s_v3": 1.920000, + "Standard_D32s_v4": 1.883000, + "Standard_D32s_v5": 1.883000, + "Standard_D3_v2": 0.275000, + "Standard_D4": 0.643000, + "Standard_D48_v3": 2.880000, + "Standard_D48_v4": 2.825000, + "Standard_D48_v5": 2.825000, + "Standard_D48a_v4": 2.825000, + "Standard_D48ads_v5": 5.254000, + "Standard_D48as_v4": 2.825000, + "Standard_D48as_v5": 4.752000, + "Standard_D48d_v4": 3.326000, + "Standard_D48d_v5": 3.326000, + "Standard_D48ds_v4": 3.326000, + "Standard_D48ds_v5": 3.326000, + "Standard_D48lds_v5": 2.880000, + "Standard_D48ls_v5": 2.534000, + "Standard_D48s_v3": 2.880000, + "Standard_D48s_v4": 2.825000, + "Standard_D48s_v5": 2.825000, + "Standard_D4_v2": 0.550000, + "Standard_D4_v3": 0.240000, + "Standard_D4_v4": 0.235000, + "Standard_D4_v5": 0.235000, + "Standard_D4a_v4": 0.235000, + "Standard_D4ads_v5": 0.254000, + "Standard_D4as_v4": 0.235000, + "Standard_D4as_v5": 0.396000, + "Standard_D4d_v4": 0.277000, + "Standard_D4d_v5": 0.277000, + "Standard_D4ds_v4": 0.277000, + "Standard_D4ds_v5": 0.277000, + "Standard_D4lds_v5": 0.240000, + "Standard_D4ls_v5": 0.211000, + "Standard_D4s_v3": 0.240000, + "Standard_D4s_v4": 0.235000, + "Standard_D4s_v5": 0.235000, + "Standard_D5_v2": 1.099000, + "Standard_D64-16s_v3": 3.840000, + "Standard_D64-32s_v3": 3.840000, + "Standard_D64_v3": 3.840000, + "Standard_D64_v4": 3.766000, + "Standard_D64_v5": 3.766000, + "Standard_D64a_v4": 3.766000, + "Standard_D64ads_v5": 4.061000, + "Standard_D64as_v4": 3.766000, + "Standard_D64as_v5": 6.336000, + "Standard_D64d_v4": 4.435000, + "Standard_D64d_v5": 4.435000, + "Standard_D64ds_v4": 4.435000, + "Standard_D64ds_v5": 4.435000, + "Standard_D64lds_v5": 3.840000, + "Standard_D64ls_v5": 3.379000, + "Standard_D64s_v3": 3.840000, + "Standard_D64s_v4": 3.766000, + "Standard_D64s_v5": 3.766000, + "Standard_D8_v3": 0.480000, + "Standard_D8_v4": 0.471000, + "Standard_D8_v5": 0.471000, + "Standard_D8a_v4": 0.471000, + "Standard_D8ads_v5": 0.508000, + "Standard_D8as_v4": 0.471000, + "Standard_D8as_v5": 0.792000, + "Standard_D8d_v4": 0.554000, + "Standard_D8d_v5": 0.554000, + "Standard_D8ds_v4": 0.554000, + "Standard_D8ds_v5": 0.554000, + "Standard_D8lds_v5": 0.480000, + "Standard_D8ls_v5": 0.422000, + "Standard_D8s_v3": 0.480000, + "Standard_D8s_v4": 0.471000, + "Standard_D8s_v5": 0.471000, + "Standard_D96_v5": 5.650000, + "Standard_D96a_v4": 5.650000, + "Standard_D96ads_v5": 6.091000, + "Standard_D96as_v4": 5.650000, + "Standard_D96as_v5": 9.504000, + "Standard_D96d_v5": 6.653000, + "Standard_D96ds_v5": 6.653000, + "Standard_D96lds_v5": 5.760000, + "Standard_D96ls_v5": 5.069000, + "Standard_D96s_v5": 5.650000, + "Standard_DC16ads_v5": 1.015000, + "Standard_DC16as_v5": 0.848000, + "Standard_DC16ds_v3": 1.193000, + "Standard_DC16s_v3": 1.014000, + "Standard_DC1ds_v3": 0.074600, + "Standard_DC1s_v3": 0.063400, + "Standard_DC24ds_v3": 1.790000, + "Standard_DC24s_v3": 1.521000, + "Standard_DC2ads_v5": 0.127000, + "Standard_DC2as_v5": 0.106000, + "Standard_DC2ds_v3": 0.149000, + "Standard_DC2s_v3": 0.127000, + "Standard_DC32ads_v5": 2.030000, + "Standard_DC32as_v5": 1.696000, + "Standard_DC32ds_v3": 2.386000, + "Standard_DC32s_v3": 2.028000, + "Standard_DC48ads_v5": 3.046000, + "Standard_DC48as_v5": 2.544000, + "Standard_DC48ds_v3": 3.579000, + "Standard_DC48s_v3": 3.042000, + "Standard_DC4ads_v5": 0.254000, + "Standard_DC4as_v5": 0.212000, + "Standard_DC4ds_v3": 0.298000, + "Standard_DC4s_v3": 0.253000, + "Standard_DC64ads_v5": 4.061000, + "Standard_DC64as_v5": 3.392000, + "Standard_DC8ads_v5": 0.508000, + "Standard_DC8as_v5": 0.424000, + "Standard_DC8ds_v3": 0.597000, + "Standard_DC8s_v3": 0.507000, + "Standard_DC96ads_v5": 6.091000, + "Standard_DC96as_v5": 5.088000, + "Standard_DS1": 0.080400, + "Standard_DS11": 0.208000, + "Standard_DS11-1_v2": 0.179000, + "Standard_DS11_v2": 0.179000, + "Standard_DS12": 0.416000, + "Standard_DS12-1_v2": 0.359000, + "Standard_DS12-2_v2": 0.359000, + "Standard_DS12_v2": 0.359000, + "Standard_DS13": 0.832000, + "Standard_DS13-2_v2": 0.718000, + "Standard_DS13-4_v2": 0.718000, + "Standard_DS13_v2": 0.718000, + "Standard_DS14": 1.665000, + "Standard_DS14-4_v2": 1.435000, + "Standard_DS14-8_v2": 1.435000, + "Standard_DS14_v2": 1.435000, + "Standard_DS15_v2": 1.794000, + "Standard_DS15i_v2": 1.794000, + "Standard_DS1_v2": 0.068700, + "Standard_DS2": 0.161000, + "Standard_DS2_v2": 0.137000, + "Standard_DS3": 0.322000, + "Standard_DS3_v2": 0.275000, + "Standard_DS4": 0.643000, + "Standard_DS4_v2": 0.550000, + "Standard_DS5_v2": 1.099000, + "Standard_E104i_v5": 8.872000, + "Standard_E104id_v5": 10.067000, + "Standard_E104ids_v5": 10.067000, + "Standard_E104is_v5": 8.872000, + "Standard_E112iads_v5": 9.911000, + "Standard_E112ias_v5": 8.624000, + "Standard_E112ibds_v5": 12.690000, + "Standard_E112ibs_v5": 11.396000, + "Standard_E16-4ads_v5": 2.023000, + "Standard_E16-4as_v4": 1.241000, + "Standard_E16-4as_v5": 1.856000, + "Standard_E16-4ds_v4": 1.408000, + "Standard_E16-4ds_v5": 1.408000, + "Standard_E16-4s_v3": 1.210000, + "Standard_E16-4s_v4": 1.241000, + "Standard_E16-4s_v5": 1.241000, + "Standard_E16-8ads_v5": 1.287000, + "Standard_E16-8as_v4": 1.241000, + "Standard_E16-8as_v5": 1.856000, + "Standard_E16-8ds_v4": 1.408000, + "Standard_E16-8ds_v5": 1.408000, + "Standard_E16-8s_v3": 1.210000, + "Standard_E16-8s_v4": 1.241000, + "Standard_E16-8s_v5": 1.241000, + "Standard_E16_v3": 1.210000, + "Standard_E16_v4": 1.241000, + "Standard_E16_v5": 1.241000, + "Standard_E16a_v4": 1.241000, + "Standard_E16ads_v5": 1.333000, + "Standard_E16as_v4": 1.241000, + "Standard_E16as_v5": 1.166000, + "Standard_E16bds_v5": 1.648000, + "Standard_E16bs_v5": 1.480000, + "Standard_E16d_v4": 1.408000, + "Standard_E16d_v5": 1.408000, + "Standard_E16ds_v4": 1.408000, + "Standard_E16ds_v5": 1.408000, + "Standard_E16s_v3": 1.210000, + "Standard_E16s_v4": 1.241000, + "Standard_E16s_v5": 1.241000, + "Standard_E20_v3": 1.600000, + "Standard_E20_v4": 1.550000, + "Standard_E20_v5": 1.551000, + "Standard_E20a_v4": 1.551000, + "Standard_E20ads_v5": 1.609000, + "Standard_E20as_v4": 1.551000, + "Standard_E20as_v5": 1.446000, + "Standard_E20d_v4": 1.760000, + "Standard_E20d_v5": 1.760000, + "Standard_E20ds_v4": 1.760000, + "Standard_E20ds_v5": 1.760000, + "Standard_E20s_v3": 1.600000, + "Standard_E20s_v4": 1.550000, + "Standard_E20s_v5": 1.551000, + "Standard_E2_v3": 0.151000, + "Standard_E2_v4": 0.155000, + "Standard_E2_v5": 0.155000, + "Standard_E2a_v4": 0.155000, + "Standard_E2ads_v5": 0.161000, + "Standard_E2as_v4": 0.155000, + "Standard_E2as_v5": 0.140000, + "Standard_E2bds_v5": 0.206000, + "Standard_E2bs_v5": 0.185000, + "Standard_E2d_v4": 0.176000, + "Standard_E2d_v5": 0.176000, + "Standard_E2ds_v4": 0.176000, + "Standard_E2ds_v5": 0.176000, + "Standard_E2s_v3": 0.151000, + "Standard_E2s_v4": 0.155000, + "Standard_E2s_v5": 0.155000, + "Standard_E32-16ads_v5": 2.574000, + "Standard_E32-16as_v4": 2.482000, + "Standard_E32-16as_v5": 2.240000, + "Standard_E32-16ds_v4": 2.816000, + "Standard_E32-16ds_v5": 2.816000, + "Standard_E32-16s_v3": 2.419000, + "Standard_E32-16s_v4": 2.482000, + "Standard_E32-16s_v5": 2.482000, + "Standard_E32-8ads_v5": 2.574000, + "Standard_E32-8as_v4": 2.482000, + "Standard_E32-8as_v5": 2.240000, + "Standard_E32-8ds_v4": 2.816000, + "Standard_E32-8ds_v5": 2.816000, + "Standard_E32-8s_v3": 2.419000, + "Standard_E32-8s_v4": 2.482000, + "Standard_E32-8s_v5": 2.482000, + "Standard_E32_v3": 2.419000, + "Standard_E32_v4": 2.482000, + "Standard_E32_v5": 2.482000, + "Standard_E32a_v4": 2.482000, + "Standard_E32ads_v5": 2.574000, + "Standard_E32as_v4": 2.482000, + "Standard_E32as_v5": 2.240000, + "Standard_E32bds_v5": 3.296000, + "Standard_E32bs_v5": 2.960000, + "Standard_E32d_v4": 2.816000, + "Standard_E32d_v5": 2.816000, + "Standard_E32ds_v4": 2.816000, + "Standard_E32ds_v5": 2.816000, + "Standard_E32s_v3": 2.419000, + "Standard_E32s_v4": 2.482000, + "Standard_E32s_v5": 2.482000, + "Standard_E4-2ads_v5": 0.322000, + "Standard_E4-2as_v4": 0.310000, + "Standard_E4-2as_v5": 0.464000, + "Standard_E4-2ds_v4": 0.352000, + "Standard_E4-2ds_v5": 0.352000, + "Standard_E4-2s_v3": 0.302000, + "Standard_E4-2s_v4": 0.310000, + "Standard_E4-2s_v5": 0.310000, + "Standard_E48_v3": 3.629000, + "Standard_E48_v4": 3.722000, + "Standard_E48_v5": 3.722000, + "Standard_E48a_v4": 3.722000, + "Standard_E48ads_v5": 3.862000, + "Standard_E48as_v4": 3.722000, + "Standard_E48as_v5": 3.360000, + "Standard_E48bds_v5": 4.944000, + "Standard_E48bs_v5": 4.440000, + "Standard_E48d_v4": 4.224000, + "Standard_E48d_v5": 4.224000, + "Standard_E48ds_v4": 4.224000, + "Standard_E48ds_v5": 4.224000, + "Standard_E48s_v3": 3.629000, + "Standard_E48s_v4": 3.722000, + "Standard_E48s_v5": 3.722000, + "Standard_E4_v3": 0.302000, + "Standard_E4_v4": 0.310000, + "Standard_E4_v5": 0.310000, + "Standard_E4a_v4": 0.310000, + "Standard_E4ads_v5": 0.322000, + "Standard_E4as_v4": 0.310000, + "Standard_E4as_v5": 0.280000, + "Standard_E4bds_v5": 0.412000, + "Standard_E4bs_v5": 0.370000, + "Standard_E4d_v4": 0.352000, + "Standard_E4d_v5": 0.352000, + "Standard_E4ds_v4": 0.352000, + "Standard_E4ds_v5": 0.352000, + "Standard_E4s_v3": 0.302000, + "Standard_E4s_v4": 0.310000, + "Standard_E4s_v5": 0.310000, + "Standard_E64-16ads_v5": 5.149000, + "Standard_E64-16as_v4": 4.963000, + "Standard_E64-16as_v5": 7.424000, + "Standard_E64-16ds_v4": 5.632000, + "Standard_E64-16ds_v5": 5.632000, + "Standard_E64-16s_v3": 4.355000, + "Standard_E64-16s_v4": 4.963000, + "Standard_E64-16s_v5": 4.963000, + "Standard_E64-32ads_v5": 5.149000, + "Standard_E64-32as_v4": 4.963000, + "Standard_E64-32as_v5": 7.424000, + "Standard_E64-32ds_v4": 5.632000, + "Standard_E64-32ds_v5": 5.632000, + "Standard_E64-32s_v3": 4.355000, + "Standard_E64-32s_v4": 4.963000, + "Standard_E64-32s_v5": 4.963000, + "Standard_E64_v3": 4.355000, + "Standard_E64_v4": 4.963000, + "Standard_E64_v5": 4.963000, + "Standard_E64a_v4": 4.963000, + "Standard_E64ads_v5": 5.195000, + "Standard_E64as_v4": 4.963000, + "Standard_E64as_v5": 4.526000, + "Standard_E64bds_v5": 6.592000, + "Standard_E64bs_v5": 5.920000, + "Standard_E64d_v4": 5.632000, + "Standard_E64d_v5": 5.632000, + "Standard_E64ds_v4": 5.632000, + "Standard_E64ds_v5": 5.632000, + "Standard_E64i_v3": 4.376000, + "Standard_E64is_v3": 4.376000, + "Standard_E64s_v3": 4.355000, + "Standard_E64s_v4": 4.963000, + "Standard_E64s_v5": 4.963000, + "Standard_E8-2ads_v5": 0.644000, + "Standard_E8-2as_v4": 0.620000, + "Standard_E8-2as_v5": 0.560000, + "Standard_E8-2ds_v4": 0.704000, + "Standard_E8-2ds_v5": 0.704000, + "Standard_E8-2s_v3": 0.605000, + "Standard_E8-2s_v4": 0.620000, + "Standard_E8-2s_v5": 0.620000, + "Standard_E8-4ads_v5": 0.644000, + "Standard_E8-4as_v4": 0.620000, + "Standard_E8-4as_v5": 0.560000, + "Standard_E8-4ds_v4": 0.704000, + "Standard_E8-4ds_v5": 0.704000, + "Standard_E8-4s_v3": 0.605000, + "Standard_E8-4s_v4": 0.620000, + "Standard_E8-4s_v5": 0.620000, + "Standard_E80ids_v4": 7.040000, + "Standard_E80is_v4": 6.200000, + "Standard_E8_v3": 0.605000, + "Standard_E8_v4": 0.620000, + "Standard_E8_v5": 0.620000, + "Standard_E8a_v4": 0.620000, + "Standard_E8ads_v5": 0.644000, + "Standard_E8as_v4": 0.620000, + "Standard_E8as_v5": 0.560000, + "Standard_E8bds_v5": 0.824000, + "Standard_E8bs_v5": 0.740000, + "Standard_E8d_v4": 0.704000, + "Standard_E8d_v5": 0.704000, + "Standard_E8ds_v4": 0.704000, + "Standard_E8ds_v5": 0.704000, + "Standard_E8s_v3": 0.605000, + "Standard_E8s_v4": 0.620000, + "Standard_E8s_v5": 0.620000, + "Standard_E96-24ads_v5": 7.723000, + "Standard_E96-24as_v4": 7.445000, + "Standard_E96-24as_v5": 6.720000, + "Standard_E96-24ds_v5": 8.448000, + "Standard_E96-24s_v5": 7.445000, + "Standard_E96-48ads_v5": 12.139000, + "Standard_E96-48as_v4": 7.445000, + "Standard_E96-48as_v5": 6.720000, + "Standard_E96-48ds_v5": 8.448000, + "Standard_E96-48s_v5": 7.445000, + "Standard_E96_v5": 7.445000, + "Standard_E96a_v4": 7.445000, + "Standard_E96ads_v5": 7.723000, + "Standard_E96as_v4": 7.445000, + "Standard_E96as_v5": 6.720000, + "Standard_E96bds_v5": 9.888000, + "Standard_E96bs_v5": 8.880000, + "Standard_E96d_v5": 8.448000, + "Standard_E96ds_v5": 8.448000, + "Standard_E96iads_v5": 8.496000, + "Standard_E96ias_v4": 7.445000, + "Standard_E96ias_v5": 7.392000, + "Standard_E96s_v5": 7.445000, + "Standard_EC16ads_v5": 1.287000, + "Standard_EC16as_v5": 1.120000, + "Standard_EC20ads_v5": 1.609000, + "Standard_EC20as_v5": 1.400000, + "Standard_EC2ads_v5": 0.161000, + "Standard_EC2as_v5": 0.140000, + "Standard_EC32ads_v5": 2.574000, + "Standard_EC32as_v5": 2.240000, + "Standard_EC48ads_v5": 3.862000, + "Standard_EC48as_v5": 3.360000, + "Standard_EC4ads_v5": 0.322000, + "Standard_EC4as_v5": 0.280000, + "Standard_EC64ads_v5": 5.149000, + "Standard_EC64as_v5": 4.480000, + "Standard_EC8ads_v5": 0.644000, + "Standard_EC8as_v5": 0.560000, + "Standard_EC96ads_v5": 7.723000, + "Standard_EC96as_v5": 6.720000, + "Standard_EC96iads_v5": 8.496000, + "Standard_EC96ias_v5": 7.392000, + "Standard_F1": 0.059700, + "Standard_F16": 0.955000, + "Standard_F16s": 0.955000, + "Standard_F16s_v2": 0.816000, + "Standard_F1s": 0.059700, + "Standard_F2": 0.119000, + "Standard_F2s": 0.119000, + "Standard_F2s_v2": 0.102000, + "Standard_F32s_v2": 1.632000, + "Standard_F4": 0.239000, + "Standard_F48s_v2": 2.448000, + "Standard_F4s": 0.239000, + "Standard_F4s_v2": 0.204000, + "Standard_F64s_v2": 3.264000, + "Standard_F72s_v2": 3.672000, + "Standard_F8": 0.478000, + "Standard_F8s": 0.478000, + "Standard_F8s_v2": 0.408000, + "Standard_HB120-16rs_v2": 4.752000, + "Standard_HB120-32rs_v2": 4.752000, + "Standard_HB120-64rs_v2": 4.752000, + "Standard_HB120-96rs_v2": 4.752000, + "Standard_HB120rs_v2": 4.752000, + "Standard_L16as_v3": 1.512000, + "Standard_L16s_v3": 1.688000, + "Standard_L32as_v3": 3.024000, + "Standard_L32s_v3": 3.376000, + "Standard_L48as_v3": 4.536000, + "Standard_L48s_v3": 5.064000, + "Standard_L64as_v3": 6.048000, + "Standard_L64s_v3": 6.752000, + "Standard_L80as_v3": 7.560000, + "Standard_L80s_v3": 8.440000, + "Standard_L8as_v3": 0.756000, + "Standard_L8s_v3": 0.844000, + "Standard_M128": 17.606000, + "Standard_M128-32ms": 35.228000, + "Standard_M128-64ms": 35.228000, + "Standard_M128dms_v2": 35.231000, + "Standard_M128ds_v2": 17.609000, + "Standard_M128m": 35.228000, + "Standard_M128ms": 35.228000, + "Standard_M128ms_v2": 34.675000, + "Standard_M128s": 17.606000, + "Standard_M128s_v2": 17.053000, + "Standard_M16-4ms": 4.056000, + "Standard_M16-8ms": 4.056000, + "Standard_M16ms": 4.056000, + "Standard_M16s": 3.151000, + "Standard_M192idms_v2": 42.324000, + "Standard_M192ids_v2": 21.162000, + "Standard_M192ims_v2": 41.769000, + "Standard_M192is_v2": 20.607000, + "Standard_M208-104ms_v2": 58.898000, + "Standard_M208-104s_v2": 29.449000, + "Standard_M208-52ms_v2": 58.898000, + "Standard_M208-52s_v2": 29.449000, + "Standard_M208ms_v2": 58.898000, + "Standard_M208s_v2": 29.449000, + "Standard_M32-16ms": 8.113000, + "Standard_M32-8ms": 8.113000, + "Standard_M32dms_v2": 8.113000, + "Standard_M32ls": 3.535000, + "Standard_M32ms": 8.113000, + "Standard_M32ms_v2": 7.974000, + "Standard_M32s": 4.402200, + "Standard_M32ts": 3.331000, + "Standard_M416-104ms_v2": 130.880000, + "Standard_M416-104s_v2": 65.446000, + "Standard_M416-208ms_v2": 130.880000, + "Standard_M416-208s_v2": 65.446000, + "Standard_M416is_v2": 65.446000, + "Standard_M416ms_v2": 130.880000, + "Standard_M416s_8_v2": 94.540000, + "Standard_M416s_v2": 65.446000, + "Standard_M64": 8.803000, + "Standard_M64-16ms": 13.645000, + "Standard_M64-32ms": 13.645000, + "Standard_M64dms_v2": 13.649000, + "Standard_M64ds_v2": 8.803000, + "Standard_M64ls": 6.664000, + "Standard_M64m": 13.645000, + "Standard_M64ms": 13.645000, + "Standard_M64ms_v2": 13.371000, + "Standard_M64s": 8.803000, + "Standard_M64s_v2": 8.525000, + "Standard_M8-2ms": 2.028000, + "Standard_M8-4ms": 2.028000, + "Standard_M8ms": 2.028000, + "Standard_ND96ams_A100_v4": 46.861000, + "Standard_ND96amsr_A100_v4": 46.861000, + "Standard_NV12ads_A10_v5": 1.298000, + "Standard_NV12s_v3": 1.391000, + "Standard_NV18ads_A10_v5": 2.288000, + "Standard_NV24s_v3": 2.782000, + "Standard_NV36adms_A10_v5": 6.464000, + "Standard_NV36ads_A10_v5": 4.576000, + "Standard_NV48s_v3": 5.563000, + "Standard_NV6ads_A10_v5": 0.649000, + "Standard_NV72ads_A10_v5": 9.324000, + } + // uksouth + initialOnDemandPrices["uksouth"] = map[string]float64{ + "Basic_A0": 0.018000, + "Basic_A1": 0.026000, + "Basic_A2": 0.078000, + "Basic_A3": 0.199000, + "Basic_A4": 0.397000, + "DCdsv3 Type1": 6.917000, + "DCsv2 Type 1": 0.977000, + "DCsv3 Type1": 5.861000, + "Dadsv5_Type1": 7.392000, + "Dasv4_Type1": 5.861000, + "Dasv4_Type2": 6.105000, + "Dasv5_Type1": 6.160000, + "Ddsv4_Type 1": 4.611000, + "Ddsv4_Type2": 5.476000, + "Ddsv5_Type1": 6.917000, + "Dsv3_Type3": 4.885000, + "Dsv3_Type4": 6.106000, + "Dsv4_Type1": 4.884000, + "Dsv4_Type2": 5.861000, + "Dsv5_Type1": 6.105000, + "Eadsv5_Type1": 8.131000, + "Easv4_Type1": 7.814000, + "Easv4_Type2": 7.814000, + "Easv5_Type1": 7.022000, + "Ebdsv5-Type1": 6.952000, + "Ebsv5-Type1": 6.213000, + "Edsv4_Type 1": 5.949000, + "Edsv4_Type2": 7.064000, + "Edsv5_Type1": 8.923000, + "Esv3_Type3": 5.043000, + "Esv3_Type4": 6.832000, + "Esv4_Type1": 5.209000, + "Esv4_Type2": 6.838000, + "Esv5_Type1": 7.814000, + "Fsv2 Type3": 4.666000, + "Fsv2_Type2": 4.000000, + "Fsv2_Type4": 5.333000, + "Lsv2_Type1": 7.962000, + "Lsv3_Type1": 8.888000, + "Mdmsv2MedMem _Type1": 36.992000, + "Mdsv2MedMem_Type1": 18.489000, + "Ms_Type1": 18.483000, + "Msm_Type1": 36.998000, + "Msmv2MedMem Type1": 36.409000, + "Msmv2_Type1": 136.330000, + "Msv2MedMem Type1": 17.906000, + "Msv2_Type1": 68.173000, + "NVasv4_Type1": 8.970000, + "NVsv3_Type1": 6.270000, + "Standard_A0": 0.022000, + "Standard_A1": 0.066000, + "Standard_A1_v2": 0.041000, + "Standard_A2": 0.132000, + "Standard_A2_v2": 0.087000, + "Standard_A2m_v2": 0.124000, + "Standard_A3": 0.264000, + "Standard_A4": 0.528000, + "Standard_A4_v2": 0.183000, + "Standard_A4m_v2": 0.260000, + "Standard_A5": 0.297000, + "Standard_A6": 0.594000, + "Standard_A7": 1.188000, + "Standard_A8_v2": 0.383000, + "Standard_A8m_v2": 0.546000, + "Standard_B12ms": 0.566000, + "Standard_B16als_v2": 0.676000, + "Standard_B16as_v2": 0.754000, + "Standard_B16ls_v2": 0.742000, + "Standard_B16ms": 0.755000, + "Standard_B16pls_v2": 0.533000, + "Standard_B16ps_v2": 0.675000, + "Standard_B16s_v2": 0.755000, + "Standard_B1ls": 0.005900, + "Standard_B1ms": 0.023600, + "Standard_B1s": 0.011800, + "Standard_B20ms": 0.944000, + "Standard_B2als_v2": 0.051700, + "Standard_B2as_v2": 0.094200, + "Standard_B2ats_v2": 0.019800, + "Standard_B2ls_v2": 0.056400, + "Standard_B2ms": 0.094400, + "Standard_B2pls_v2": 0.037600, + "Standard_B2ps_v2": 0.075200, + "Standard_B2pts_v2": 0.018600, + "Standard_B2s": 0.047200, + "Standard_B2s_v2": 0.094400, + "Standard_B2ts_v2": 0.011800, + "Standard_B32als_v2": 1.351000, + "Standard_B32as_v2": 1.507000, + "Standard_B32ls_v2": 1.337000, + "Standard_B32s_v2": 1.510000, + "Standard_B4als_v2": 0.151000, + "Standard_B4as_v2": 0.170000, + "Standard_B4ls_v2": 0.186000, + "Standard_B4ms": 0.189000, + "Standard_B4pls_v2": 0.133000, + "Standard_B4ps_v2": 0.150000, + "Standard_B4s_v2": 0.207000, + "Standard_B8als_v2": 0.301000, + "Standard_B8as_v2": 0.377000, + "Standard_B8ls_v2": 0.371000, + "Standard_B8ms": 0.378000, + "Standard_B8pls_v2": 0.303000, + "Standard_B8ps_v2": 0.301000, + "Standard_B8s_v2": 0.378000, + "Standard_D11_v2": 0.234000, + "Standard_D11_v2_Promo": 0.234000, + "Standard_D12_v2": 0.469000, + "Standard_D12_v2_Promo": 0.469000, + "Standard_D13_v2": 0.937000, + "Standard_D13_v2_Promo": 0.937000, + "Standard_D14_v2": 1.874000, + "Standard_D14_v2_Promo": 1.874000, + "Standard_D15_v2": 2.343000, + "Standard_D15i_v2": 2.343000, + "Standard_D16_v3": 0.928000, + "Standard_D16_v4": 0.888000, + "Standard_D16_v5": 0.888000, + "Standard_D16a_v4": 0.888000, + "Standard_D16ads_v5": 1.696000, + "Standard_D16as_v4": 0.888000, + "Standard_D16as_v5": 0.800000, + "Standard_D16d_v4": 1.048000, + "Standard_D16d_v5": 1.048000, + "Standard_D16ds_v4": 1.048000, + "Standard_D16ds_v5": 1.048000, + "Standard_D16lds_v5": 0.920000, + "Standard_D16ls_v5": 0.808000, + "Standard_D16pds_v5": 0.834000, + "Standard_D16plds_v5": 0.736000, + "Standard_D16pls_v5": 0.646000, + "Standard_D16ps_v5": 0.710000, + "Standard_D16s_v3": 0.928000, + "Standard_D16s_v4": 0.888000, + "Standard_D16s_v5": 0.888000, + "Standard_D1_v2": 0.087800, + "Standard_D2_v2": 0.176000, + "Standard_D2_v2_Promo": 0.176000, + "Standard_D2_v3": 0.116000, + "Standard_D2_v4": 0.111000, + "Standard_D2_v5": 0.111000, + "Standard_D2a_v4": 0.111000, + "Standard_D2ads_v5": 0.212000, + "Standard_D2as_v4": 0.111000, + "Standard_D2as_v5": 0.100000, + "Standard_D2d_v4": 0.131000, + "Standard_D2d_v5": 0.131000, + "Standard_D2ds_v4": 0.131000, + "Standard_D2ds_v5": 0.131000, + "Standard_D2lds_v5": 0.115000, + "Standard_D2ls_v5": 0.101000, + "Standard_D2pds_v5": 0.104000, + "Standard_D2plds_v5": 0.092000, + "Standard_D2pls_v5": 0.080800, + "Standard_D2ps_v5": 0.088800, + "Standard_D2s_v3": 0.116000, + "Standard_D2s_v4": 0.111000, + "Standard_D2s_v5": 0.111000, + "Standard_D32-16s_v3": 1.856000, + "Standard_D32-8s_v3": 1.856000, + "Standard_D32_v3": 1.856000, + "Standard_D32_v4": 1.776000, + "Standard_D32_v5": 1.776000, + "Standard_D32a_v4": 1.776000, + "Standard_D32ads_v5": 1.920000, + "Standard_D32as_v4": 1.776000, + "Standard_D32as_v5": 1.600000, + "Standard_D32d_v4": 2.096000, + "Standard_D32d_v5": 2.096000, + "Standard_D32ds_v4": 2.096000, + "Standard_D32ds_v5": 2.096000, + "Standard_D32lds_v5": 1.840000, + "Standard_D32ls_v5": 1.616000, + "Standard_D32pds_v5": 1.667000, + "Standard_D32plds_v5": 1.472000, + "Standard_D32pls_v5": 1.293000, + "Standard_D32ps_v5": 1.421000, + "Standard_D32s_v3": 1.856000, + "Standard_D32s_v4": 1.776000, + "Standard_D32s_v5": 1.776000, + "Standard_D3_v2": 0.351000, + "Standard_D3_v2_Promo": 0.351000, + "Standard_D48_v3": 2.784000, + "Standard_D48_v4": 2.664000, + "Standard_D48_v5": 2.664000, + "Standard_D48a_v4": 2.664000, + "Standard_D48ads_v5": 5.088000, + "Standard_D48as_v4": 2.664000, + "Standard_D48as_v5": 4.608000, + "Standard_D48d_v4": 3.144000, + "Standard_D48d_v5": 3.144000, + "Standard_D48ds_v4": 3.144000, + "Standard_D48ds_v5": 3.144000, + "Standard_D48lds_v5": 2.760000, + "Standard_D48ls_v5": 2.424000, + "Standard_D48pds_v5": 2.501000, + "Standard_D48plds_v5": 2.208000, + "Standard_D48pls_v5": 1.939000, + "Standard_D48ps_v5": 2.131000, + "Standard_D48s_v3": 2.784000, + "Standard_D48s_v4": 2.664000, + "Standard_D48s_v5": 2.664000, + "Standard_D4_v2": 0.702000, + "Standard_D4_v2_Promo": 0.702000, + "Standard_D4_v3": 0.232000, + "Standard_D4_v4": 0.222000, + "Standard_D4_v5": 0.222000, + "Standard_D4a_v4": 0.222000, + "Standard_D4ads_v5": 0.424000, + "Standard_D4as_v4": 0.222000, + "Standard_D4as_v5": 0.384000, + "Standard_D4d_v4": 0.262000, + "Standard_D4d_v5": 0.262000, + "Standard_D4ds_v4": 0.262000, + "Standard_D4ds_v5": 0.262000, + "Standard_D4lds_v5": 0.230000, + "Standard_D4ls_v5": 0.202000, + "Standard_D4pds_v5": 0.208000, + "Standard_D4plds_v5": 0.184000, + "Standard_D4pls_v5": 0.162000, + "Standard_D4ps_v5": 0.178000, + "Standard_D4s_v3": 0.232000, + "Standard_D4s_v4": 0.222000, + "Standard_D4s_v5": 0.222000, + "Standard_D5_v2": 1.405000, + "Standard_D5_v2_Promo": 1.405000, + "Standard_D64-16s_v3": 3.712000, + "Standard_D64-32s_v3": 3.712000, + "Standard_D64_v3": 3.712000, + "Standard_D64_v4": 3.552000, + "Standard_D64_v5": 3.552000, + "Standard_D64a_v4": 3.552000, + "Standard_D64ads_v5": 3.840000, + "Standard_D64as_v4": 3.552000, + "Standard_D64as_v5": 3.200000, + "Standard_D64d_v4": 4.192000, + "Standard_D64d_v5": 4.192000, + "Standard_D64ds_v4": 4.192000, + "Standard_D64ds_v5": 4.192000, + "Standard_D64lds_v5": 3.680000, + "Standard_D64ls_v5": 3.232000, + "Standard_D64pds_v5": 3.334000, + "Standard_D64plds_v5": 2.944000, + "Standard_D64pls_v5": 2.586000, + "Standard_D64ps_v5": 2.842000, + "Standard_D64s_v3": 3.712000, + "Standard_D64s_v4": 3.552000, + "Standard_D64s_v5": 3.552000, + "Standard_D8_v3": 0.464000, + "Standard_D8_v4": 0.444000, + "Standard_D8_v5": 0.444000, + "Standard_D8a_v4": 0.444000, + "Standard_D8ads_v5": 0.480000, + "Standard_D8as_v4": 0.444000, + "Standard_D8as_v5": 0.400000, + "Standard_D8d_v4": 0.524000, + "Standard_D8d_v5": 0.524000, + "Standard_D8ds_v4": 0.524000, + "Standard_D8ds_v5": 0.524000, + "Standard_D8lds_v5": 0.460000, + "Standard_D8ls_v5": 0.404000, + "Standard_D8pds_v5": 0.417000, + "Standard_D8plds_v5": 0.368000, + "Standard_D8pls_v5": 0.323000, + "Standard_D8ps_v5": 0.355000, + "Standard_D8s_v3": 0.464000, + "Standard_D8s_v4": 0.444000, + "Standard_D8s_v5": 0.444000, + "Standard_D96_v5": 5.328000, + "Standard_D96a_v4": 5.328000, + "Standard_D96ads_v5": 10.176000, + "Standard_D96as_v4": 5.328000, + "Standard_D96as_v5": 9.216000, + "Standard_D96d_v5": 6.288000, + "Standard_D96ds_v5": 6.288000, + "Standard_D96lds_v5": 5.520000, + "Standard_D96ls_v5": 4.848000, + "Standard_D96s_v5": 5.328000, + "Standard_DC16ads_v5": 0.960000, + "Standard_DC16as_v5": 0.800000, + "Standard_DC16ds_v3": 2.096000, + "Standard_DC16s_v3": 1.776000, + "Standard_DC1ds_v3": 0.131000, + "Standard_DC1s_v2": 0.111000, + "Standard_DC1s_v3": 0.111000, + "Standard_DC24ds_v3": 3.144000, + "Standard_DC24s_v3": 2.664000, + "Standard_DC2ads_v5": 0.120000, + "Standard_DC2as_v5": 0.100000, + "Standard_DC2ds_v3": 0.262000, + "Standard_DC2s_v2": 0.222000, + "Standard_DC2s_v3": 0.222000, + "Standard_DC32ads_v5": 1.920000, + "Standard_DC32as_v5": 1.600000, + "Standard_DC32ds_v3": 4.192000, + "Standard_DC32s_v3": 3.552000, + "Standard_DC48ads_v5": 2.880000, + "Standard_DC48as_v5": 2.400000, + "Standard_DC48ds_v3": 6.288000, + "Standard_DC48s_v3": 5.328000, + "Standard_DC4ads_v5": 0.240000, + "Standard_DC4as_v5": 0.200000, + "Standard_DC4ds_v3": 0.524000, + "Standard_DC4s_v2": 0.444000, + "Standard_DC4s_v3": 0.444000, + "Standard_DC64ads_v5": 3.840000, + "Standard_DC64as_v5": 3.200000, + "Standard_DC8_v2": 0.888000, + "Standard_DC8ads_v5": 0.480000, + "Standard_DC8as_v5": 0.400000, + "Standard_DC8ds_v3": 1.048000, + "Standard_DC8s_v3": 0.888000, + "Standard_DC96ads_v5": 5.760000, + "Standard_DC96as_v5": 4.800000, + "Standard_DS11-1_v2": 0.234000, + "Standard_DS11_v2": 0.234000, + "Standard_DS11_v2_Promo": 0.234000, + "Standard_DS12-1_v2": 0.469000, + "Standard_DS12-2_v2": 0.469000, + "Standard_DS12_v2": 0.469000, + "Standard_DS12_v2_Promo": 0.469000, + "Standard_DS13-2_v2": 0.937000, + "Standard_DS13-4_v2": 0.937000, + "Standard_DS13_v2": 0.937000, + "Standard_DS13_v2_Promo": 0.937000, + "Standard_DS14-4_v2": 1.874000, + "Standard_DS14-8_v2": 1.874000, + "Standard_DS14_v2": 1.874000, + "Standard_DS14_v2_Promo": 1.874000, + "Standard_DS15_v2": 2.343000, + "Standard_DS15i_v2": 2.343000, + "Standard_DS1_v2": 0.087800, + "Standard_DS2_v2": 0.176000, + "Standard_DS2_v2_Promo": 0.176000, + "Standard_DS3_v2": 0.351000, + "Standard_DS3_v2_Promo": 0.351000, + "Standard_DS4_v2": 0.702000, + "Standard_DS4_v2_Promo": 0.702000, + "Standard_DS5_v2": 1.405000, + "Standard_DS5_v2_Promo": 1.405000, + "Standard_E104i_v5": 8.466000, + "Standard_E104id_v5": 9.667000, + "Standard_E104ids_v5": 9.667000, + "Standard_E104is_v5": 8.466000, + "Standard_E112iads_v5": 9.486000, + "Standard_E112ias_v5": 8.193000, + "Standard_E112ibds_v5": 12.166000, + "Standard_E112ibs_v5": 10.872000, + "Standard_E16-4ads_v5": 1.968000, + "Standard_E16-4as_v4": 1.184000, + "Standard_E16-4as_v5": 1.064000, + "Standard_E16-4ds_v4": 1.352000, + "Standard_E16-4ds_v5": 1.352000, + "Standard_E16-4s_v3": 1.248000, + "Standard_E16-4s_v4": 1.184000, + "Standard_E16-4s_v5": 1.184000, + "Standard_E16-8ads_v5": 1.968000, + "Standard_E16-8as_v4": 1.184000, + "Standard_E16-8as_v5": 1.800000, + "Standard_E16-8ds_v4": 1.352000, + "Standard_E16-8ds_v5": 1.352000, + "Standard_E16-8s_v3": 1.248000, + "Standard_E16-8s_v4": 1.184000, + "Standard_E16-8s_v5": 1.184000, + "Standard_E16_v3": 1.248000, + "Standard_E16_v4": 1.184000, + "Standard_E16_v5": 1.184000, + "Standard_E16a_v4": 1.184000, + "Standard_E16ads_v5": 1.232000, + "Standard_E16as_v4": 1.184000, + "Standard_E16as_v5": 1.064000, + "Standard_E16bds_v5": 1.580000, + "Standard_E16bs_v5": 1.412000, + "Standard_E16d_v4": 1.352000, + "Standard_E16d_v5": 1.352000, + "Standard_E16ds_v4": 1.352000, + "Standard_E16ds_v5": 1.352000, + "Standard_E16pds_v5": 1.083000, + "Standard_E16ps_v5": 0.947000, + "Standard_E16s_v3": 1.248000, + "Standard_E16s_v4": 1.184000, + "Standard_E16s_v5": 1.184000, + "Standard_E20_v3": 1.560000, + "Standard_E20_v4": 1.480000, + "Standard_E20_v5": 1.480000, + "Standard_E20a_v4": 1.480000, + "Standard_E20ads_v5": 1.586000, + "Standard_E20as_v4": 1.480000, + "Standard_E20as_v5": 1.376000, + "Standard_E20d_v4": 1.690000, + "Standard_E20d_v5": 1.690000, + "Standard_E20ds_v4": 1.690000, + "Standard_E20ds_v5": 1.690000, + "Standard_E20pds_v5": 1.354000, + "Standard_E20ps_v5": 1.184000, + "Standard_E20s_v3": 1.560000, + "Standard_E20s_v4": 1.480000, + "Standard_E20s_v5": 1.480000, + "Standard_E2_v3": 0.156000, + "Standard_E2_v4": 0.148000, + "Standard_E2_v5": 0.148000, + "Standard_E2a_v4": 0.148000, + "Standard_E2ads_v5": 0.154000, + "Standard_E2as_v4": 0.148000, + "Standard_E2as_v5": 0.133000, + "Standard_E2bds_v5": 0.198000, + "Standard_E2bs_v5": 0.177000, + "Standard_E2d_v4": 0.169000, + "Standard_E2d_v5": 0.169000, + "Standard_E2ds_v4": 0.169000, + "Standard_E2ds_v5": 0.169000, + "Standard_E2pds_v5": 0.135000, + "Standard_E2ps_v5": 0.118000, + "Standard_E2s_v3": 0.156000, + "Standard_E2s_v4": 0.148000, + "Standard_E2s_v5": 0.148000, + "Standard_E32-16ads_v5": 3.936000, + "Standard_E32-16as_v4": 2.368000, + "Standard_E32-16as_v5": 2.128000, + "Standard_E32-16ds_v4": 2.704000, + "Standard_E32-16ds_v5": 2.704000, + "Standard_E32-16s_v3": 2.496000, + "Standard_E32-16s_v4": 2.368000, + "Standard_E32-16s_v5": 2.368000, + "Standard_E32-8ads_v5": 2.464000, + "Standard_E32-8as_v4": 2.368000, + "Standard_E32-8as_v5": 2.128000, + "Standard_E32-8ds_v4": 2.704000, + "Standard_E32-8ds_v5": 2.704000, + "Standard_E32-8s_v3": 2.496000, + "Standard_E32-8s_v4": 2.368000, + "Standard_E32-8s_v5": 2.368000, + "Standard_E32_v3": 2.496000, + "Standard_E32_v4": 2.368000, + "Standard_E32_v5": 2.368000, + "Standard_E32a_v4": 2.368000, + "Standard_E32ads_v5": 2.510000, + "Standard_E32as_v4": 2.368000, + "Standard_E32as_v5": 2.174000, + "Standard_E32bds_v5": 3.160000, + "Standard_E32bs_v5": 2.824000, + "Standard_E32d_v4": 2.704000, + "Standard_E32d_v5": 2.704000, + "Standard_E32ds_v4": 2.704000, + "Standard_E32ds_v5": 2.704000, + "Standard_E32pds_v5": 2.166000, + "Standard_E32ps_v5": 1.894000, + "Standard_E32s_v3": 2.496000, + "Standard_E32s_v4": 2.368000, + "Standard_E32s_v5": 2.368000, + "Standard_E4-2ads_v5": 0.308000, + "Standard_E4-2as_v4": 0.296000, + "Standard_E4-2as_v5": 0.450000, + "Standard_E4-2ds_v4": 0.338000, + "Standard_E4-2ds_v5": 0.338000, + "Standard_E4-2s_v3": 0.312000, + "Standard_E4-2s_v4": 0.296000, + "Standard_E4-2s_v5": 0.296000, + "Standard_E48_v3": 3.553000, + "Standard_E48_v4": 3.552000, + "Standard_E48_v5": 3.552000, + "Standard_E48a_v4": 3.552000, + "Standard_E48ads_v5": 3.696000, + "Standard_E48as_v4": 3.552000, + "Standard_E48as_v5": 3.192000, + "Standard_E48bds_v5": 4.740000, + "Standard_E48bs_v5": 4.236000, + "Standard_E48d_v4": 4.056000, + "Standard_E48d_v5": 4.056000, + "Standard_E48ds_v4": 4.056000, + "Standard_E48ds_v5": 4.056000, + "Standard_E48s_v3": 3.553000, + "Standard_E48s_v4": 3.552000, + "Standard_E48s_v5": 3.552000, + "Standard_E4_v3": 0.312000, + "Standard_E4_v4": 0.296000, + "Standard_E4_v5": 0.296000, + "Standard_E4a_v4": 0.296000, + "Standard_E4ads_v5": 0.308000, + "Standard_E4as_v4": 0.296000, + "Standard_E4as_v5": 0.266000, + "Standard_E4bds_v5": 0.395000, + "Standard_E4bs_v5": 0.353000, + "Standard_E4d_v4": 0.338000, + "Standard_E4d_v5": 0.338000, + "Standard_E4ds_v4": 0.338000, + "Standard_E4ds_v5": 0.338000, + "Standard_E4pds_v5": 0.271000, + "Standard_E4ps_v5": 0.237000, + "Standard_E4s_v3": 0.312000, + "Standard_E4s_v4": 0.296000, + "Standard_E4s_v5": 0.296000, + "Standard_E64-16ads_v5": 7.872000, + "Standard_E64-16as_v4": 4.736000, + "Standard_E64-16as_v5": 7.200000, + "Standard_E64-16ds_v4": 5.408000, + "Standard_E64-16ds_v5": 5.408000, + "Standard_E64-16s_v3": 4.264000, + "Standard_E64-16s_v4": 4.736000, + "Standard_E64-16s_v5": 4.736000, + "Standard_E64-32ads_v5": 7.872000, + "Standard_E64-32as_v4": 4.736000, + "Standard_E64-32as_v5": 7.200000, + "Standard_E64-32ds_v4": 5.408000, + "Standard_E64-32ds_v5": 5.408000, + "Standard_E64-32s_v3": 4.264000, + "Standard_E64-32s_v4": 4.736000, + "Standard_E64-32s_v5": 4.736000, + "Standard_E64_v3": 4.264000, + "Standard_E64_v4": 4.736000, + "Standard_E64_v5": 4.736000, + "Standard_E64a_v4": 4.736000, + "Standard_E64ads_v5": 4.974000, + "Standard_E64as_v4": 4.736000, + "Standard_E64as_v5": 4.302000, + "Standard_E64bds_v5": 6.320000, + "Standard_E64bs_v5": 5.648000, + "Standard_E64d_v4": 5.408000, + "Standard_E64d_v5": 5.408000, + "Standard_E64ds_v4": 5.408000, + "Standard_E64ds_v5": 5.408000, + "Standard_E64i_v3": 4.264000, + "Standard_E64is_v3": 4.264000, + "Standard_E64s_v3": 4.264000, + "Standard_E64s_v4": 4.736000, + "Standard_E64s_v5": 4.736000, + "Standard_E8-2ads_v5": 0.984000, + "Standard_E8-2as_v4": 0.592000, + "Standard_E8-2as_v5": 0.532000, + "Standard_E8-2ds_v4": 0.676000, + "Standard_E8-2ds_v5": 0.676000, + "Standard_E8-2s_v3": 0.624000, + "Standard_E8-2s_v4": 0.592000, + "Standard_E8-2s_v5": 0.592000, + "Standard_E8-4ads_v5": 0.984000, + "Standard_E8-4as_v4": 0.592000, + "Standard_E8-4as_v5": 0.532000, + "Standard_E8-4ds_v4": 0.676000, + "Standard_E8-4ds_v5": 0.676000, + "Standard_E8-4s_v3": 0.624000, + "Standard_E8-4s_v4": 0.592000, + "Standard_E8-4s_v5": 0.592000, + "Standard_E80ids_v4": 6.760000, + "Standard_E80is_v4": 5.920000, + "Standard_E8_v3": 0.624000, + "Standard_E8_v4": 0.592000, + "Standard_E8_v5": 0.592000, + "Standard_E8a_v4": 0.592000, + "Standard_E8ads_v5": 0.616000, + "Standard_E8as_v4": 0.592000, + "Standard_E8as_v5": 0.532000, + "Standard_E8bds_v5": 0.790000, + "Standard_E8bs_v5": 0.706000, + "Standard_E8d_v4": 0.676000, + "Standard_E8d_v5": 0.676000, + "Standard_E8ds_v4": 0.676000, + "Standard_E8ds_v5": 0.676000, + "Standard_E8pds_v5": 0.542000, + "Standard_E8ps_v5": 0.474000, + "Standard_E8s_v3": 0.624000, + "Standard_E8s_v4": 0.592000, + "Standard_E8s_v5": 0.592000, + "Standard_E96-24ads_v5": 11.808000, + "Standard_E96-24as_v4": 7.104000, + "Standard_E96-24as_v5": 6.384000, + "Standard_E96-24ds_v5": 8.112000, + "Standard_E96-24s_v5": 7.104000, + "Standard_E96-48ads_v5": 11.808000, + "Standard_E96-48as_v4": 7.104000, + "Standard_E96-48as_v5": 6.384000, + "Standard_E96-48ds_v5": 8.112000, + "Standard_E96-48s_v5": 7.104000, + "Standard_E96_v5": 7.104000, + "Standard_E96a_v4": 7.104000, + "Standard_E96ads_v5": 7.392000, + "Standard_E96as_v4": 7.104000, + "Standard_E96as_v5": 6.384000, + "Standard_E96bds_v5": 9.480000, + "Standard_E96bs_v5": 8.472000, + "Standard_E96d_v5": 8.112000, + "Standard_E96ds_v5": 8.112000, + "Standard_E96iads_v5": 8.131000, + "Standard_E96ias_v5": 7.022000, + "Standard_E96s_v5": 7.104000, + "Standard_EC16ads_v5": 1.232000, + "Standard_EC16as_v5": 1.064000, + "Standard_EC20ads_v5": 1.540000, + "Standard_EC20as_v5": 1.330000, + "Standard_EC2ads_v5": 0.154000, + "Standard_EC2as_v5": 0.133000, + "Standard_EC32ads_v5": 2.464000, + "Standard_EC32as_v5": 2.128000, + "Standard_EC48ads_v5": 3.696000, + "Standard_EC48as_v5": 3.192000, + "Standard_EC4ads_v5": 0.308000, + "Standard_EC4as_v5": 0.266000, + "Standard_EC64ads_v5": 4.928000, + "Standard_EC64as_v5": 4.256000, + "Standard_EC8ads_v5": 0.616000, + "Standard_EC8as_v5": 0.532000, + "Standard_EC96ads_v5": 7.392000, + "Standard_EC96as_v5": 6.384000, + "Standard_EC96iads_v5": 8.131000, + "Standard_EC96ias_v5": 7.022000, + "Standard_F1": 0.056800, + "Standard_F16": 0.909000, + "Standard_F16s": 0.909000, + "Standard_F16s_v2": 0.808000, + "Standard_F1s": 0.056800, + "Standard_F2": 0.114000, + "Standard_F2s": 0.114000, + "Standard_F2s_v2": 0.101000, + "Standard_F32s_v2": 1.616000, + "Standard_F4": 0.227000, + "Standard_F48s_v2": 2.424000, + "Standard_F4s": 0.227000, + "Standard_F4s_v2": 0.202000, + "Standard_F64s_v2": 3.232000, + "Standard_F72s_v2": 3.636000, + "Standard_F8": 0.454000, + "Standard_F8s": 0.454000, + "Standard_F8s_v2": 0.404000, + "Standard_FX12mds": 1.320000, + "Standard_FX24mds": 2.640000, + "Standard_FX36mds": 3.960000, + "Standard_FX48mds": 5.280000, + "Standard_FX4mds": 0.440000, + "Standard_G1": 0.439000, + "Standard_G2": 0.878000, + "Standard_G3": 1.757000, + "Standard_G4": 3.514000, + "Standard_G5": 7.027000, + "Standard_GS1": 0.439000, + "Standard_GS2": 0.878000, + "Standard_GS3": 1.757000, + "Standard_GS4": 3.514000, + "Standard_GS4-4": 3.514000, + "Standard_GS4-8": 3.514000, + "Standard_GS5": 7.027000, + "Standard_GS5-16": 7.027000, + "Standard_GS5-8": 7.027000, + "Standard_H16": 1.902000, + "Standard_H16m": 2.549000, + "Standard_H16mr": 2.803000, + "Standard_H16r": 2.092000, + "Standard_H8": 0.951000, + "Standard_H8m": 1.274000, + "Standard_HB120-16rs_v3": 4.500000, + "Standard_HB120-32rs_v3": 4.500000, + "Standard_HB120-64rs_v3": 4.500000, + "Standard_HB120-96rs_v3": 4.500000, + "Standard_HB120rs_v3": 4.500000, + "Standard_HC44-16rs": 3.960000, + "Standard_HC44-32rs": 3.960000, + "Standard_HC44rs": 3.960000, + "Standard_L112ias_v3": 11.150000, + "Standard_L16as_v3": 1.448000, + "Standard_L16s": 1.448000, + "Standard_L16s_v2": 1.448000, + "Standard_L16s_v3": 1.616000, + "Standard_L32as_v3": 2.896000, + "Standard_L32s": 2.895000, + "Standard_L32s_v2": 2.895000, + "Standard_L32s_v3": 3.232000, + "Standard_L48as_v3": 4.344000, + "Standard_L48s_v2": 4.343000, + "Standard_L48s_v3": 4.848000, + "Standard_L4s": 0.362000, + "Standard_L64as_v3": 5.792000, + "Standard_L64s_v2": 5.791000, + "Standard_L64s_v3": 6.464000, + "Standard_L80as_v3": 7.240000, + "Standard_L80s_v2": 7.238000, + "Standard_L80s_v3": 8.080000, + "Standard_L8as_v3": 0.724000, + "Standard_L8s": 0.724000, + "Standard_L8s_v2": 0.724000, + "Standard_L8s_v3": 0.808000, + "Standard_L96ias_v3": 9.557000, + "Standard_M128": 16.806000, + "Standard_M128-32ms": 33.624000, + "Standard_M128-64ms": 33.624000, + "Standard_M128dms_v2": 33.629000, + "Standard_M128ds_v2": 16.808000, + "Standard_M128m": 33.624000, + "Standard_M128ms": 33.624000, + "Standard_M128ms_v2": 33.099000, + "Standard_M128s": 16.806000, + "Standard_M128s_v2": 16.278000, + "Standard_M16-4ms": 3.872000, + "Standard_M16-8ms": 3.872000, + "Standard_M16ms": 3.872000, + "Standard_M16s": 3.007600, + "Standard_M192idms_v2": 40.400000, + "Standard_M192ids_v2": 20.200000, + "Standard_M192ims_v2": 39.870000, + "Standard_M192is_v2": 19.670000, + "Standard_M208-104ms_v2": 55.775000, + "Standard_M208-104s_v2": 27.888000, + "Standard_M208-52ms_v2": 55.775000, + "Standard_M208-52s_v2": 27.888000, + "Standard_M208ms_v2": 55.775000, + "Standard_M208s_v2": 27.888000, + "Standard_M32-16ms": 7.744000, + "Standard_M32-8ms": 7.744000, + "Standard_M32dms_v2": 7.744000, + "Standard_M32ls": 3.361000, + "Standard_M32ms": 7.744000, + "Standard_M32ms_v2": 7.611000, + "Standard_M32s": 4.202000, + "Standard_M32ts": 3.167000, + "Standard_M416-104ms_v2": 123.940000, + "Standard_M416-104s_v2": 61.975000, + "Standard_M416-208ms_v2": 123.940000, + "Standard_M416-208s_v2": 61.975000, + "Standard_M416is_v2": 61.975000, + "Standard_M416ms_v2": 123.940000, + "Standard_M416s_8_v2": 82.640000, + "Standard_M416s_v2": 61.975000, + "Standard_M64": 8.403000, + "Standard_M64-16ms": 13.024000, + "Standard_M64-32ms": 13.024000, + "Standard_M64dms_v2": 13.028000, + "Standard_M64ds_v2": 8.403000, + "Standard_M64ls": 6.335550, + "Standard_M64m": 13.024000, + "Standard_M64ms": 13.024000, + "Standard_M64ms_v2": 12.763000, + "Standard_M64s": 8.403000, + "Standard_M64s_v2": 8.138000, + "Standard_M8-2ms": 1.936000, + "Standard_M8-4ms": 1.936000, + "Standard_M8ms": 1.936000, + "Standard_NC12": 2.332000, + "Standard_NC12s_v3": 7.178000, + "Standard_NC16as_T4_v3": 1.409000, + "Standard_NC24": 4.664000, + "Standard_NC24ads_A100_v4": 4.591000, + "Standard_NC24r": 5.130000, + "Standard_NC24rs_v3": 15.792000, + "Standard_NC24s_v3": 14.356000, + "Standard_NC48ads_A100_v4": 9.183000, + "Standard_NC4as_T4_v3": 0.615000, + "Standard_NC6": 1.166000, + "Standard_NC64as_T4_v3": 5.092000, + "Standard_NC6s_v3": 3.589000, + "Standard_NC8as_T4_v3": 0.880000, + "Standard_NC96ads_A100_v4": 18.365000, + "Standard_ND96ams_A100_v4": 40.962000, + "Standard_ND96amsr_A100_v4": 40.962000, + "Standard_NV12": 3.003000, + "Standard_NV12ads_A10_v5": 1.135000, + "Standard_NV12s_v3": 1.425000, + "Standard_NV16as_v4": 1.165000, + "Standard_NV18ads_A10_v5": 2.000000, + "Standard_NV24": 6.006000, + "Standard_NV24s_v3": 2.850000, + "Standard_NV32as_v4": 2.330000, + "Standard_NV36adms_A10_v5": 5.650000, + "Standard_NV36ads_A10_v5": 4.000000, + "Standard_NV48s_v3": 5.700000, + "Standard_NV4as_v4": 0.291000, + "Standard_NV6": 1.502000, + "Standard_NV6ads_A10_v5": 0.567000, + "Standard_NV72ads_A10_v5": 8.150000, + "Standard_NV8as_v4": 0.582000, + } + // ukwest + initialOnDemandPrices["ukwest"] = map[string]float64{ + "Basic_A0": 0.020000, + "Basic_A1": 0.026000, + "Basic_A2": 0.101000, + "Basic_A3": 0.224000, + "Basic_A4": 0.449000, + "DCsv2 Type 1": 1.022000, + "Dadsv5_Type1": 7.677000, + "Dasv5_Type1": 6.410000, + "Ddsv4_Type 1": 4.813000, + "Ddsv4_Type2": 5.715000, + "Ddsv5_Type1": 7.219000, + "Dsv3_Type3": 4.885000, + "Dsv3_Type4": 6.106000, + "Dsv4_Type1": 5.111000, + "Dsv4_Type2": 6.389000, + "Dsv5_Type1": 6.389000, + "Eadsv5_Type1": 8.369000, + "Easv5_Type1": 7.219000, + "Ebdsv5-Type1": 7.113000, + "Ebsv5-Type1": 6.346000, + "Edsv4_Type 1": 6.133000, + "Edsv4_Type2": 7.283000, + "Edsv5_Type1": 9.200000, + "Esv3_Type3": 5.049000, + "Esv3_Type4": 6.841000, + "Esv4_Type1": 5.367000, + "Esv4_Type2": 7.044000, + "Esv5_Type1": 8.050000, + "Fsv2 Type3": 5.111000, + "Fsv2_Type2": 4.000000, + "Fsv2_Type4": 5.556000, + "Lasv3_Type1": 8.305000, + "Lsv3_Type1": 9.290000, + "Mdmsv2MedMem _Type1": 35.524000, + "Mdsv2MedMem_Type1": 17.756000, + "Msmv2MedMem Type1": 34.964000, + "Msmv2_Type1": 131.970000, + "Msv2MedMem Type1": 17.195000, + "Msv2_Type1": 65.991000, + "Standard_A0": 0.022000, + "Standard_A1": 0.066000, + "Standard_A1_v2": 0.050000, + "Standard_A2": 0.132000, + "Standard_A2_v2": 0.106000, + "Standard_A2m_v2": 0.158000, + "Standard_A3": 0.264000, + "Standard_A4": 0.528000, + "Standard_A4_v2": 0.222000, + "Standard_A4m_v2": 0.333000, + "Standard_A5": 0.297000, + "Standard_A6": 0.594000, + "Standard_A7": 1.188000, + "Standard_A8_v2": 0.467000, + "Standard_A8m_v2": 0.699000, + "Standard_B12ms": 0.564000, + "Standard_B16als_v2": 0.645000, + "Standard_B16as_v2": 0.802000, + "Standard_B16ls_v2": 0.713000, + "Standard_B16ms": 0.752000, + "Standard_B16s_v2": 0.805000, + "Standard_B1ls": 0.005880, + "Standard_B1ms": 0.023500, + "Standard_B1s": 0.011800, + "Standard_B20ms": 0.940000, + "Standard_B2als_v2": 0.054700, + "Standard_B2as_v2": 0.100000, + "Standard_B2ats_v2": 0.020600, + "Standard_B2ls_v2": 0.059500, + "Standard_B2ms": 0.094000, + "Standard_B2s": 0.047000, + "Standard_B2s_v2": 0.110000, + "Standard_B2ts_v2": 0.021800, + "Standard_B32als_v2": 1.289000, + "Standard_B32as_v2": 1.603000, + "Standard_B32ls_v2": 1.426000, + "Standard_B32s_v2": 1.611000, + "Standard_B4als_v2": 0.161000, + "Standard_B4as_v2": 0.182000, + "Standard_B4ls_v2": 0.197000, + "Standard_B4ms": 0.188000, + "Standard_B4s_v2": 0.220000, + "Standard_B8als_v2": 0.322000, + "Standard_B8as_v2": 0.401000, + "Standard_B8ls_v2": 0.357000, + "Standard_B8ms": 0.376000, + "Standard_B8s_v2": 0.439000, + "Standard_D11_v2": 0.234000, + "Standard_D11_v2_Promo": 0.234000, + "Standard_D12_v2": 0.469000, + "Standard_D12_v2_Promo": 0.469000, + "Standard_D13_v2": 0.937000, + "Standard_D13_v2_Promo": 0.937000, + "Standard_D14_v2": 1.874000, + "Standard_D14_v2_Promo": 1.874000, + "Standard_D15_v2": 2.343000, + "Standard_D15i_v2": 2.343000, + "Standard_D16_v3": 0.928000, + "Standard_D16_v4": 0.929000, + "Standard_D16_v5": 0.929000, + "Standard_D16a_v4": 0.929000, + "Standard_D16ads_v5": 1.733000, + "Standard_D16as_v4": 0.929000, + "Standard_D16as_v5": 0.832000, + "Standard_D16d_v4": 1.094000, + "Standard_D16d_v5": 1.094000, + "Standard_D16ds_v4": 1.094000, + "Standard_D16ds_v5": 1.094000, + "Standard_D16lds_v5": 0.929000, + "Standard_D16ls_v5": 0.823000, + "Standard_D16s_v3": 0.928000, + "Standard_D16s_v4": 0.929000, + "Standard_D16s_v5": 0.929000, + "Standard_D1_v2": 0.087700, + "Standard_D2_v2": 0.175000, + "Standard_D2_v2_Promo": 0.175000, + "Standard_D2_v3": 0.116000, + "Standard_D2_v4": 0.116000, + "Standard_D2_v5": 0.116000, + "Standard_D2a_v4": 0.116000, + "Standard_D2ads_v5": 0.125000, + "Standard_D2as_v4": 0.116000, + "Standard_D2as_v5": 0.196000, + "Standard_D2d_v4": 0.137000, + "Standard_D2d_v5": 0.137000, + "Standard_D2ds_v4": 0.137000, + "Standard_D2ds_v5": 0.137000, + "Standard_D2lds_v5": 0.116000, + "Standard_D2ls_v5": 0.103000, + "Standard_D2s_v3": 0.116000, + "Standard_D2s_v4": 0.116000, + "Standard_D2s_v5": 0.116000, + "Standard_D32-16s_v3": 1.856000, + "Standard_D32-8s_v3": 1.856000, + "Standard_D32_v3": 1.856000, + "Standard_D32_v4": 1.859000, + "Standard_D32_v5": 1.859000, + "Standard_D32a_v4": 1.859000, + "Standard_D32ads_v5": 3.466000, + "Standard_D32as_v4": 1.859000, + "Standard_D32as_v5": 3.137000, + "Standard_D32d_v4": 2.188000, + "Standard_D32d_v5": 2.188000, + "Standard_D32ds_v4": 2.188000, + "Standard_D32ds_v5": 2.188000, + "Standard_D32lds_v5": 1.859000, + "Standard_D32ls_v5": 1.646000, + "Standard_D32s_v3": 1.856000, + "Standard_D32s_v4": 1.859000, + "Standard_D32s_v5": 1.859000, + "Standard_D3_v2": 0.351000, + "Standard_D3_v2_Promo": 0.351000, + "Standard_D48_v3": 2.784000, + "Standard_D48_v4": 2.788000, + "Standard_D48_v5": 2.788000, + "Standard_D48a_v4": 2.788000, + "Standard_D48ads_v5": 5.199000, + "Standard_D48as_v4": 2.788000, + "Standard_D48as_v5": 4.705000, + "Standard_D48d_v4": 3.282000, + "Standard_D48d_v5": 3.282000, + "Standard_D48ds_v4": 3.282000, + "Standard_D48ds_v5": 3.282000, + "Standard_D48lds_v5": 2.788000, + "Standard_D48ls_v5": 2.468000, + "Standard_D48s_v3": 2.784000, + "Standard_D48s_v4": 2.788000, + "Standard_D48s_v5": 2.788000, + "Standard_D4_v2": 0.702000, + "Standard_D4_v2_Promo": 0.702000, + "Standard_D4_v3": 0.232000, + "Standard_D4_v4": 0.232000, + "Standard_D4_v5": 0.232000, + "Standard_D4a_v4": 0.232000, + "Standard_D4ads_v5": 0.249000, + "Standard_D4as_v4": 0.232000, + "Standard_D4as_v5": 0.392000, + "Standard_D4d_v4": 0.273000, + "Standard_D4d_v5": 0.273000, + "Standard_D4ds_v4": 0.273000, + "Standard_D4ds_v5": 0.273000, + "Standard_D4lds_v5": 0.232000, + "Standard_D4ls_v5": 0.206000, + "Standard_D4s_v3": 0.232000, + "Standard_D4s_v4": 0.232000, + "Standard_D4s_v5": 0.232000, + "Standard_D5_v2": 1.404000, + "Standard_D5_v2_Promo": 1.404000, + "Standard_D64-16s_v3": 3.712000, + "Standard_D64-32s_v3": 3.712000, + "Standard_D64_v3": 3.712000, + "Standard_D64_v4": 3.717000, + "Standard_D64_v5": 3.717000, + "Standard_D64a_v4": 3.717000, + "Standard_D64ads_v5": 6.932000, + "Standard_D64as_v4": 3.717000, + "Standard_D64as_v5": 6.274000, + "Standard_D64d_v4": 4.375000, + "Standard_D64d_v5": 4.375000, + "Standard_D64ds_v4": 4.375000, + "Standard_D64ds_v5": 4.375000, + "Standard_D64lds_v5": 3.717000, + "Standard_D64ls_v5": 3.291000, + "Standard_D64s_v3": 3.712000, + "Standard_D64s_v4": 3.717000, + "Standard_D64s_v5": 3.717000, + "Standard_D8_v3": 0.464000, + "Standard_D8_v4": 0.465000, + "Standard_D8_v5": 0.465000, + "Standard_D8a_v4": 0.465000, + "Standard_D8ads_v5": 0.867000, + "Standard_D8as_v4": 0.465000, + "Standard_D8as_v5": 0.416000, + "Standard_D8d_v4": 0.547000, + "Standard_D8d_v5": 0.547000, + "Standard_D8ds_v4": 0.547000, + "Standard_D8ds_v5": 0.547000, + "Standard_D8lds_v5": 0.465000, + "Standard_D8ls_v5": 0.411000, + "Standard_D8s_v3": 0.464000, + "Standard_D8s_v4": 0.465000, + "Standard_D8s_v5": 0.465000, + "Standard_D96_v5": 5.576000, + "Standard_D96a_v4": 5.576000, + "Standard_D96ads_v5": 10.398000, + "Standard_D96as_v4": 5.576000, + "Standard_D96as_v5": 4.995000, + "Standard_D96d_v5": 6.563000, + "Standard_D96ds_v5": 6.563000, + "Standard_D96lds_v5": 5.576000, + "Standard_D96ls_v5": 4.937000, + "Standard_D96s_v5": 5.576000, + "Standard_DC1s_v2": 0.116000, + "Standard_DC2s_v2": 0.232000, + "Standard_DC4s_v2": 0.465000, + "Standard_DC8_v2": 0.929000, + "Standard_DS11-1_v2": 0.234000, + "Standard_DS11_v2": 0.234000, + "Standard_DS11_v2_Promo": 0.234000, + "Standard_DS12-1_v2": 0.469000, + "Standard_DS12-2_v2": 0.469000, + "Standard_DS12_v2": 0.469000, + "Standard_DS12_v2_Promo": 0.469000, + "Standard_DS13-2_v2": 0.937000, + "Standard_DS13-4_v2": 0.937000, + "Standard_DS13_v2": 0.937000, + "Standard_DS13_v2_Promo": 0.937000, + "Standard_DS14-4_v2": 1.874000, + "Standard_DS14-8_v2": 1.874000, + "Standard_DS14_v2": 1.874000, + "Standard_DS14_v2_Promo": 1.874000, + "Standard_DS15_v2": 2.343000, + "Standard_DS15i_v2": 2.343000, + "Standard_DS1_v2": 0.087700, + "Standard_DS2_v2": 0.175000, + "Standard_DS2_v2_Promo": 0.175000, + "Standard_DS3_v2": 0.351000, + "Standard_DS3_v2_Promo": 0.351000, + "Standard_DS4_v2": 0.702000, + "Standard_DS4_v2_Promo": 0.702000, + "Standard_DS5_v2": 1.404000, + "Standard_DS5_v2_Promo": 1.404000, + "Standard_E104i_v5": 8.721000, + "Standard_E104id_v5": 9.967000, + "Standard_E104ids_v5": 9.967000, + "Standard_E104is_v5": 8.721000, + "Standard_E112iads_v5": 9.764000, + "Standard_E112ias_v5": 8.423000, + "Standard_E112ibds_v5": 12.448000, + "Standard_E112ibs_v5": 11.106000, + "Standard_E16-4ads_v5": 1.268000, + "Standard_E16-4as_v4": 1.220000, + "Standard_E16-4as_v5": 1.830000, + "Standard_E16-4ds_v4": 1.394000, + "Standard_E16-4ds_v5": 1.394000, + "Standard_E16-4s_v3": 1.248000, + "Standard_E16-4s_v4": 1.220000, + "Standard_E16-4s_v5": 1.220000, + "Standard_E16-8ads_v5": 2.004000, + "Standard_E16-8as_v4": 1.220000, + "Standard_E16-8as_v5": 1.094000, + "Standard_E16-8ds_v4": 1.394000, + "Standard_E16-8ds_v5": 1.394000, + "Standard_E16-8s_v3": 1.248000, + "Standard_E16-8s_v4": 1.220000, + "Standard_E16-8s_v5": 1.220000, + "Standard_E16_v3": 1.248000, + "Standard_E16_v4": 1.220000, + "Standard_E16_v5": 1.220000, + "Standard_E16a_v4": 1.220000, + "Standard_E16ads_v5": 1.314000, + "Standard_E16as_v4": 1.220000, + "Standard_E16as_v5": 1.140000, + "Standard_E16bds_v5": 1.617000, + "Standard_E16bs_v5": 1.442000, + "Standard_E16d_v4": 1.394000, + "Standard_E16d_v5": 1.394000, + "Standard_E16ds_v4": 1.394000, + "Standard_E16ds_v5": 1.394000, + "Standard_E16s_v3": 1.248000, + "Standard_E16s_v4": 1.220000, + "Standard_E16s_v5": 1.220000, + "Standard_E20_v3": 1.560000, + "Standard_E20_v4": 1.525000, + "Standard_E20_v5": 1.525000, + "Standard_E20a_v4": 1.525000, + "Standard_E20ads_v5": 1.585000, + "Standard_E20as_v4": 1.525000, + "Standard_E20as_v5": 1.367000, + "Standard_E20d_v4": 1.742000, + "Standard_E20d_v5": 1.742000, + "Standard_E20ds_v4": 1.742000, + "Standard_E20ds_v5": 1.742000, + "Standard_E20s_v3": 1.560000, + "Standard_E20s_v4": 1.525000, + "Standard_E20s_v5": 1.525000, + "Standard_E2_v3": 0.156000, + "Standard_E2_v4": 0.152000, + "Standard_E2_v5": 0.152000, + "Standard_E2a_v4": 0.152000, + "Standard_E2ads_v5": 0.159000, + "Standard_E2as_v4": 0.152000, + "Standard_E2as_v5": 0.183000, + "Standard_E2bds_v5": 0.202000, + "Standard_E2bs_v5": 0.180000, + "Standard_E2d_v4": 0.174000, + "Standard_E2d_v5": 0.174000, + "Standard_E2ds_v4": 0.174000, + "Standard_E2ds_v5": 0.174000, + "Standard_E2s_v3": 0.156000, + "Standard_E2s_v4": 0.152000, + "Standard_E2s_v5": 0.152000, + "Standard_E32-16ads_v5": 4.008000, + "Standard_E32-16as_v4": 2.439000, + "Standard_E32-16as_v5": 2.188000, + "Standard_E32-16ds_v4": 2.788000, + "Standard_E32-16ds_v5": 2.788000, + "Standard_E32-16s_v3": 2.496000, + "Standard_E32-16s_v4": 2.439000, + "Standard_E32-16s_v5": 2.439000, + "Standard_E32-8ads_v5": 2.536000, + "Standard_E32-8as_v4": 2.439000, + "Standard_E32-8as_v5": 3.660000, + "Standard_E32-8ds_v4": 2.788000, + "Standard_E32-8ds_v5": 2.788000, + "Standard_E32-8s_v3": 2.496000, + "Standard_E32-8s_v4": 2.439000, + "Standard_E32-8s_v5": 2.439000, + "Standard_E32_v3": 2.496000, + "Standard_E32_v4": 2.439000, + "Standard_E32_v5": 2.439000, + "Standard_E32a_v4": 2.439000, + "Standard_E32ads_v5": 2.536000, + "Standard_E32as_v4": 2.439000, + "Standard_E32as_v5": 2.188000, + "Standard_E32bds_v5": 3.233000, + "Standard_E32bs_v5": 2.885000, + "Standard_E32d_v4": 2.788000, + "Standard_E32d_v5": 2.788000, + "Standard_E32ds_v4": 2.788000, + "Standard_E32ds_v5": 2.788000, + "Standard_E32s_v3": 2.496000, + "Standard_E32s_v4": 2.439000, + "Standard_E32s_v5": 2.439000, + "Standard_E4-2ads_v5": 0.501000, + "Standard_E4-2as_v4": 0.305000, + "Standard_E4-2as_v5": 0.457000, + "Standard_E4-2ds_v4": 0.348000, + "Standard_E4-2ds_v5": 0.348000, + "Standard_E4-2s_v3": 0.312000, + "Standard_E4-2s_v4": 0.305000, + "Standard_E4-2s_v5": 0.305000, + "Standard_E48_v3": 3.553000, + "Standard_E48_v4": 3.659000, + "Standard_E48_v5": 3.659000, + "Standard_E48a_v4": 3.659000, + "Standard_E48ads_v5": 3.850000, + "Standard_E48as_v4": 3.659000, + "Standard_E48as_v5": 3.282000, + "Standard_E48bds_v5": 4.850000, + "Standard_E48bs_v5": 4.327000, + "Standard_E48d_v4": 4.182000, + "Standard_E48d_v5": 4.182000, + "Standard_E48ds_v4": 4.182000, + "Standard_E48ds_v5": 4.182000, + "Standard_E48s_v3": 3.553000, + "Standard_E48s_v4": 3.659000, + "Standard_E48s_v5": 3.659000, + "Standard_E4_v3": 0.312000, + "Standard_E4_v4": 0.305000, + "Standard_E4_v5": 0.305000, + "Standard_E4a_v4": 0.305000, + "Standard_E4ads_v5": 0.363000, + "Standard_E4as_v4": 0.305000, + "Standard_E4as_v5": 0.273000, + "Standard_E4bds_v5": 0.404000, + "Standard_E4bs_v5": 0.361000, + "Standard_E4d_v4": 0.348000, + "Standard_E4d_v5": 0.348000, + "Standard_E4ds_v4": 0.348000, + "Standard_E4ds_v5": 0.348000, + "Standard_E4s_v3": 0.312000, + "Standard_E4s_v4": 0.305000, + "Standard_E4s_v5": 0.305000, + "Standard_E64-16ads_v5": 5.072000, + "Standard_E64-16as_v4": 4.879000, + "Standard_E64-16as_v5": 7.319000, + "Standard_E64-16ds_v4": 5.576000, + "Standard_E64-16ds_v5": 5.576000, + "Standard_E64-16s_v3": 4.264000, + "Standard_E64-16s_v4": 4.879000, + "Standard_E64-16s_v5": 4.879000, + "Standard_E64-32ads_v5": 8.016000, + "Standard_E64-32as_v4": 4.879000, + "Standard_E64-32as_v5": 7.319000, + "Standard_E64-32ds_v4": 5.576000, + "Standard_E64-32ds_v5": 5.576000, + "Standard_E64-32s_v3": 4.264000, + "Standard_E64-32s_v4": 4.879000, + "Standard_E64-32s_v5": 4.879000, + "Standard_E64_v3": 4.264000, + "Standard_E64_v4": 4.879000, + "Standard_E64_v5": 4.879000, + "Standard_E64a_v4": 4.879000, + "Standard_E64ads_v5": 5.118000, + "Standard_E64as_v4": 4.879000, + "Standard_E64as_v5": 4.375000, + "Standard_E64bds_v5": 6.466000, + "Standard_E64bs_v5": 5.769000, + "Standard_E64d_v4": 5.576000, + "Standard_E64d_v5": 5.576000, + "Standard_E64ds_v4": 5.576000, + "Standard_E64ds_v5": 5.576000, + "Standard_E64i_v3": 4.264000, + "Standard_E64is_v3": 4.264000, + "Standard_E64s_v3": 4.264000, + "Standard_E64s_v4": 4.879000, + "Standard_E64s_v5": 4.879000, + "Standard_E8-2ads_v5": 1.002000, + "Standard_E8-2as_v4": 0.610000, + "Standard_E8-2as_v5": 0.915000, + "Standard_E8-2ds_v4": 0.697000, + "Standard_E8-2ds_v5": 0.697000, + "Standard_E8-2s_v3": 0.624000, + "Standard_E8-2s_v4": 0.610000, + "Standard_E8-2s_v5": 0.610000, + "Standard_E8-4ads_v5": 0.634000, + "Standard_E8-4as_v4": 0.610000, + "Standard_E8-4as_v5": 0.915000, + "Standard_E8-4ds_v4": 0.697000, + "Standard_E8-4ds_v5": 0.697000, + "Standard_E8-4s_v3": 0.624000, + "Standard_E8-4s_v4": 0.610000, + "Standard_E8-4s_v5": 0.610000, + "Standard_E80ids_v4": 6.970000, + "Standard_E80is_v4": 6.098000, + "Standard_E8_v3": 0.624000, + "Standard_E8_v4": 0.610000, + "Standard_E8_v5": 0.610000, + "Standard_E8a_v4": 0.610000, + "Standard_E8ads_v5": 0.680000, + "Standard_E8as_v4": 0.610000, + "Standard_E8as_v5": 0.547000, + "Standard_E8bds_v5": 0.808000, + "Standard_E8bs_v5": 0.721000, + "Standard_E8d_v4": 0.697000, + "Standard_E8d_v5": 0.697000, + "Standard_E8ds_v4": 0.697000, + "Standard_E8ds_v5": 0.697000, + "Standard_E8s_v3": 0.624000, + "Standard_E8s_v4": 0.610000, + "Standard_E8s_v5": 0.610000, + "Standard_E96-24ads_v5": 12.024000, + "Standard_E96-24as_v4": 7.318000, + "Standard_E96-24as_v5": 6.563000, + "Standard_E96-24ds_v5": 8.364000, + "Standard_E96-24s_v5": 7.318000, + "Standard_E96-48ads_v5": 7.608000, + "Standard_E96-48as_v4": 7.318000, + "Standard_E96-48as_v5": 10.979000, + "Standard_E96-48ds_v5": 8.364000, + "Standard_E96-48s_v5": 7.318000, + "Standard_E96_v5": 7.318000, + "Standard_E96a_v4": 7.318000, + "Standard_E96ads_v5": 7.654000, + "Standard_E96as_v4": 7.318000, + "Standard_E96as_v5": 6.563000, + "Standard_E96bds_v5": 9.699000, + "Standard_E96bs_v5": 8.654000, + "Standard_E96d_v5": 8.364000, + "Standard_E96ds_v5": 8.364000, + "Standard_E96iads_v5": 8.369000, + "Standard_E96ias_v4": 7.318000, + "Standard_E96ias_v5": 7.219000, + "Standard_E96s_v5": 7.318000, + "Standard_F1": 0.059000, + "Standard_F16": 0.950000, + "Standard_F16s": 0.950000, + "Standard_F16s_v2": 0.809000, + "Standard_F1s": 0.059000, + "Standard_F2": 0.119000, + "Standard_F2s": 0.119000, + "Standard_F2s_v2": 0.101000, + "Standard_F32s_v2": 1.618000, + "Standard_F4": 0.237000, + "Standard_F48s_v2": 2.428000, + "Standard_F4s": 0.237000, + "Standard_F4s_v2": 0.202000, + "Standard_F64s_v2": 3.237000, + "Standard_F72s_v2": 3.641000, + "Standard_F8": 0.475000, + "Standard_F8s": 0.475000, + "Standard_F8s_v2": 0.405000, + "Standard_L16as_v3": 1.510000, + "Standard_L16s_v3": 1.689000, + "Standard_L32as_v3": 3.020000, + "Standard_L32s_v3": 3.378000, + "Standard_L48as_v3": 4.530000, + "Standard_L48s_v3": 5.068000, + "Standard_L64as_v3": 6.040000, + "Standard_L64s_v3": 6.757000, + "Standard_L80as_v3": 7.550000, + "Standard_L80s_v3": 8.446000, + "Standard_L8as_v3": 0.755000, + "Standard_L8s_v3": 0.845000, + "Standard_M128": 16.806000, + "Standard_M128-32ms": 33.624000, + "Standard_M128-64ms": 33.624000, + "Standard_M128dms_v2": 32.295000, + "Standard_M128ds_v2": 16.141000, + "Standard_M128m": 33.624000, + "Standard_M128ms": 33.624000, + "Standard_M128ms_v2": 31.785000, + "Standard_M128s": 16.806000, + "Standard_M128s_v2": 15.632000, + "Standard_M16-4ms": 3.872000, + "Standard_M16-8ms": 3.872000, + "Standard_M16ms": 3.872000, + "Standard_M16s": 3.007600, + "Standard_M192idms_v2": 38.797000, + "Standard_M192ids_v2": 19.398000, + "Standard_M192ims_v2": 38.287000, + "Standard_M192is_v2": 18.888000, + "Standard_M208-104ms_v2": 53.990000, + "Standard_M208-104s_v2": 26.995000, + "Standard_M208-52ms_v2": 53.990000, + "Standard_M208-52s_v2": 26.995000, + "Standard_M208ms_v2": 53.990000, + "Standard_M208s_v2": 26.995000, + "Standard_M32-16ms": 7.744000, + "Standard_M32-8ms": 7.744000, + "Standard_M32dms_v2": 7.436000, + "Standard_M32ls": 3.361000, + "Standard_M32ms": 7.744000, + "Standard_M32ms_v2": 7.310000, + "Standard_M32s": 4.202000, + "Standard_M32ts": 3.167000, + "Standard_M416-104ms_v2": 119.970000, + "Standard_M416-104s_v2": 59.992000, + "Standard_M416-208ms_v2": 119.970000, + "Standard_M416-208s_v2": 59.992000, + "Standard_M416is_v2": 59.992000, + "Standard_M416ms_v2": 119.970000, + "Standard_M416s_8_v2": 86.600000, + "Standard_M416s_v2": 59.992000, + "Standard_M64": 8.403000, + "Standard_M64-16ms": 13.024000, + "Standard_M64-32ms": 13.024000, + "Standard_M64dms_v2": 12.511000, + "Standard_M64ds_v2": 8.069000, + "Standard_M64ls": 6.335550, + "Standard_M64m": 13.024000, + "Standard_M64ms": 13.024000, + "Standard_M64ms_v2": 12.257000, + "Standard_M64s": 8.403000, + "Standard_M64s_v2": 7.815000, + "Standard_M8-2ms": 1.936000, + "Standard_M8-4ms": 1.936000, + "Standard_M8ms": 1.936000, + "Standard_ND96ams_A100_v4": 42.929000, + "Standard_ND96amsr_A100_v4": 42.929000, + } + // westcentralus + initialOnDemandPrices["westcentralus"] = map[string]float64{ + "Basic_A0": 0.018000, + "Basic_A1": 0.025000, + "Basic_A2": 0.079000, + "Basic_A3": 0.176000, + "Basic_A4": 0.352000, + "Dadsv5_Type1": 7.614000, + "Dasv4_Type1": 6.083000, + "Dasv4_Type2": 6.083000, + "Dasv5_Type1": 6.357000, + "Ddsv4_Type 1": 4.773000, + "Ddsv4_Type2": 5.668000, + "Ddsv5_Type1": 7.160000, + "Dsv3_Type3": 5.069000, + "Dsv3_Type4": 6.336000, + "Dsv4_Type1": 5.069000, + "Dsv4_Type2": 6.336000, + "Dsv5_Type1": 6.336000, + "Eadsv5_Type1": 8.300000, + "Easv4_Type1": 7.983000, + "Easv4_Type2": 7.983000, + "Easv5_Type1": 7.160000, + "Ebdsv5-Type1": 7.054000, + "Ebsv5-Type1": 6.294000, + "Edsv4_Type 1": 6.083000, + "Edsv4_Type2": 7.223000, + "Edsv5_Type1": 9.124000, + "Esv3_Type3": 4.972000, + "Esv3_Type4": 6.736000, + "Esv4_Type1": 5.322000, + "Esv4_Type2": 6.985000, + "Esv5_Type1": 7.983000, + "Fsv2 Type3": 5.161000, + "Fsv2_Type2": 4.039000, + "Fsv2_Type4": 5.610000, + "Lsv3_Type1": 9.214000, + "Standard_A0": 0.020000, + "Standard_A1": 0.051000, + "Standard_A1_v2": 0.040000, + "Standard_A2": 0.101000, + "Standard_A2_v2": 0.083000, + "Standard_A2m_v2": 0.119000, + "Standard_A3": 0.202000, + "Standard_A4": 0.405000, + "Standard_A4_v2": 0.175000, + "Standard_A4m_v2": 0.249000, + "Standard_A5": 0.220000, + "Standard_A6": 0.440000, + "Standard_A7": 0.880000, + "Standard_A8_v2": 0.367000, + "Standard_A8m_v2": 0.524000, + "Standard_B12ms": 0.599000, + "Standard_B16als_v2": 0.713000, + "Standard_B16as_v2": 0.796000, + "Standard_B16ls_v2": 0.707000, + "Standard_B16ms": 0.799000, + "Standard_B16pls_v2": 0.645000, + "Standard_B16ps_v2": 0.719000, + "Standard_B16s_v2": 0.872000, + "Standard_B1ls": 0.006240, + "Standard_B1ms": 0.025000, + "Standard_B1s": 0.012500, + "Standard_B20ms": 0.998000, + "Standard_B2als_v2": 0.045100, + "Standard_B2as_v2": 0.090200, + "Standard_B2ats_v2": 0.020500, + "Standard_B2ls_v2": 0.049900, + "Standard_B2ms": 0.099800, + "Standard_B2pls_v2": 0.049500, + "Standard_B2ps_v2": 0.089800, + "Standard_B2pts_v2": 0.010100, + "Standard_B2s": 0.049900, + "Standard_B2s_v2": 0.099800, + "Standard_B2ts_v2": 0.021700, + "Standard_B32als_v2": 1.426000, + "Standard_B32as_v2": 1.444000, + "Standard_B32ls_v2": 1.415000, + "Standard_B32s_v2": 1.745000, + "Standard_B4als_v2": 0.160000, + "Standard_B4as_v2": 0.180000, + "Standard_B4ls_v2": 0.195000, + "Standard_B4ms": 0.200000, + "Standard_B4pls_v2": 0.161000, + "Standard_B4ps_v2": 0.161000, + "Standard_B4s_v2": 0.218000, + "Standard_B8als_v2": 0.357000, + "Standard_B8as_v2": 0.361000, + "Standard_B8ls_v2": 0.390000, + "Standard_B8ms": 0.399000, + "Standard_B8pls_v2": 0.286000, + "Standard_B8ps_v2": 0.323000, + "Standard_B8s_v2": 0.399000, + "Standard_D11_v2": 0.166000, + "Standard_D11_v2_Promo": 0.166000, + "Standard_D12_v2": 0.332000, + "Standard_D12_v2_Promo": 0.332000, + "Standard_D13_v2": 0.665000, + "Standard_D13_v2_Promo": 0.665000, + "Standard_D14_v2": 1.330000, + "Standard_D14_v2_Promo": 1.330000, + "Standard_D15_v2": 1.662000, + "Standard_D15i_v2": 1.662000, + "Standard_D16_v3": 0.880000, + "Standard_D16_v4": 0.922000, + "Standard_D16_v5": 0.922000, + "Standard_D16a_v4": 0.922000, + "Standard_D16ads_v5": 0.989000, + "Standard_D16as_v4": 0.922000, + "Standard_D16as_v5": 0.826000, + "Standard_D16as_v5_Promo": 0.739000, + "Standard_D16d_v4": 1.085000, + "Standard_D16d_v5": 1.085000, + "Standard_D16ds_v4": 1.085000, + "Standard_D16ds_v5": 1.085000, + "Standard_D16lds_v5": 0.922000, + "Standard_D16ls_v5": 0.816000, + "Standard_D16pds_v5": 0.868000, + "Standard_D16plds_v5": 0.737000, + "Standard_D16pls_v5": 0.653000, + "Standard_D16ps_v5": 0.739000, + "Standard_D16s_v3": 0.880000, + "Standard_D16s_v4": 0.922000, + "Standard_D16s_v5": 0.922000, + "Standard_D1_v2": 0.063600, + "Standard_D2_v2": 0.127000, + "Standard_D2_v2_Promo": 0.127000, + "Standard_D2_v3": 0.110000, + "Standard_D2_v4": 0.115000, + "Standard_D2_v5": 0.115000, + "Standard_D2a_v4": 0.115000, + "Standard_D2ads_v5": 0.216000, + "Standard_D2as_v4": 0.115000, + "Standard_D2as_v5": 0.103000, + "Standard_D2as_v5_Promo": 0.092400, + "Standard_D2d_v4": 0.136000, + "Standard_D2d_v5": 0.136000, + "Standard_D2ds_v4": 0.136000, + "Standard_D2ds_v5": 0.136000, + "Standard_D2lds_v5": 0.115000, + "Standard_D2ls_v5": 0.102000, + "Standard_D2pds_v5": 0.108000, + "Standard_D2plds_v5": 0.092200, + "Standard_D2pls_v5": 0.081600, + "Standard_D2ps_v5": 0.092400, + "Standard_D2s_v3": 0.110000, + "Standard_D2s_v4": 0.115000, + "Standard_D2s_v5": 0.115000, + "Standard_D32-16s_v3": 1.760000, + "Standard_D32-8s_v3": 1.760000, + "Standard_D32_v3": 1.760000, + "Standard_D32_v4": 1.843000, + "Standard_D32_v5": 1.843000, + "Standard_D32a_v4": 1.843000, + "Standard_D32ads_v5": 3.450000, + "Standard_D32as_v4": 1.843000, + "Standard_D32as_v5": 1.651000, + "Standard_D32as_v5_Promo": 1.478000, + "Standard_D32d_v4": 2.170000, + "Standard_D32d_v5": 2.170000, + "Standard_D32ds_v4": 2.170000, + "Standard_D32ds_v5": 2.170000, + "Standard_D32lds_v5": 1.843000, + "Standard_D32ls_v5": 1.632000, + "Standard_D32pds_v5": 1.736000, + "Standard_D32plds_v5": 1.475000, + "Standard_D32pls_v5": 1.306000, + "Standard_D32ps_v5": 1.478000, + "Standard_D32s_v3": 1.760000, + "Standard_D32s_v4": 1.843000, + "Standard_D32s_v5": 1.843000, + "Standard_D3_v2": 0.254000, + "Standard_D3_v2_Promo": 0.254000, + "Standard_D48_v3": 2.640000, + "Standard_D48_v4": 2.765000, + "Standard_D48_v5": 2.765000, + "Standard_D48a_v4": 2.765000, + "Standard_D48ads_v5": 2.966000, + "Standard_D48as_v4": 2.765000, + "Standard_D48as_v5": 2.477000, + "Standard_D48as_v5_Promo": 2.218000, + "Standard_D48d_v4": 3.254000, + "Standard_D48d_v5": 3.254000, + "Standard_D48ds_v4": 3.254000, + "Standard_D48ds_v5": 3.254000, + "Standard_D48lds_v5": 2.765000, + "Standard_D48ls_v5": 2.448000, + "Standard_D48pds_v5": 2.604000, + "Standard_D48plds_v5": 2.212000, + "Standard_D48pls_v5": 1.958000, + "Standard_D48ps_v5": 2.218000, + "Standard_D48s_v3": 2.640000, + "Standard_D48s_v4": 2.765000, + "Standard_D48s_v5": 2.765000, + "Standard_D4_v2": 0.509000, + "Standard_D4_v2_Promo": 0.509000, + "Standard_D4_v3": 0.220000, + "Standard_D4_v4": 0.230000, + "Standard_D4_v5": 0.230000, + "Standard_D4a_v4": 0.230000, + "Standard_D4ads_v5": 0.431000, + "Standard_D4as_v4": 0.230000, + "Standard_D4as_v5": 0.390000, + "Standard_D4as_v5_Promo": 0.185000, + "Standard_D4d_v4": 0.271000, + "Standard_D4d_v5": 0.271000, + "Standard_D4ds_v4": 0.271000, + "Standard_D4ds_v5": 0.271000, + "Standard_D4lds_v5": 0.230000, + "Standard_D4ls_v5": 0.204000, + "Standard_D4pds_v5": 0.217000, + "Standard_D4plds_v5": 0.184000, + "Standard_D4pls_v5": 0.163000, + "Standard_D4ps_v5": 0.185000, + "Standard_D4s_v3": 0.220000, + "Standard_D4s_v4": 0.230000, + "Standard_D4s_v5": 0.230000, + "Standard_D5_v2": 1.017000, + "Standard_D5_v2_Promo": 1.017000, + "Standard_D64-16s_v3": 3.520000, + "Standard_D64-32s_v3": 3.520000, + "Standard_D64_v3": 3.520000, + "Standard_D64_v4": 3.686000, + "Standard_D64_v5": 3.686000, + "Standard_D64a_v4": 3.686000, + "Standard_D64ads_v5": 3.955000, + "Standard_D64as_v4": 3.686000, + "Standard_D64as_v5": 3.302000, + "Standard_D64as_v5_Promo": 2.957000, + "Standard_D64d_v4": 4.339000, + "Standard_D64d_v5": 4.339000, + "Standard_D64ds_v4": 4.339000, + "Standard_D64ds_v5": 4.339000, + "Standard_D64lds_v5": 3.686000, + "Standard_D64ls_v5": 3.264000, + "Standard_D64pds_v5": 3.471000, + "Standard_D64plds_v5": 2.949000, + "Standard_D64pls_v5": 2.611000, + "Standard_D64ps_v5": 2.957000, + "Standard_D64s_v3": 3.520000, + "Standard_D64s_v4": 3.686000, + "Standard_D64s_v5": 3.686000, + "Standard_D8_v3": 0.440000, + "Standard_D8_v4": 0.461000, + "Standard_D8_v5": 0.461000, + "Standard_D8a_v4": 0.461000, + "Standard_D8ads_v5": 0.862000, + "Standard_D8as_v4": 0.461000, + "Standard_D8as_v5": 0.781000, + "Standard_D8as_v5_Promo": 0.370000, + "Standard_D8d_v4": 0.542000, + "Standard_D8d_v5": 0.542000, + "Standard_D8ds_v4": 0.542000, + "Standard_D8ds_v5": 0.542000, + "Standard_D8lds_v5": 0.461000, + "Standard_D8ls_v5": 0.408000, + "Standard_D8pds_v5": 0.434000, + "Standard_D8plds_v5": 0.369000, + "Standard_D8pls_v5": 0.326000, + "Standard_D8ps_v5": 0.370000, + "Standard_D8s_v3": 0.440000, + "Standard_D8s_v4": 0.461000, + "Standard_D8s_v5": 0.461000, + "Standard_D96_v5": 5.530000, + "Standard_D96a_v4": 5.530000, + "Standard_D96ads_v5": 10.349000, + "Standard_D96as_v4": 5.530000, + "Standard_D96as_v5": 9.370000, + "Standard_D96as_v5_Promo": 4.435000, + "Standard_D96d_v5": 6.509000, + "Standard_D96ds_v5": 6.509000, + "Standard_D96lds_v5": 5.530000, + "Standard_D96ls_v5": 4.896000, + "Standard_D96s_v5": 5.530000, + "Standard_DS11-1_v2": 0.166000, + "Standard_DS11_v2": 0.166000, + "Standard_DS11_v2_Promo": 0.166000, + "Standard_DS12-1_v2": 0.332000, + "Standard_DS12-2_v2": 0.332000, + "Standard_DS12_v2": 0.332000, + "Standard_DS12_v2_Promo": 0.332000, + "Standard_DS13-2_v2": 0.665000, + "Standard_DS13-4_v2": 0.665000, + "Standard_DS13_v2": 0.665000, + "Standard_DS13_v2_Promo": 0.665000, + "Standard_DS14-4_v2": 1.330000, + "Standard_DS14-8_v2": 1.330000, + "Standard_DS14_v2": 1.330000, + "Standard_DS14_v2_Promo": 1.330000, + "Standard_DS15_v2": 1.662000, + "Standard_DS15i_v2": 1.662000, + "Standard_DS1_v2": 0.063600, + "Standard_DS2_v2": 0.127000, + "Standard_DS2_v2_Promo": 0.127000, + "Standard_DS3_v2": 0.254000, + "Standard_DS3_v2_Promo": 0.254000, + "Standard_DS4_v2": 0.509000, + "Standard_DS4_v2_Promo": 0.509000, + "Standard_DS5_v2": 1.017000, + "Standard_DS5_v2_Promo": 1.017000, + "Standard_E104i_v5": 8.649000, + "Standard_E104id_v5": 9.884000, + "Standard_E104ids_v5": 9.884000, + "Standard_E104is_v5": 8.649000, + "Standard_E112iads_v5": 9.684000, + "Standard_E112ias_v5": 8.353000, + "Standard_E112ibds_v5": 12.345000, + "Standard_E112ibs_v5": 11.014000, + "Standard_E16-4ads_v5": 1.258000, + "Standard_E16-4as_v4": 1.210000, + "Standard_E16-4as_v5": 1.821000, + "Standard_E16-4ds_v4": 1.382000, + "Standard_E16-4ds_v5": 1.382000, + "Standard_E16-4s_v3": 1.166000, + "Standard_E16-4s_v4": 1.210000, + "Standard_E16-4s_v5": 1.210000, + "Standard_E16-8ads_v5": 1.258000, + "Standard_E16-8as_v4": 1.210000, + "Standard_E16-8as_v5": 1.821000, + "Standard_E16-8ds_v4": 1.382000, + "Standard_E16-8ds_v5": 1.382000, + "Standard_E16-8s_v3": 1.166000, + "Standard_E16-8s_v4": 1.210000, + "Standard_E16-8s_v5": 1.210000, + "Standard_E16_v3": 1.166000, + "Standard_E16_v4": 1.210000, + "Standard_E16_v5": 1.210000, + "Standard_E16a_v4": 1.210000, + "Standard_E16ads_v5": 1.258000, + "Standard_E16as_v4": 1.210000, + "Standard_E16as_v5": 1.131000, + "Standard_E16bds_v5": 1.603000, + "Standard_E16bs_v5": 1.430000, + "Standard_E16d_v4": 1.382000, + "Standard_E16d_v5": 1.382000, + "Standard_E16ds_v4": 1.382000, + "Standard_E16ds_v5": 1.382000, + "Standard_E16pds_v5": 1.106000, + "Standard_E16ps_v5": 0.968000, + "Standard_E16s_v3": 1.166000, + "Standard_E16s_v4": 1.210000, + "Standard_E16s_v5": 1.210000, + "Standard_E20_v3": 1.460000, + "Standard_E20_v4": 1.512000, + "Standard_E20_v5": 1.512000, + "Standard_E20a_v4": 1.512000, + "Standard_E20ads_v5": 1.618000, + "Standard_E20as_v4": 1.512000, + "Standard_E20as_v5": 1.356000, + "Standard_E20d_v4": 1.728000, + "Standard_E20d_v5": 1.728000, + "Standard_E20ds_v4": 1.728000, + "Standard_E20ds_v5": 1.728000, + "Standard_E20pds_v5": 1.382000, + "Standard_E20ps_v5": 1.210000, + "Standard_E20s_v3": 1.460000, + "Standard_E20s_v4": 1.512000, + "Standard_E20s_v5": 1.512000, + "Standard_E2_v3": 0.146000, + "Standard_E2_v4": 0.151000, + "Standard_E2_v5": 0.151000, + "Standard_E2a_v4": 0.151000, + "Standard_E2ads_v5": 0.157000, + "Standard_E2as_v4": 0.151000, + "Standard_E2as_v5": 0.182000, + "Standard_E2bds_v5": 0.200000, + "Standard_E2bs_v5": 0.179000, + "Standard_E2d_v4": 0.173000, + "Standard_E2d_v5": 0.173000, + "Standard_E2ds_v4": 0.173000, + "Standard_E2ds_v5": 0.173000, + "Standard_E2pds_v5": 0.138000, + "Standard_E2ps_v5": 0.121000, + "Standard_E2s_v3": 0.146000, + "Standard_E2s_v4": 0.151000, + "Standard_E2s_v5": 0.151000, + "Standard_E32-16ads_v5": 2.515000, + "Standard_E32-16as_v4": 2.419000, + "Standard_E32-16as_v5": 2.170000, + "Standard_E32-16ds_v4": 2.765000, + "Standard_E32-16ds_v5": 2.765000, + "Standard_E32-16s_v3": 2.333000, + "Standard_E32-16s_v4": 2.419000, + "Standard_E32-16s_v5": 2.419000, + "Standard_E32-8ads_v5": 3.987000, + "Standard_E32-8as_v4": 2.419000, + "Standard_E32-8as_v5": 3.642000, + "Standard_E32-8ds_v4": 2.765000, + "Standard_E32-8ds_v5": 2.765000, + "Standard_E32-8s_v3": 2.333000, + "Standard_E32-8s_v4": 2.419000, + "Standard_E32-8s_v5": 2.419000, + "Standard_E32_v3": 2.333000, + "Standard_E32_v4": 2.419000, + "Standard_E32_v5": 2.419000, + "Standard_E32a_v4": 2.419000, + "Standard_E32ads_v5": 2.561000, + "Standard_E32as_v4": 2.419000, + "Standard_E32as_v5": 2.170000, + "Standard_E32bds_v5": 3.206000, + "Standard_E32bs_v5": 2.861000, + "Standard_E32d_v4": 2.765000, + "Standard_E32d_v5": 2.765000, + "Standard_E32ds_v4": 2.765000, + "Standard_E32ds_v5": 2.765000, + "Standard_E32pds_v5": 2.212000, + "Standard_E32ps_v5": 1.935000, + "Standard_E32s_v3": 2.333000, + "Standard_E32s_v4": 2.419000, + "Standard_E32s_v5": 2.419000, + "Standard_E4-2ads_v5": 0.498000, + "Standard_E4-2as_v4": 0.302000, + "Standard_E4-2as_v5": 0.455000, + "Standard_E4-2ds_v4": 0.346000, + "Standard_E4-2ds_v5": 0.346000, + "Standard_E4-2s_v3": 0.292000, + "Standard_E4-2s_v4": 0.302000, + "Standard_E4-2s_v5": 0.302000, + "Standard_E48_v3": 3.499000, + "Standard_E48_v4": 3.629000, + "Standard_E48_v5": 3.629000, + "Standard_E48a_v4": 3.629000, + "Standard_E48ads_v5": 3.773000, + "Standard_E48as_v4": 3.629000, + "Standard_E48as_v5": 3.300000, + "Standard_E48bds_v5": 4.810000, + "Standard_E48bs_v5": 4.291000, + "Standard_E48d_v4": 4.147000, + "Standard_E48d_v5": 4.147000, + "Standard_E48ds_v4": 4.147000, + "Standard_E48ds_v5": 4.147000, + "Standard_E48s_v4": 3.629000, + "Standard_E48s_v5": 3.629000, + "Standard_E4_v3": 0.292000, + "Standard_E4_v4": 0.302000, + "Standard_E4_v5": 0.302000, + "Standard_E4a_v4": 0.302000, + "Standard_E4ads_v5": 0.314000, + "Standard_E4as_v4": 0.302000, + "Standard_E4as_v5": 0.271000, + "Standard_E4bds_v5": 0.401000, + "Standard_E4bs_v5": 0.358000, + "Standard_E4d_v4": 0.346000, + "Standard_E4d_v5": 0.346000, + "Standard_E4ds_v4": 0.346000, + "Standard_E4ds_v5": 0.346000, + "Standard_E4pds_v5": 0.276000, + "Standard_E4ps_v5": 0.242000, + "Standard_E4s_v3": 0.292000, + "Standard_E4s_v4": 0.302000, + "Standard_E4s_v5": 0.302000, + "Standard_E64-16ads_v5": 7.974000, + "Standard_E64-16as_v4": 4.838000, + "Standard_E64-16as_v5": 7.283000, + "Standard_E64-16ds_v4": 5.530000, + "Standard_E64-16ds_v5": 5.530000, + "Standard_E64-16s_v3": 4.199000, + "Standard_E64-16s_v4": 4.838000, + "Standard_E64-16s_v5": 4.838000, + "Standard_E64-32ads_v5": 7.974000, + "Standard_E64-32as_v4": 4.838000, + "Standard_E64-32as_v5": 4.339000, + "Standard_E64-32ds_v4": 5.530000, + "Standard_E64-32ds_v5": 5.530000, + "Standard_E64-32s_v3": 4.199000, + "Standard_E64-32s_v4": 4.838000, + "Standard_E64-32s_v5": 4.838000, + "Standard_E64_v3": 4.199000, + "Standard_E64_v4": 4.838000, + "Standard_E64_v5": 4.838000, + "Standard_E64a_v4": 4.838000, + "Standard_E64ads_v5": 5.030000, + "Standard_E64as_v4": 4.838000, + "Standard_E64as_v5": 4.339000, + "Standard_E64bds_v5": 6.413000, + "Standard_E64bs_v5": 5.722000, + "Standard_E64d_v4": 5.530000, + "Standard_E64d_v5": 5.530000, + "Standard_E64ds_v4": 5.530000, + "Standard_E64ds_v5": 5.530000, + "Standard_E64i_v3": 4.199000, + "Standard_E64is_v3": 4.199000, + "Standard_E64s_v3": 4.199000, + "Standard_E64s_v4": 4.838000, + "Standard_E64s_v5": 4.838000, + "Standard_E8-2ads_v5": 0.997000, + "Standard_E8-2as_v4": 0.605000, + "Standard_E8-2as_v5": 0.910000, + "Standard_E8-2ds_v4": 0.691000, + "Standard_E8-2ds_v5": 0.691000, + "Standard_E8-2s_v3": 0.583000, + "Standard_E8-2s_v4": 0.605000, + "Standard_E8-2s_v5": 0.605000, + "Standard_E8-4ads_v5": 0.997000, + "Standard_E8-4as_v4": 0.605000, + "Standard_E8-4as_v5": 0.910000, + "Standard_E8-4ds_v4": 0.691000, + "Standard_E8-4ds_v5": 0.691000, + "Standard_E8-4s_v3": 0.583000, + "Standard_E8-4s_v4": 0.605000, + "Standard_E8-4s_v5": 0.605000, + "Standard_E80ids_v4": 6.912000, + "Standard_E80is_v4": 6.048000, + "Standard_E8_v3": 0.583000, + "Standard_E8_v4": 0.605000, + "Standard_E8_v5": 0.605000, + "Standard_E8a_v4": 0.605000, + "Standard_E8ads_v5": 0.629000, + "Standard_E8as_v4": 0.605000, + "Standard_E8as_v5": 0.588000, + "Standard_E8bds_v5": 0.802000, + "Standard_E8bs_v5": 0.715000, + "Standard_E8d_v4": 0.691000, + "Standard_E8d_v5": 0.691000, + "Standard_E8ds_v4": 0.691000, + "Standard_E8ds_v5": 0.691000, + "Standard_E8pds_v5": 0.553000, + "Standard_E8ps_v5": 0.484000, + "Standard_E8s_v3": 0.583000, + "Standard_E8s_v4": 0.605000, + "Standard_E8s_v5": 0.605000, + "Standard_E96-24ads_v5": 11.962000, + "Standard_E96-24as_v4": 7.258000, + "Standard_E96-24as_v5": 6.509000, + "Standard_E96-24ds_v5": 8.294000, + "Standard_E96-24s_v5": 7.258000, + "Standard_E96-48ads_v5": 7.546000, + "Standard_E96-48as_v4": 7.258000, + "Standard_E96-48as_v5": 10.925000, + "Standard_E96-48ds_v5": 8.294000, + "Standard_E96-48s_v5": 7.258000, + "Standard_E96_v5": 7.258000, + "Standard_E96a_v4": 7.258000, + "Standard_E96ads_v5": 7.592000, + "Standard_E96as_v4": 7.258000, + "Standard_E96as_v5": 6.555000, + "Standard_E96bds_v5": 9.619000, + "Standard_E96bs_v5": 8.582000, + "Standard_E96d_v5": 8.294000, + "Standard_E96ds_v5": 8.294000, + "Standard_E96iads_v5": 8.300000, + "Standard_E96ias_v5": 7.160000, + "Standard_E96s_v5": 7.258000, + "Standard_F1": 0.054700, + "Standard_F16": 0.875000, + "Standard_F16s": 0.875000, + "Standard_F16s_v2": 0.816000, + "Standard_F1s": 0.054700, + "Standard_F2": 0.109000, + "Standard_F2s": 0.109000, + "Standard_F2s_v2": 0.102000, + "Standard_F32s_v2": 1.632000, + "Standard_F4": 0.219000, + "Standard_F48s_v2": 2.448000, + "Standard_F4s": 0.219000, + "Standard_F4s_v2": 0.204000, + "Standard_F64s_v2": 3.264000, + "Standard_F72s_v2": 3.672000, + "Standard_F8": 0.438000, + "Standard_F8s": 0.438000, + "Standard_F8s_v2": 0.408000, + "Standard_L16s_v3": 1.675000, + "Standard_L32s_v3": 3.350000, + "Standard_L48s_v3": 5.026000, + "Standard_L64s_v3": 6.701000, + "Standard_L80s_v3": 8.376000, + "Standard_L8s_v3": 0.838000, + } + // westeurope + initialOnDemandPrices["westeurope"] = map[string]float64{ + "Basic_A0": 0.018000, + "Basic_A1": 0.027000, + "Basic_A2": 0.078000, + "Basic_A3": 0.204000, + "Basic_A4": 0.408000, + "DCadsv5 Type 1": 7.700000, + "DCasv5 Type 1": 6.406000, + "DCdsv3 Type1": 7.181000, + "DCsv2 Type 1": 1.012000, + "DCsv3 Type1": 6.072000, + "Dadsv5_Type1": 7.700000, + "Dasv4_Type1": 6.101000, + "Dasv4_Type2": 6.101000, + "Dasv5_Type1": 6.406000, + "Ddsv4_Type 1": 4.787000, + "Ddsv4_Type2": 5.685000, + "Ddsv5_Type1": 7.181000, + "Dsv3_Type3": 5.060000, + "Dsv3_Type4": 6.325000, + "Dsv4_Type1": 5.060000, + "Dsv4_Type2": 6.325000, + "Dsv5_Type1": 6.325000, + "ECadsv5 Type 1": 8.596000, + "ECasv5 Type 1": 7.957000, + "Eadsv5_Type1": 8.342000, + "Easv4_Type1": 7.993000, + "Easv4_Type2": 7.993000, + "Easv5_Type1": 7.234000, + "Ebdsv5-Type1": 7.005000, + "Ebsv5-Type1": 6.266000, + "Edsv4_Type 1": 6.090000, + "Edsv4_Type2": 7.231000, + "Edsv5_Type1": 9.134000, + "Esv3_Type3": 5.162000, + "Esv3_Type4": 6.994000, + "Esv4_Type1": 5.351000, + "Esv4_Type2": 7.023000, + "Esv5_Type1": 8.026000, + "FXmds Type1": 5.940000, + "Fsv2 Type3": 4.908000, + "Fsv2_Type2": 3.841000, + "Fsv2_Type4": 5.334000, + "Lasv3_Type1": 8.184000, + "Lsv2_Type1": 8.184000, + "Lsv3_Type1": 9.152000, + "Mdmsv2MedMem _Type1": 41.103000, + "Mdsv2MedMem_Type1": 20.544000, + "Ms_Type1": 20.536000, + "Msm_Type1": 41.108000, + "Msmv2MedMem Type1": 40.455000, + "Msmv2_Type1": 141.780000, + "Msv2MedMem Type1": 19.896000, + "Msv2_Type1": 70.899000, + "NDamsrA100v4_Type1": 45.059000, + "NDasrA100v4_Type1": 38.893000, + "NVasv4_Type1": 8.970000, + "NVsv3_Type1": 6.270000, + "Standard_A0": 0.020000, + "Standard_A1": 0.060000, + "Standard_A1_v2": 0.041000, + "Standard_A2": 0.120000, + "Standard_A2_v2": 0.087000, + "Standard_A2m_v2": 0.124000, + "Standard_A3": 0.240000, + "Standard_A4": 0.480000, + "Standard_A4_v2": 0.183000, + "Standard_A4m_v2": 0.260000, + "Standard_A5": 0.270000, + "Standard_A6": 0.540000, + "Standard_A7": 1.080000, + "Standard_A8_v2": 0.383000, + "Standard_A8m_v2": 0.546000, + "Standard_B12ms": 0.576000, + "Standard_B16als_v2": 0.612000, + "Standard_B16as_v2": 0.691000, + "Standard_B16ls_v2": 0.754000, + "Standard_B16ms": 0.768000, + "Standard_B16pls_v2": 0.618000, + "Standard_B16ps_v2": 0.614000, + "Standard_B16s_v2": 0.768000, + "Standard_B1ls": 0.006000, + "Standard_B1ms": 0.024000, + "Standard_B1s": 0.012000, + "Standard_B20ms": 0.960000, + "Standard_B2als_v2": 0.043200, + "Standard_B2as_v2": 0.086400, + "Standard_B2ats_v2": 0.010800, + "Standard_B2ls_v2": 0.057200, + "Standard_B2ms": 0.096000, + "Standard_B2pls_v2": 0.047600, + "Standard_B2ps_v2": 0.086000, + "Standard_B2pts_v2": 0.018800, + "Standard_B2s": 0.048000, + "Standard_B2s_v2": 0.096000, + "Standard_B2ts_v2": 0.012000, + "Standard_B32als_v2": 1.371000, + "Standard_B32as_v2": 1.530000, + "Standard_B32ls_v2": 1.507000, + "Standard_B32s_v2": 1.683000, + "Standard_B4als_v2": 0.153000, + "Standard_B4as_v2": 0.173000, + "Standard_B4ls_v2": 0.170000, + "Standard_B4ms": 0.192000, + "Standard_B4pls_v2": 0.154000, + "Standard_B4ps_v2": 0.154000, + "Standard_B4s_v2": 0.192000, + "Standard_B8als_v2": 0.343000, + "Standard_B8as_v2": 0.382000, + "Standard_B8ls_v2": 0.340000, + "Standard_B8ms": 0.384000, + "Standard_B8pls_v2": 0.272000, + "Standard_B8ps_v2": 0.344000, + "Standard_B8s_v2": 0.421000, + "Standard_D1": 0.084000, + "Standard_D11": 0.221000, + "Standard_D11_v2": 0.190000, + "Standard_D11_v2_Promo": 0.190000, + "Standard_D12": 0.442000, + "Standard_D12_v2": 0.379000, + "Standard_D12_v2_Promo": 0.379000, + "Standard_D13": 0.885000, + "Standard_D13_v2": 0.759000, + "Standard_D13_v2_Promo": 0.759000, + "Standard_D14": 1.769000, + "Standard_D14_v2": 1.518000, + "Standard_D14_v2_Promo": 1.518000, + "Standard_D15_v2": 1.897000, + "Standard_D15i_v2": 1.897000, + "Standard_D16_v3": 0.960000, + "Standard_D16_v4": 0.920000, + "Standard_D16_v5": 0.920000, + "Standard_D16a_v4": 0.920000, + "Standard_D16ads_v5": 1.736000, + "Standard_D16as_v4": 0.920000, + "Standard_D16as_v5": 1.568000, + "Standard_D16d_v4": 1.088000, + "Standard_D16d_v5": 1.088000, + "Standard_D16ds_v4": 1.088000, + "Standard_D16ds_v5": 1.088000, + "Standard_D16lds_v5": 0.888000, + "Standard_D16ls_v5": 0.776000, + "Standard_D16pds_v5": 0.872000, + "Standard_D16plds_v5": 0.712000, + "Standard_D16pls_v5": 0.621000, + "Standard_D16ps_v5": 0.736000, + "Standard_D16s_v3": 0.960000, + "Standard_D16s_v4": 0.920000, + "Standard_D16s_v5": 0.920000, + "Standard_D1_v2": 0.067900, + "Standard_D2": 0.168000, + "Standard_D2_v2": 0.136000, + "Standard_D2_v2_Promo": 0.136000, + "Standard_D2_v3": 0.120000, + "Standard_D2_v4": 0.115000, + "Standard_D2_v5": 0.115000, + "Standard_D2a_v4": 0.115000, + "Standard_D2ads_v5": 0.125000, + "Standard_D2as_v4": 0.115000, + "Standard_D2as_v5": 0.104000, + "Standard_D2d_v4": 0.136000, + "Standard_D2d_v5": 0.136000, + "Standard_D2ds_v4": 0.136000, + "Standard_D2ds_v5": 0.136000, + "Standard_D2lds_v5": 0.111000, + "Standard_D2ls_v5": 0.097000, + "Standard_D2pds_v5": 0.109000, + "Standard_D2plds_v5": 0.089000, + "Standard_D2pls_v5": 0.077600, + "Standard_D2ps_v5": 0.092000, + "Standard_D2s_v3": 0.120000, + "Standard_D2s_v4": 0.115000, + "Standard_D2s_v5": 0.115000, + "Standard_D3": 0.336000, + "Standard_D32-16s_v3": 1.920000, + "Standard_D32-8s_v3": 1.920000, + "Standard_D32_v3": 1.920000, + "Standard_D32_v4": 1.840000, + "Standard_D32_v5": 1.840000, + "Standard_D32a_v4": 1.840000, + "Standard_D32ads_v5": 2.000000, + "Standard_D32as_v4": 1.840000, + "Standard_D32as_v5": 3.136000, + "Standard_D32d_v4": 2.176000, + "Standard_D32d_v5": 2.176000, + "Standard_D32ds_v4": 2.176000, + "Standard_D32ds_v5": 2.176000, + "Standard_D32lds_v5": 1.776000, + "Standard_D32ls_v5": 1.552000, + "Standard_D32pds_v5": 1.744000, + "Standard_D32plds_v5": 1.424000, + "Standard_D32pls_v5": 1.242000, + "Standard_D32ps_v5": 1.472000, + "Standard_D32s_v3": 1.920000, + "Standard_D32s_v4": 1.840000, + "Standard_D32s_v5": 1.840000, + "Standard_D3_v2": 0.272000, + "Standard_D3_v2_Promo": 0.272000, + "Standard_D4": 0.672000, + "Standard_D48_v3": 2.880000, + "Standard_D48_v4": 2.760000, + "Standard_D48_v5": 2.760000, + "Standard_D48a_v4": 2.760000, + "Standard_D48ads_v5": 5.208000, + "Standard_D48as_v4": 2.760000, + "Standard_D48as_v5": 4.704000, + "Standard_D48d_v4": 3.264000, + "Standard_D48d_v5": 3.264000, + "Standard_D48ds_v4": 3.264000, + "Standard_D48ds_v5": 3.264000, + "Standard_D48lds_v5": 2.664000, + "Standard_D48ls_v5": 2.328000, + "Standard_D48pds_v5": 2.616000, + "Standard_D48plds_v5": 2.136000, + "Standard_D48pls_v5": 1.862000, + "Standard_D48ps_v5": 2.208000, + "Standard_D48s_v3": 2.880000, + "Standard_D48s_v4": 2.760000, + "Standard_D48s_v5": 2.760000, + "Standard_D4_v2": 0.544000, + "Standard_D4_v2_Promo": 0.544000, + "Standard_D4_v3": 0.240000, + "Standard_D4_v4": 0.230000, + "Standard_D4_v5": 0.230000, + "Standard_D4a_v4": 0.230000, + "Standard_D4ads_v5": 0.250000, + "Standard_D4as_v4": 0.230000, + "Standard_D4as_v5": 0.208000, + "Standard_D4d_v4": 0.272000, + "Standard_D4d_v5": 0.272000, + "Standard_D4ds_v4": 0.272000, + "Standard_D4ds_v5": 0.272000, + "Standard_D4lds_v5": 0.222000, + "Standard_D4ls_v5": 0.194000, + "Standard_D4pds_v5": 0.218000, + "Standard_D4plds_v5": 0.178000, + "Standard_D4pls_v5": 0.155000, + "Standard_D4ps_v5": 0.184000, + "Standard_D4s_v3": 0.240000, + "Standard_D4s_v4": 0.230000, + "Standard_D4s_v5": 0.230000, + "Standard_D5_v2": 1.087000, + "Standard_D5_v2_Promo": 1.087000, + "Standard_D64-16s_v3": 3.840000, + "Standard_D64-32s_v3": 3.840000, + "Standard_D64_v3": 3.840000, + "Standard_D64_v4": 3.680000, + "Standard_D64_v5": 3.680000, + "Standard_D64a_v4": 3.680000, + "Standard_D64ads_v5": 6.944000, + "Standard_D64as_v4": 3.680000, + "Standard_D64as_v5": 3.328000, + "Standard_D64d_v4": 4.352000, + "Standard_D64d_v5": 4.352000, + "Standard_D64ds_v4": 4.352000, + "Standard_D64ds_v5": 4.352000, + "Standard_D64lds_v5": 3.552000, + "Standard_D64ls_v5": 3.104000, + "Standard_D64pds_v5": 3.488000, + "Standard_D64plds_v5": 2.848000, + "Standard_D64pls_v5": 2.483000, + "Standard_D64ps_v5": 2.944000, + "Standard_D64s_v3": 3.840000, + "Standard_D64s_v4": 3.680000, + "Standard_D64s_v5": 3.680000, + "Standard_D8_v3": 0.480000, + "Standard_D8_v4": 0.460000, + "Standard_D8_v5": 0.460000, + "Standard_D8a_v4": 0.460000, + "Standard_D8ads_v5": 0.868000, + "Standard_D8as_v4": 0.460000, + "Standard_D8as_v5": 0.416000, + "Standard_D8d_v4": 0.544000, + "Standard_D8d_v5": 0.544000, + "Standard_D8ds_v4": 0.544000, + "Standard_D8ds_v5": 0.544000, + "Standard_D8lds_v5": 0.444000, + "Standard_D8ls_v5": 0.388000, + "Standard_D8pds_v5": 0.436000, + "Standard_D8plds_v5": 0.356000, + "Standard_D8pls_v5": 0.310000, + "Standard_D8ps_v5": 0.368000, + "Standard_D8s_v3": 0.480000, + "Standard_D8s_v4": 0.460000, + "Standard_D8s_v5": 0.460000, + "Standard_D96_v5": 5.520000, + "Standard_D96a_v4": 5.520000, + "Standard_D96ads_v5": 10.416000, + "Standard_D96as_v4": 5.520000, + "Standard_D96as_v5": 9.408000, + "Standard_D96d_v5": 6.528000, + "Standard_D96ds_v5": 6.528000, + "Standard_D96lds_v5": 5.328000, + "Standard_D96ls_v5": 4.656000, + "Standard_D96s_v5": 5.520000, + "Standard_DC16ads_cc_v5": 1.000000, + "Standard_DC16ads_v5": 1.000000, + "Standard_DC16as_cc_v5": 0.832000, + "Standard_DC16as_v5": 0.832000, + "Standard_DC16ds_v3": 2.176000, + "Standard_DC16eds_v5": 1.088000, + "Standard_DC16es_v5": 0.920000, + "Standard_DC16s_v3": 1.840000, + "Standard_DC1ds_v3": 0.136000, + "Standard_DC1s_v2": 0.115000, + "Standard_DC1s_v3": 0.115000, + "Standard_DC24ds_v3": 3.264000, + "Standard_DC24s_v3": 2.760000, + "Standard_DC2ads_v5": 0.125000, + "Standard_DC2as_v5": 0.104000, + "Standard_DC2ds_v3": 0.272000, + "Standard_DC2eds_v5": 0.136000, + "Standard_DC2es_v5": 0.115000, + "Standard_DC2s": 0.257000, + "Standard_DC2s_v2": 0.230000, + "Standard_DC2s_v3": 0.230000, + "Standard_DC32ads_cc_v5": 2.000000, + "Standard_DC32ads_v5": 2.000000, + "Standard_DC32as_cc_v5": 1.664000, + "Standard_DC32as_v5": 1.664000, + "Standard_DC32ds_v3": 4.352000, + "Standard_DC32eds_v5": 2.176000, + "Standard_DC32es_v5": 1.840000, + "Standard_DC32s_v3": 3.680000, + "Standard_DC48ads_cc_v5": 3.000000, + "Standard_DC48ads_v5": 3.000000, + "Standard_DC48as_cc_v5": 2.496000, + "Standard_DC48as_v5": 2.496000, + "Standard_DC48ds_v3": 6.528000, + "Standard_DC48eds_v5": 3.264000, + "Standard_DC48es_v5": 2.760000, + "Standard_DC48s_v3": 5.520000, + "Standard_DC4ads_cc_v5": 0.250000, + "Standard_DC4ads_v5": 0.250000, + "Standard_DC4as_cc_v5": 0.208000, + "Standard_DC4as_v5": 0.208000, + "Standard_DC4ds_v3": 0.544000, + "Standard_DC4eds_v5": 0.272000, + "Standard_DC4es_v5": 0.230000, + "Standard_DC4s": 0.514000, + "Standard_DC4s_v2": 0.460000, + "Standard_DC4s_v3": 0.460000, + "Standard_DC64ads_cc_v5": 4.000000, + "Standard_DC64ads_v5": 4.000000, + "Standard_DC64as_cc_v5": 3.328000, + "Standard_DC64as_v5": 3.328000, + "Standard_DC64eds_v5": 4.352000, + "Standard_DC64es_v5": 3.680000, + "Standard_DC8_v2": 0.920000, + "Standard_DC8ads_cc_v5": 0.500000, + "Standard_DC8ads_v5": 0.500000, + "Standard_DC8as_cc_v5": 0.416000, + "Standard_DC8as_v5": 0.416000, + "Standard_DC8ds_v3": 1.088000, + "Standard_DC8eds_v5": 0.544000, + "Standard_DC8es_v5": 0.460000, + "Standard_DC8s_v3": 0.920000, + "Standard_DC96ads_cc_v5": 6.000000, + "Standard_DC96ads_v5": 6.000000, + "Standard_DC96as_cc_v5": 4.992000, + "Standard_DC96as_v5": 4.992000, + "Standard_DC96eds_v5": 6.528000, + "Standard_DC96es_v5": 5.520000, + "Standard_DS1": 0.084000, + "Standard_DS11": 0.221000, + "Standard_DS11-1_v2": 0.190000, + "Standard_DS11_v2": 0.190000, + "Standard_DS11_v2_Promo": 0.190000, + "Standard_DS12": 0.442000, + "Standard_DS12-1_v2": 0.379000, + "Standard_DS12-2_v2": 0.379000, + "Standard_DS12_v2": 0.379000, + "Standard_DS12_v2_Promo": 0.379000, + "Standard_DS13": 0.885000, + "Standard_DS13-2_v2": 0.759000, + "Standard_DS13-4_v2": 0.759000, + "Standard_DS13_v2": 0.759000, + "Standard_DS13_v2_Promo": 0.759000, + "Standard_DS14": 1.769000, + "Standard_DS14-4_v2": 1.518000, + "Standard_DS14-8_v2": 1.518000, + "Standard_DS14_v2": 1.518000, + "Standard_DS14_v2_Promo": 1.518000, + "Standard_DS15_v2": 1.897000, + "Standard_DS15i_v2": 1.897000, + "Standard_DS1_v2": 0.067900, + "Standard_DS2": 0.168000, + "Standard_DS2_v2": 0.136000, + "Standard_DS2_v2_Promo": 0.136000, + "Standard_DS3": 0.336000, + "Standard_DS3_v2": 0.272000, + "Standard_DS3_v2_Promo": 0.272000, + "Standard_DS4": 0.672000, + "Standard_DS4_v2": 0.544000, + "Standard_DS4_v2_Promo": 0.544000, + "Standard_DS5_v2": 1.087000, + "Standard_DS5_v2_Promo": 1.087000, + "Standard_E104i_v5": 8.694000, + "Standard_E104id_v5": 9.896000, + "Standard_E104ids_v5": 9.896000, + "Standard_E104is_v5": 8.694000, + "Standard_E112iads_v5": 9.733000, + "Standard_E112ias_v5": 8.439000, + "Standard_E112ibds_v5": 12.258000, + "Standard_E112ibs_v5": 10.965000, + "Standard_E16-4ads_v5": 2.000000, + "Standard_E16-4as_v4": 1.216000, + "Standard_E16-4as_v5": 1.096000, + "Standard_E16-4ds_v4": 1.384000, + "Standard_E16-4ds_v5": 1.384000, + "Standard_E16-4s_v3": 1.280000, + "Standard_E16-4s_v4": 1.216000, + "Standard_E16-4s_v5": 1.216000, + "Standard_E16-8ads_v5": 1.264000, + "Standard_E16-8as_v4": 1.216000, + "Standard_E16-8as_v5": 1.096000, + "Standard_E16-8ds_v4": 1.384000, + "Standard_E16-8ds_v5": 1.384000, + "Standard_E16-8s_v3": 1.280000, + "Standard_E16-8s_v4": 1.216000, + "Standard_E16-8s_v5": 1.216000, + "Standard_E16_v3": 1.280000, + "Standard_E16_v4": 1.216000, + "Standard_E16_v5": 1.216000, + "Standard_E16a_v4": 1.216000, + "Standard_E16ads_v5": 1.310000, + "Standard_E16as_v4": 1.216000, + "Standard_E16as_v5": 1.142000, + "Standard_E16bds_v5": 1.592000, + "Standard_E16bs_v5": 1.424000, + "Standard_E16d_v4": 1.384000, + "Standard_E16d_v5": 1.384000, + "Standard_E16ds_v4": 1.384000, + "Standard_E16ds_v5": 1.384000, + "Standard_E16pds_v5": 1.104000, + "Standard_E16ps_v5": 0.973000, + "Standard_E16s_v3": 1.280000, + "Standard_E16s_v4": 1.216000, + "Standard_E16s_v5": 1.216000, + "Standard_E20_v3": 1.600000, + "Standard_E20_v4": 1.520000, + "Standard_E20_v5": 1.520000, + "Standard_E20a_v4": 1.520000, + "Standard_E20ads_v5": 1.580000, + "Standard_E20as_v4": 1.520000, + "Standard_E20as_v5": 1.370000, + "Standard_E20d_v4": 1.730000, + "Standard_E20d_v5": 1.730000, + "Standard_E20ds_v4": 1.730000, + "Standard_E20ds_v5": 1.730000, + "Standard_E20pds_v5": 1.380000, + "Standard_E20ps_v5": 1.216000, + "Standard_E20s_v3": 1.600000, + "Standard_E20s_v4": 1.520000, + "Standard_E20s_v5": 1.520000, + "Standard_E2_v3": 0.160000, + "Standard_E2_v4": 0.152000, + "Standard_E2_v5": 0.152000, + "Standard_E2a_v4": 0.152000, + "Standard_E2ads_v5": 0.158000, + "Standard_E2as_v4": 0.152000, + "Standard_E2as_v5": 0.183000, + "Standard_E2bds_v5": 0.199000, + "Standard_E2bs_v5": 0.178000, + "Standard_E2d_v4": 0.173000, + "Standard_E2d_v5": 0.173000, + "Standard_E2ds_v4": 0.173000, + "Standard_E2ds_v5": 0.173000, + "Standard_E2pds_v5": 0.138000, + "Standard_E2ps_v5": 0.122000, + "Standard_E2s_v3": 0.160000, + "Standard_E2s_v4": 0.152000, + "Standard_E2s_v5": 0.152000, + "Standard_E32-16ads_v5": 4.000000, + "Standard_E32-16as_v4": 2.432000, + "Standard_E32-16as_v5": 2.192000, + "Standard_E32-16ds_v4": 2.768000, + "Standard_E32-16ds_v5": 2.768000, + "Standard_E32-16s_v3": 2.560000, + "Standard_E32-16s_v4": 2.432000, + "Standard_E32-16s_v5": 2.432000, + "Standard_E32-8ads_v5": 2.528000, + "Standard_E32-8as_v4": 2.432000, + "Standard_E32-8as_v5": 3.664000, + "Standard_E32-8ds_v4": 2.768000, + "Standard_E32-8ds_v5": 2.768000, + "Standard_E32-8s_v3": 2.560000, + "Standard_E32-8s_v4": 2.432000, + "Standard_E32-8s_v5": 2.432000, + "Standard_E32_v3": 2.560000, + "Standard_E32_v4": 2.432000, + "Standard_E32_v5": 2.432000, + "Standard_E32a_v4": 2.432000, + "Standard_E32ads_v5": 2.528000, + "Standard_E32as_v4": 2.432000, + "Standard_E32as_v5": 2.192000, + "Standard_E32bds_v5": 3.184000, + "Standard_E32bs_v5": 2.848000, + "Standard_E32d_v4": 2.768000, + "Standard_E32d_v5": 2.768000, + "Standard_E32ds_v4": 2.768000, + "Standard_E32ds_v5": 2.768000, + "Standard_E32pds_v5": 2.208000, + "Standard_E32ps_v5": 1.946000, + "Standard_E32s_v3": 2.560000, + "Standard_E32s_v4": 2.432000, + "Standard_E32s_v5": 2.432000, + "Standard_E4-2ads_v5": 0.500000, + "Standard_E4-2as_v4": 0.304000, + "Standard_E4-2as_v5": 0.274000, + "Standard_E4-2ds_v4": 0.346000, + "Standard_E4-2ds_v5": 0.346000, + "Standard_E4-2s_v3": 0.320000, + "Standard_E4-2s_v4": 0.304000, + "Standard_E4-2s_v5": 0.304000, + "Standard_E48_v3": 3.647000, + "Standard_E48_v4": 3.648000, + "Standard_E48_v5": 3.648000, + "Standard_E48a_v4": 3.648000, + "Standard_E48ads_v5": 3.838000, + "Standard_E48as_v4": 3.648000, + "Standard_E48as_v5": 3.334000, + "Standard_E48bds_v5": 4.776000, + "Standard_E48bs_v5": 4.272000, + "Standard_E48d_v4": 4.152000, + "Standard_E48d_v5": 4.152000, + "Standard_E48ds_v4": 4.152000, + "Standard_E48ds_v5": 4.152000, + "Standard_E48s_v3": 3.647000, + "Standard_E48s_v4": 3.648000, + "Standard_E48s_v5": 3.648000, + "Standard_E4_v3": 0.320000, + "Standard_E4_v4": 0.304000, + "Standard_E4_v5": 0.304000, + "Standard_E4a_v4": 0.304000, + "Standard_E4ads_v5": 0.362000, + "Standard_E4as_v4": 0.304000, + "Standard_E4as_v5": 0.274000, + "Standard_E4bds_v5": 0.398000, + "Standard_E4bs_v5": 0.356000, + "Standard_E4d_v4": 0.346000, + "Standard_E4d_v5": 0.346000, + "Standard_E4ds_v4": 0.346000, + "Standard_E4ds_v5": 0.346000, + "Standard_E4pds_v5": 0.276000, + "Standard_E4ps_v5": 0.243000, + "Standard_E4s_v3": 0.320000, + "Standard_E4s_v4": 0.304000, + "Standard_E4s_v5": 0.304000, + "Standard_E64-16ads_v5": 8.000000, + "Standard_E64-16as_v4": 4.864000, + "Standard_E64-16as_v5": 4.384000, + "Standard_E64-16ds_v4": 5.536000, + "Standard_E64-16ds_v5": 5.536000, + "Standard_E64-16s_v3": 4.376000, + "Standard_E64-16s_v4": 4.864000, + "Standard_E64-16s_v5": 4.864000, + "Standard_E64-32ads_v5": 5.056000, + "Standard_E64-32as_v4": 4.864000, + "Standard_E64-32as_v5": 4.384000, + "Standard_E64-32ds_v4": 5.536000, + "Standard_E64-32ds_v5": 5.536000, + "Standard_E64-32s_v3": 4.376000, + "Standard_E64-32s_v4": 4.864000, + "Standard_E64-32s_v5": 4.864000, + "Standard_E64_v3": 4.376000, + "Standard_E64_v4": 4.864000, + "Standard_E64_v5": 4.864000, + "Standard_E64a_v4": 4.864000, + "Standard_E64ads_v5": 5.102000, + "Standard_E64as_v4": 4.864000, + "Standard_E64as_v5": 4.384000, + "Standard_E64bds_v5": 6.368000, + "Standard_E64bs_v5": 5.696000, + "Standard_E64d_v4": 5.536000, + "Standard_E64d_v5": 5.536000, + "Standard_E64ds_v4": 5.536000, + "Standard_E64ds_v5": 5.536000, + "Standard_E64i_v3": 4.376000, + "Standard_E64is_v3": 4.376000, + "Standard_E64s_v3": 4.376000, + "Standard_E64s_v4": 4.864000, + "Standard_E64s_v5": 4.864000, + "Standard_E8-2ads_v5": 0.632000, + "Standard_E8-2as_v4": 0.608000, + "Standard_E8-2as_v5": 0.916000, + "Standard_E8-2ds_v4": 0.692000, + "Standard_E8-2ds_v5": 0.692000, + "Standard_E8-2s_v3": 0.640000, + "Standard_E8-2s_v4": 0.608000, + "Standard_E8-2s_v5": 0.608000, + "Standard_E8-4ads_v5": 0.632000, + "Standard_E8-4as_v4": 0.608000, + "Standard_E8-4as_v5": 0.916000, + "Standard_E8-4ds_v4": 0.692000, + "Standard_E8-4ds_v5": 0.692000, + "Standard_E8-4s_v3": 0.640000, + "Standard_E8-4s_v4": 0.608000, + "Standard_E8-4s_v5": 0.608000, + "Standard_E80ids_v4": 6.920000, + "Standard_E80is_v4": 6.080000, + "Standard_E8_v3": 0.640000, + "Standard_E8_v4": 0.608000, + "Standard_E8_v5": 0.608000, + "Standard_E8a_v4": 0.608000, + "Standard_E8ads_v5": 0.632000, + "Standard_E8as_v4": 0.608000, + "Standard_E8as_v5": 0.594000, + "Standard_E8bds_v5": 0.796000, + "Standard_E8bs_v5": 0.712000, + "Standard_E8d_v4": 0.692000, + "Standard_E8d_v5": 0.692000, + "Standard_E8ds_v4": 0.692000, + "Standard_E8ds_v5": 0.692000, + "Standard_E8pds_v5": 0.552000, + "Standard_E8ps_v5": 0.486000, + "Standard_E8s_v3": 0.640000, + "Standard_E8s_v4": 0.608000, + "Standard_E8s_v5": 0.608000, + "Standard_E96-24ads_v5": 12.000000, + "Standard_E96-24as_v4": 7.296000, + "Standard_E96-24as_v5": 6.576000, + "Standard_E96-24ds_v5": 8.304000, + "Standard_E96-24s_v5": 7.296000, + "Standard_E96-48ads_v5": 12.000000, + "Standard_E96-48as_v4": 7.296000, + "Standard_E96-48as_v5": 6.576000, + "Standard_E96-48ds_v5": 8.304000, + "Standard_E96-48s_v5": 7.296000, + "Standard_E96_v5": 7.296000, + "Standard_E96a_v4": 7.296000, + "Standard_E96ads_v5": 7.584000, + "Standard_E96as_v4": 7.296000, + "Standard_E96as_v5": 6.622000, + "Standard_E96bds_v5": 9.552000, + "Standard_E96bs_v5": 8.544000, + "Standard_E96d_v5": 8.304000, + "Standard_E96ds_v5": 8.304000, + "Standard_E96iads_v5": 8.342000, + "Standard_E96ias_v5": 7.234000, + "Standard_E96s_v5": 7.296000, + "Standard_EC16ads_cc_v5": 1.264000, + "Standard_EC16ads_v5": 1.184000, + "Standard_EC16as_cc_v5": 1.096000, + "Standard_EC16as_v5": 1.096000, + "Standard_EC16eds_v5": 1.384000, + "Standard_EC16es_v5": 1.216000, + "Standard_EC20ads_cc_v5": 1.580000, + "Standard_EC20ads_v5": 1.480000, + "Standard_EC20as_cc_v5": 1.370000, + "Standard_EC20as_v5": 1.370000, + "Standard_EC2ads_v5": 0.148000, + "Standard_EC2as_v5": 0.137000, + "Standard_EC2eds_v5": 0.173000, + "Standard_EC2es_v5": 0.152000, + "Standard_EC32ads_cc_v5": 2.528000, + "Standard_EC32ads_v5": 2.368000, + "Standard_EC32as_cc_v5": 2.192000, + "Standard_EC32as_v5": 2.192000, + "Standard_EC32eds_v5": 2.768000, + "Standard_EC32es_v5": 2.432000, + "Standard_EC48ads_cc_v5": 3.792000, + "Standard_EC48ads_v5": 3.552000, + "Standard_EC48as_cc_v5": 3.288000, + "Standard_EC48as_v5": 3.288000, + "Standard_EC48eds_v5": 4.152000, + "Standard_EC48es_v5": 3.648000, + "Standard_EC4ads_cc_v5": 0.316000, + "Standard_EC4ads_v5": 0.296000, + "Standard_EC4as_cc_v5": 0.274000, + "Standard_EC4as_v5": 0.274000, + "Standard_EC4eds_v5": 0.346000, + "Standard_EC4es_v5": 0.304000, + "Standard_EC64ads_cc_v5": 5.056000, + "Standard_EC64ads_v5": 4.736000, + "Standard_EC64as_cc_v5": 4.384000, + "Standard_EC64as_v5": 4.384000, + "Standard_EC64eds_v5": 5.536000, + "Standard_EC64es_v5": 4.864000, + "Standard_EC8ads_cc_v5": 0.632000, + "Standard_EC8ads_v5": 0.592000, + "Standard_EC8as_cc_v5": 0.548000, + "Standard_EC8as_v5": 0.548000, + "Standard_EC8eds_v5": 0.692000, + "Standard_EC8es_v5": 0.608000, + "Standard_EC96ads_cc_v5": 7.584000, + "Standard_EC96ads_v5": 7.104000, + "Standard_EC96as_cc_v5": 6.576000, + "Standard_EC96as_v5": 6.576000, + "Standard_EC96eds_v5": 8.304000, + "Standard_EC96es_v5": 7.296000, + "Standard_EC96iads_v5": 7.814000, + "Standard_EC96ias_v5": 7.234000, + "Standard_F1": 0.056800, + "Standard_F16": 0.909000, + "Standard_F16s": 0.909000, + "Standard_F16s_v2": 0.776000, + "Standard_F1s": 0.056800, + "Standard_F2": 0.114000, + "Standard_F2s": 0.114000, + "Standard_F2s_v2": 0.097000, + "Standard_F32s_v2": 1.552000, + "Standard_F4": 0.227000, + "Standard_F48s_v2": 2.328000, + "Standard_F4s": 0.227000, + "Standard_F4s_v2": 0.194000, + "Standard_F64s_v2": 3.104000, + "Standard_F72s_v2": 3.492000, + "Standard_F8": 0.454000, + "Standard_F8s": 0.454000, + "Standard_F8s_v2": 0.388000, + "Standard_FX12mds": 1.350000, + "Standard_FX24mds": 2.700000, + "Standard_FX36mds": 4.050000, + "Standard_FX48mds": 5.400000, + "Standard_FX4mds": 0.450000, + "Standard_G1": 0.700000, + "Standard_G2": 1.400000, + "Standard_G3": 2.800000, + "Standard_G4": 5.600000, + "Standard_G5": 9.990000, + "Standard_GS1": 0.700000, + "Standard_GS2": 1.400000, + "Standard_GS3": 2.800000, + "Standard_GS4": 5.600000, + "Standard_GS4-4": 5.600000, + "Standard_GS4-8": 5.600000, + "Standard_GS5": 9.990000, + "Standard_GS5-16": 9.990000, + "Standard_GS5-8": 9.990000, + "Standard_H16": 2.065000, + "Standard_H16m": 2.767000, + "Standard_H16mr": 3.043000, + "Standard_H16r": 2.271000, + "Standard_H8": 1.032000, + "Standard_H8m": 1.383000, + "Standard_HB120-16rs_v2": 4.680000, + "Standard_HB120-16rs_v3": 4.680000, + "Standard_HB120-32rs_v2": 4.680000, + "Standard_HB120-32rs_v3": 4.680000, + "Standard_HB120-64rs_v2": 4.680000, + "Standard_HB120-64rs_v3": 4.680000, + "Standard_HB120-96rs_v2": 4.680000, + "Standard_HB120-96rs_v3": 4.680000, + "Standard_HB120rs_v2": 4.680000, + "Standard_HB120rs_v3": 4.680000, + "Standard_HB176-144rs_v4": 9.360000, + "Standard_HB176-24rs_v4": 9.360000, + "Standard_HB176-48rs_v4": 9.360000, + "Standard_HB176-96rs_v4": 9.360000, + "Standard_HB176rs_v4": 9.360000, + "Standard_HB60-15rs": 2.964000, + "Standard_HB60-30rs": 2.964000, + "Standard_HB60-45rs": 2.964000, + "Standard_HB60rs": 2.964000, + "Standard_HC44-16rs": 4.118000, + "Standard_HC44-32rs": 4.118000, + "Standard_HC44rs": 4.118000, + "Standard_HX176-144rs": 11.232000, + "Standard_HX176-24rs": 11.232000, + "Standard_HX176-48rs": 11.232000, + "Standard_HX176-96rs": 11.232000, + "Standard_HX176rs": 11.232000, + "Standard_L16as_v3": 1.488000, + "Standard_L16s": 1.488000, + "Standard_L16s_v2": 1.488000, + "Standard_L16s_v3": 1.664000, + "Standard_L32as_v3": 2.976000, + "Standard_L32s": 2.976000, + "Standard_L32s_v2": 2.976000, + "Standard_L32s_v3": 3.328000, + "Standard_L48as_v3": 4.464000, + "Standard_L48s_v2": 4.464000, + "Standard_L48s_v3": 4.992000, + "Standard_L4s": 0.372000, + "Standard_L64as_v3": 5.952000, + "Standard_L64s_v2": 5.952000, + "Standard_L64s_v3": 6.656000, + "Standard_L80as_v3": 7.440000, + "Standard_L80s_v2": 7.440000, + "Standard_L80s_v3": 8.320000, + "Standard_L88is_v2": 8.184000, + "Standard_L8as_v3": 0.744000, + "Standard_L8s": 0.744000, + "Standard_L8s_v2": 0.744000, + "Standard_L8s_v3": 0.832000, + "Standard_M128": 18.673000, + "Standard_M128-32ms": 37.363200, + "Standard_M128-64ms": 37.363200, + "Standard_M128dms_v2": 37.366000, + "Standard_M128ds_v2": 18.676000, + "Standard_M128m": 37.363200, + "Standard_M128ms": 37.363200, + "Standard_M128ms_v2": 36.777000, + "Standard_M128s": 18.673000, + "Standard_M128s_v2": 18.087000, + "Standard_M12ds_v3": 0.000000, + "Standard_M12s_v3": 0.000000, + "Standard_M16-4ms": 4.302200, + "Standard_M16-8ms": 4.302200, + "Standard_M16ms": 4.302200, + "Standard_M16s": 3.341800, + "Standard_M176ds_3_v3": 0.000000, + "Standard_M176s_3_v3": 0.000000, + "Standard_M192idms_v2": 44.889000, + "Standard_M192ids_v2": 22.445000, + "Standard_M192ims_v2": 44.300000, + "Standard_M192is_v2": 21.856000, + "Standard_M208-104ms_v2": 53.544000, + "Standard_M208-104s_v2": 26.772000, + "Standard_M208-52ms_v2": 53.544000, + "Standard_M208-52s_v2": 26.772000, + "Standard_M208ms_v2": 53.544000, + "Standard_M208s_v2": 26.772000, + "Standard_M24ds_v3": 0.000000, + "Standard_M24s_v3": 0.000000, + "Standard_M32-16ms": 8.604400, + "Standard_M32-8ms": 8.604400, + "Standard_M32dms_v2": 8.604000, + "Standard_M32ls": 3.447600, + "Standard_M32ms": 8.604400, + "Standard_M32ms_v2": 8.457000, + "Standard_M32s": 4.669000, + "Standard_M32ts": 3.248000, + "Standard_M416-104ms_v2": 118.980000, + "Standard_M416-104s_v2": 59.496000, + "Standard_M416-208ms_v2": 118.980000, + "Standard_M416-208s_v2": 59.496000, + "Standard_M416is_v2": 59.496000, + "Standard_M416ms_v2": 118.980000, + "Standard_M416s_8_v2": 79.750000, + "Standard_M416s_v2": 59.496000, + "Standard_M48ds_1_v3": 0.000000, + "Standard_M48s_1_v3": 0.000000, + "Standard_M64": 9.337000, + "Standard_M64-16ms": 14.472000, + "Standard_M64-32ms": 14.472000, + "Standard_M64dms_v2": 14.476000, + "Standard_M64ds_v2": 9.337000, + "Standard_M64ls": 6.498000, + "Standard_M64m": 14.472000, + "Standard_M64ms": 14.472000, + "Standard_M64ms_v2": 14.181000, + "Standard_M64s": 9.337000, + "Standard_M64s_v2": 9.042000, + "Standard_M8-2ms": 2.151100, + "Standard_M8-4ms": 2.151100, + "Standard_M8ms": 2.151100, + "Standard_M96ds_1_v3": 0.000000, + "Standard_M96ds_2_v3": 0.000000, + "Standard_M96s_1_v3": 0.000000, + "Standard_M96s_2_v3": 0.000000, + "Standard_NC12": 2.333000, + "Standard_NC12s_v2": 5.364000, + "Standard_NC12s_v3": 7.646000, + "Standard_NC16ads_A10_v4": 2.860000, + "Standard_NC16as_T4_v3": 1.505000, + "Standard_NC24": 4.666000, + "Standard_NC24ads_A100_v4": 4.775000, + "Standard_NC24r": 5.132000, + "Standard_NC24rs_v2": 11.804000, + "Standard_NC24rs_v3": 16.821000, + "Standard_NC24s_v2": 10.728000, + "Standard_NC24s_v3": 15.292000, + "Standard_NC32ads_A10_v4": 5.720000, + "Standard_NC48ads_A100_v4": 9.550000, + "Standard_NC4as_T4_v3": 0.658000, + "Standard_NC6": 1.166000, + "Standard_NC64as_T4_v3": 5.440000, + "Standard_NC6s_v2": 2.682000, + "Standard_NC6s_v3": 3.823000, + "Standard_NC8ads_A10_v4": 1.430000, + "Standard_NC8as_T4_v3": 0.940000, + "Standard_NC96ads_A100_v4": 19.100000, + "Standard_ND12s": 5.363000, + "Standard_ND24rs": 11.804000, + "Standard_ND24s": 10.727000, + "Standard_ND40rs_v2": 27.526000, + "Standard_ND40s_v2": 18.030000, + "Standard_ND6s": 2.682000, + "Standard_ND96ams_A100_v4": 40.962000, + "Standard_ND96amsr_A100_v4": 40.962000, + "Standard_ND96asr_A100_v4": 35.357000, + "Standard_ND96asr_v4": 35.357000, + "Standard_NG16ads_V620_v1": 2.847000, + "Standard_NG32adms_V620": 7.402000, + "Standard_NG32ads_V620_v1": 5.694000, + "Standard_NG8ads_V620_v1": 1.424000, + "Standard_NP10s": 2.145000, + "Standard_NP20s": 4.290000, + "Standard_NP40s": 8.580000, + "Standard_NV12": 2.730000, + "Standard_NV12ads_A10_v5": 1.180000, + "Standard_NV12s_v3": 1.426000, + "Standard_NV16as_v4": 1.165000, + "Standard_NV18ads_A10_v5": 2.080000, + "Standard_NV24": 5.460000, + "Standard_NV24s_v3": 2.850000, + "Standard_NV32as_v4": 2.330000, + "Standard_NV36adms_A10_v5": 5.876000, + "Standard_NV36ads_A10_v5": 4.160000, + "Standard_NV48s_v3": 5.700000, + "Standard_NV4as_v4": 0.291000, + "Standard_NV6": 1.365000, + "Standard_NV6ads_A10_v5": 0.590000, + "Standard_NV72ads_A10_v5": 8.476000, + "Standard_NV8as_v4": 0.582000, + "Standard_PB12s": 2.158000, + "Standard_PB24s": 4.316000, + "Standard_PB6s": 1.079000, + } + // westindia + initialOnDemandPrices["westindia"] = map[string]float64{ + "Basic_A0": 0.018000, + "Basic_A1": 0.030000, + "Basic_A2": 0.096000, + "Basic_A3": 0.232000, + "Basic_A4": 0.464000, + "Dadsv5_Type1": 8.121000, + "Dasv4_Type1": 6.758000, + "Dasv4_Type2": 6.758000, + "Dasv5_Type1": 6.781000, + "Ddsv5_Type1": 7.637000, + "Dsv3_Type3": 4.445000, + "Dsv3_Type4": 5.556000, + "Dsv5_Type1": 6.758000, + "Eadsv5_Type1": 8.854000, + "Easv4_Type1": 8.516000, + "Easv4_Type2": 8.516000, + "Easv5_Type1": 7.637000, + "Ebdsv5-Type1": 7.524000, + "Ebsv5-Type1": 6.713000, + "Edsv5_Type1": 9.732000, + "Esv3_Type3": 5.172000, + "Esv3_Type4": 7.007000, + "Esv5_Type1": 8.516000, + "Fsv2_Type2": 3.366000, + "Lasv3_Type1": 8.786000, + "Lsv3_Type1": 9.828000, + "Standard_A0": 0.020000, + "Standard_A1": 0.066000, + "Standard_A1_v2": 0.047000, + "Standard_A2": 0.131000, + "Standard_A2_v2": 0.098000, + "Standard_A2m_v2": 0.158000, + "Standard_A3": 0.262000, + "Standard_A4": 0.524000, + "Standard_A4_v2": 0.206000, + "Standard_A4m_v2": 0.333000, + "Standard_A5": 0.260000, + "Standard_A6": 0.520000, + "Standard_A7": 1.040000, + "Standard_A8_v2": 0.433000, + "Standard_A8m_v2": 0.699000, + "Standard_B12ms": 0.714000, + "Standard_B16als_v2": 0.755000, + "Standard_B16as_v2": 0.770000, + "Standard_B16ls_v2": 0.755000, + "Standard_B16ms": 0.951000, + "Standard_B16s_v2": 0.926000, + "Standard_B1ls": 0.007800, + "Standard_B1ms": 0.030000, + "Standard_B1s": 0.014700, + "Standard_B20ms": 1.189000, + "Standard_B2als_v2": 0.048100, + "Standard_B2as_v2": 0.096300, + "Standard_B2ats_v2": 0.021200, + "Standard_B2ls_v2": 0.053200, + "Standard_B2ms": 0.119000, + "Standard_B2s": 0.059000, + "Standard_B2s_v2": 0.116000, + "Standard_B2ts_v2": 0.013300, + "Standard_B32als_v2": 1.511000, + "Standard_B32as_v2": 1.687000, + "Standard_B32ls_v2": 1.509000, + "Standard_B32s_v2": 1.851000, + "Standard_B4als_v2": 0.189000, + "Standard_B4as_v2": 0.211000, + "Standard_B4ls_v2": 0.189000, + "Standard_B4ms": 0.238000, + "Standard_B4s_v2": 0.213000, + "Standard_B8als_v2": 0.341000, + "Standard_B8as_v2": 0.422000, + "Standard_B8ls_v2": 0.414000, + "Standard_B8ms": 0.476000, + "Standard_B8s_v2": 0.426000, + "Standard_D1": 0.067900, + "Standard_D11": 0.176000, + "Standard_D11_v2": 0.189000, + "Standard_D11_v2_Promo": 0.189000, + "Standard_D12": 0.352000, + "Standard_D12_v2": 0.379000, + "Standard_D12_v2_Promo": 0.379000, + "Standard_D13": 0.704000, + "Standard_D13_v2": 0.758000, + "Standard_D13_v2_Promo": 0.758000, + "Standard_D14": 1.407000, + "Standard_D14_v2": 1.516000, + "Standard_D14_v2_Promo": 1.516000, + "Standard_D15_v2": 1.895000, + "Standard_D15i_v2": 1.895000, + "Standard_D16_v3": 0.984000, + "Standard_D16_v4": 0.944000, + "Standard_D16_v5": 0.983000, + "Standard_D16a_v4": 0.983000, + "Standard_D16ads_v5": 1.055000, + "Standard_D16as_v4": 0.983000, + "Standard_D16as_v5": 1.617000, + "Standard_D16d_v4": 1.112000, + "Standard_D16d_v5": 1.157000, + "Standard_D16ds_v4": 1.112000, + "Standard_D16ds_v5": 1.157000, + "Standard_D16lds_v5": 0.983000, + "Standard_D16ls_v5": 0.870000, + "Standard_D16s_v3": 0.984000, + "Standard_D16s_v4": 0.944000, + "Standard_D16s_v5": 0.983000, + "Standard_D1_v2": 0.084000, + "Standard_D2": 0.136000, + "Standard_D2_v2": 0.169000, + "Standard_D2_v2_Promo": 0.169000, + "Standard_D2_v3": 0.123000, + "Standard_D2_v4": 0.118000, + "Standard_D2_v5": 0.123000, + "Standard_D2a_v4": 0.123000, + "Standard_D2ads_v5": 0.132000, + "Standard_D2as_v4": 0.123000, + "Standard_D2as_v5": 0.110000, + "Standard_D2d_v4": 0.139000, + "Standard_D2d_v5": 0.145000, + "Standard_D2ds_v4": 0.139000, + "Standard_D2ds_v5": 0.145000, + "Standard_D2lds_v5": 0.123000, + "Standard_D2ls_v5": 0.109000, + "Standard_D2s_v3": 0.123000, + "Standard_D2s_v4": 0.118000, + "Standard_D2s_v5": 0.123000, + "Standard_D3": 0.272000, + "Standard_D32-16s_v3": 1.968000, + "Standard_D32-8s_v3": 1.968000, + "Standard_D32_v3": 1.968000, + "Standard_D32_v4": 1.888000, + "Standard_D32_v5": 1.966000, + "Standard_D32a_v4": 1.966000, + "Standard_D32ads_v5": 2.109000, + "Standard_D32as_v4": 1.966000, + "Standard_D32as_v5": 1.761000, + "Standard_D32d_v4": 2.224000, + "Standard_D32d_v5": 2.314000, + "Standard_D32ds_v4": 2.224000, + "Standard_D32ds_v5": 2.314000, + "Standard_D32lds_v5": 1.966000, + "Standard_D32ls_v5": 1.741000, + "Standard_D32s_v3": 1.968000, + "Standard_D32s_v4": 1.888000, + "Standard_D32s_v5": 1.966000, + "Standard_D3_v2": 0.337000, + "Standard_D3_v2_Promo": 0.337000, + "Standard_D4": 0.544000, + "Standard_D48_v3": 2.952000, + "Standard_D48_v4": 2.832000, + "Standard_D48_v5": 2.949000, + "Standard_D48a_v4": 2.949000, + "Standard_D48ads_v5": 5.372000, + "Standard_D48as_v4": 2.949000, + "Standard_D48as_v5": 4.850000, + "Standard_D48d_v4": 3.336000, + "Standard_D48d_v5": 3.471000, + "Standard_D48ds_v4": 3.336000, + "Standard_D48ds_v5": 3.471000, + "Standard_D48lds_v5": 2.949000, + "Standard_D48ls_v5": 2.611000, + "Standard_D48s_v3": 2.952000, + "Standard_D48s_v4": 2.832000, + "Standard_D48s_v5": 2.949000, + "Standard_D4_v2": 0.675000, + "Standard_D4_v2_Promo": 0.675000, + "Standard_D4_v3": 0.246000, + "Standard_D4_v4": 0.236000, + "Standard_D4_v5": 0.246000, + "Standard_D4a_v4": 0.246000, + "Standard_D4ads_v5": 0.264000, + "Standard_D4as_v4": 0.246000, + "Standard_D4as_v5": 0.404000, + "Standard_D4d_v4": 0.278000, + "Standard_D4d_v5": 0.289000, + "Standard_D4ds_v4": 0.278000, + "Standard_D4ds_v5": 0.289000, + "Standard_D4lds_v5": 0.246000, + "Standard_D4ls_v5": 0.218000, + "Standard_D4s_v3": 0.246000, + "Standard_D4s_v4": 0.236000, + "Standard_D4s_v5": 0.246000, + "Standard_D5_v2": 1.350000, + "Standard_D5_v2_Promo": 1.350000, + "Standard_D64-16s_v3": 3.936000, + "Standard_D64-32s_v3": 3.936000, + "Standard_D64_v3": 3.936000, + "Standard_D64_v4": 3.776000, + "Standard_D64_v5": 3.932000, + "Standard_D64a_v4": 3.932000, + "Standard_D64ads_v5": 7.163000, + "Standard_D64as_v4": 3.932000, + "Standard_D64as_v5": 6.467000, + "Standard_D64d_v4": 4.448000, + "Standard_D64d_v5": 4.628000, + "Standard_D64ds_v4": 4.448000, + "Standard_D64ds_v5": 4.628000, + "Standard_D64lds_v5": 3.932000, + "Standard_D64ls_v5": 3.482000, + "Standard_D64s_v3": 3.936000, + "Standard_D64s_v4": 3.776000, + "Standard_D64s_v5": 3.932000, + "Standard_D8_v3": 0.492000, + "Standard_D8_v4": 0.472000, + "Standard_D8_v5": 0.492000, + "Standard_D8a_v4": 0.492000, + "Standard_D8ads_v5": 0.527000, + "Standard_D8as_v4": 0.492000, + "Standard_D8as_v5": 0.440000, + "Standard_D8d_v4": 0.556000, + "Standard_D8d_v5": 0.579000, + "Standard_D8ds_v4": 0.556000, + "Standard_D8ds_v5": 0.579000, + "Standard_D8lds_v5": 0.492000, + "Standard_D8ls_v5": 0.435000, + "Standard_D8s_v3": 0.492000, + "Standard_D8s_v4": 0.472000, + "Standard_D8s_v5": 0.492000, + "Standard_D96_v5": 5.898000, + "Standard_D96a_v4": 5.898000, + "Standard_D96ads_v5": 6.328000, + "Standard_D96as_v4": 5.898000, + "Standard_D96as_v5": 5.284000, + "Standard_D96d_v5": 6.943000, + "Standard_D96ds_v5": 6.943000, + "Standard_D96lds_v5": 5.898000, + "Standard_D96ls_v5": 5.222000, + "Standard_D96s_v5": 5.898000, + "Standard_DS1": 0.067900, + "Standard_DS11": 0.176000, + "Standard_DS11-1_v2": 0.189000, + "Standard_DS11_v2": 0.189000, + "Standard_DS11_v2_Promo": 0.189000, + "Standard_DS12": 0.352000, + "Standard_DS12-1_v2": 0.379000, + "Standard_DS12-2_v2": 0.379000, + "Standard_DS12_v2": 0.379000, + "Standard_DS12_v2_Promo": 0.379000, + "Standard_DS13": 0.704000, + "Standard_DS13-2_v2": 0.758000, + "Standard_DS13-4_v2": 0.758000, + "Standard_DS13_v2": 0.758000, + "Standard_DS13_v2_Promo": 0.758000, + "Standard_DS14": 1.407000, + "Standard_DS14-4_v2": 1.516000, + "Standard_DS14-8_v2": 1.516000, + "Standard_DS14_v2": 1.516000, + "Standard_DS14_v2_Promo": 1.516000, + "Standard_DS15_v2": 1.895000, + "Standard_DS15i_v2": 1.895000, + "Standard_DS1_v2": 0.084000, + "Standard_DS2": 0.136000, + "Standard_DS2_v2": 0.169000, + "Standard_DS2_v2_Promo": 0.169000, + "Standard_DS3": 0.272000, + "Standard_DS3_v2": 0.337000, + "Standard_DS3_v2_Promo": 0.337000, + "Standard_DS4": 0.544000, + "Standard_DS4_v2": 0.675000, + "Standard_DS4_v2_Promo": 0.675000, + "Standard_DS5_v2": 1.350000, + "Standard_DS5_v2_Promo": 1.350000, + "Standard_E104i_v5": 9.225000, + "Standard_E104id_v5": 10.543000, + "Standard_E104ids_v5": 10.543000, + "Standard_E104is_v5": 9.225000, + "Standard_E112iads_v5": 10.329000, + "Standard_E112ias_v5": 8.910000, + "Standard_E112ibds_v5": 13.168000, + "Standard_E112ibs_v5": 11.748000, + "Standard_E16-4ads_v5": 2.077000, + "Standard_E16-4as_v4": 1.290000, + "Standard_E16-4as_v5": 1.893000, + "Standard_E16-4ds_v4": 1.390000, + "Standard_E16-4ds_v5": 1.475000, + "Standard_E16-4s_v3": 1.216000, + "Standard_E16-4s_v4": 1.216000, + "Standard_E16-4s_v5": 1.290000, + "Standard_E16-8ads_v5": 2.077000, + "Standard_E16-8as_v4": 1.290000, + "Standard_E16-8as_v5": 1.893000, + "Standard_E16-8ds_v4": 1.390000, + "Standard_E16-8ds_v5": 1.475000, + "Standard_E16-8s_v3": 1.216000, + "Standard_E16-8s_v4": 1.216000, + "Standard_E16-8s_v5": 1.290000, + "Standard_E16_v3": 1.216000, + "Standard_E16_v4": 1.216000, + "Standard_E16_v5": 1.290000, + "Standard_E16a_v4": 1.290000, + "Standard_E16ads_v5": 1.341000, + "Standard_E16as_v4": 1.290000, + "Standard_E16as_v5": 1.203000, + "Standard_E16bds_v5": 1.710000, + "Standard_E16bs_v5": 1.526000, + "Standard_E16d_v4": 1.390000, + "Standard_E16d_v5": 1.475000, + "Standard_E16ds_v4": 1.390000, + "Standard_E16ds_v5": 1.475000, + "Standard_E16s_v3": 1.216000, + "Standard_E16s_v4": 1.216000, + "Standard_E16s_v5": 1.290000, + "Standard_E20_v3": 1.520000, + "Standard_E20_v4": 1.520000, + "Standard_E20_v5": 1.613000, + "Standard_E20a_v4": 1.613000, + "Standard_E20ads_v5": 1.677000, + "Standard_E20as_v4": 1.613000, + "Standard_E20as_v5": 1.492000, + "Standard_E20d_v4": 1.737000, + "Standard_E20d_v5": 1.843000, + "Standard_E20ds_v4": 1.737000, + "Standard_E20ds_v5": 1.843000, + "Standard_E20s_v3": 1.520000, + "Standard_E20s_v4": 1.520000, + "Standard_E20s_v5": 1.613000, + "Standard_E2_v3": 0.152000, + "Standard_E2_v4": 0.152000, + "Standard_E2_v5": 0.161000, + "Standard_E2a_v4": 0.161000, + "Standard_E2ads_v5": 0.168000, + "Standard_E2as_v4": 0.161000, + "Standard_E2as_v5": 0.191000, + "Standard_E2bds_v5": 0.214000, + "Standard_E2bs_v5": 0.191000, + "Standard_E2d_v4": 0.174000, + "Standard_E2d_v5": 0.184000, + "Standard_E2ds_v4": 0.174000, + "Standard_E2ds_v5": 0.184000, + "Standard_E2s_v3": 0.152000, + "Standard_E2s_v4": 0.152000, + "Standard_E2s_v5": 0.161000, + "Standard_E32-16ads_v5": 2.683000, + "Standard_E32-16as_v4": 2.580000, + "Standard_E32-16as_v5": 3.786000, + "Standard_E32-16ds_v4": 2.779000, + "Standard_E32-16ds_v5": 2.949000, + "Standard_E32-16s_v3": 2.432000, + "Standard_E32-16s_v4": 2.432000, + "Standard_E32-16s_v5": 2.580000, + "Standard_E32-8ads_v5": 2.683000, + "Standard_E32-8as_v4": 2.580000, + "Standard_E32-8as_v5": 2.314000, + "Standard_E32-8ds_v4": 2.779000, + "Standard_E32-8ds_v5": 2.949000, + "Standard_E32-8s_v3": 2.432000, + "Standard_E32-8s_v4": 2.432000, + "Standard_E32-8s_v5": 2.580000, + "Standard_E32_v3": 2.432000, + "Standard_E32_v4": 2.432000, + "Standard_E32_v5": 2.580000, + "Standard_E32a_v4": 2.580000, + "Standard_E32ads_v5": 2.683000, + "Standard_E32as_v4": 2.580000, + "Standard_E32as_v5": 2.314000, + "Standard_E32bds_v5": 3.420000, + "Standard_E32bs_v5": 3.052000, + "Standard_E32d_v4": 2.779000, + "Standard_E32d_v5": 2.949000, + "Standard_E32ds_v4": 2.779000, + "Standard_E32ds_v5": 2.949000, + "Standard_E32s_v3": 2.432000, + "Standard_E32s_v4": 2.432000, + "Standard_E32s_v5": 2.580000, + "Standard_E4-2ads_v5": 0.335000, + "Standard_E4-2as_v4": 0.323000, + "Standard_E4-2as_v5": 0.289000, + "Standard_E4-2ds_v4": 0.347000, + "Standard_E4-2ds_v5": 0.369000, + "Standard_E4-2s_v3": 0.304000, + "Standard_E4-2s_v4": 0.304000, + "Standard_E4-2s_v5": 0.323000, + "Standard_E48_v3": 3.640000, + "Standard_E48_v4": 3.648000, + "Standard_E48_v5": 3.871000, + "Standard_E48a_v4": 3.871000, + "Standard_E48ads_v5": 4.024000, + "Standard_E48as_v4": 3.871000, + "Standard_E48as_v5": 3.471000, + "Standard_E48bds_v5": 5.130000, + "Standard_E48bs_v5": 4.577000, + "Standard_E48d_v4": 4.169000, + "Standard_E48d_v5": 4.424000, + "Standard_E48ds_v4": 4.169000, + "Standard_E48ds_v5": 4.424000, + "Standard_E48s_v3": 3.640000, + "Standard_E48s_v4": 3.648000, + "Standard_E48s_v5": 3.871000, + "Standard_E4_v3": 0.304000, + "Standard_E4_v4": 0.304000, + "Standard_E4_v5": 0.323000, + "Standard_E4a_v4": 0.323000, + "Standard_E4ads_v5": 0.335000, + "Standard_E4as_v4": 0.323000, + "Standard_E4as_v5": 0.289000, + "Standard_E4bds_v5": 0.428000, + "Standard_E4bs_v5": 0.381000, + "Standard_E4d_v4": 0.347000, + "Standard_E4d_v5": 0.369000, + "Standard_E4ds_v4": 0.347000, + "Standard_E4ds_v5": 0.369000, + "Standard_E4s_v3": 0.304000, + "Standard_E4s_v4": 0.304000, + "Standard_E4s_v5": 0.323000, + "Standard_E64-16ads_v5": 8.310000, + "Standard_E64-16as_v4": 5.161000, + "Standard_E64-16as_v5": 7.572000, + "Standard_E64-16ds_v4": 5.559000, + "Standard_E64-16ds_v5": 5.898000, + "Standard_E64-16s_v3": 4.368000, + "Standard_E64-16s_v4": 4.864000, + "Standard_E64-16s_v5": 5.161000, + "Standard_E64-32ads_v5": 5.366000, + "Standard_E64-32as_v4": 5.161000, + "Standard_E64-32as_v5": 4.628000, + "Standard_E64-32ds_v4": 5.559000, + "Standard_E64-32ds_v5": 5.898000, + "Standard_E64-32s_v3": 4.368000, + "Standard_E64-32s_v4": 4.864000, + "Standard_E64-32s_v5": 5.161000, + "Standard_E64_v3": 4.368000, + "Standard_E64_v4": 4.864000, + "Standard_E64_v5": 5.161000, + "Standard_E64a_v4": 5.161000, + "Standard_E64ads_v5": 5.412000, + "Standard_E64as_v4": 5.161000, + "Standard_E64as_v5": 4.628000, + "Standard_E64bds_v5": 6.840000, + "Standard_E64bs_v5": 6.103000, + "Standard_E64d_v4": 5.559000, + "Standard_E64d_v5": 5.898000, + "Standard_E64ds_v4": 5.559000, + "Standard_E64ds_v5": 5.898000, + "Standard_E64i_v3": 4.368000, + "Standard_E64is_v3": 4.368000, + "Standard_E64s_v3": 4.368000, + "Standard_E64s_v4": 4.864000, + "Standard_E64s_v5": 5.161000, + "Standard_E8-2ads_v5": 1.039000, + "Standard_E8-2as_v4": 0.645000, + "Standard_E8-2as_v5": 0.579000, + "Standard_E8-2ds_v4": 0.695000, + "Standard_E8-2ds_v5": 0.737000, + "Standard_E8-2s_v3": 0.608000, + "Standard_E8-2s_v4": 0.608000, + "Standard_E8-2s_v5": 0.645000, + "Standard_E8-4ads_v5": 1.039000, + "Standard_E8-4as_v4": 0.645000, + "Standard_E8-4as_v5": 0.579000, + "Standard_E8-4ds_v4": 0.695000, + "Standard_E8-4ds_v5": 0.737000, + "Standard_E8-4s_v3": 0.608000, + "Standard_E8-4s_v4": 0.608000, + "Standard_E8-4s_v5": 0.645000, + "Standard_E80ids_v4": 6.948000, + "Standard_E80is_v4": 6.080000, + "Standard_E8_v3": 0.608000, + "Standard_E8_v4": 0.608000, + "Standard_E8_v5": 0.645000, + "Standard_E8a_v4": 0.645000, + "Standard_E8ads_v5": 0.671000, + "Standard_E8as_v4": 0.645000, + "Standard_E8as_v5": 0.579000, + "Standard_E8bds_v5": 0.855000, + "Standard_E8bs_v5": 0.763000, + "Standard_E8d_v4": 0.695000, + "Standard_E8d_v5": 0.737000, + "Standard_E8ds_v4": 0.695000, + "Standard_E8ds_v5": 0.737000, + "Standard_E8s_v3": 0.608000, + "Standard_E8s_v4": 0.608000, + "Standard_E8s_v5": 0.645000, + "Standard_E96-24ads_v5": 12.465000, + "Standard_E96-24as_v4": 7.741000, + "Standard_E96-24as_v5": 6.943000, + "Standard_E96-24ds_v5": 8.847000, + "Standard_E96-24s_v5": 7.741000, + "Standard_E96-48ads_v5": 12.465000, + "Standard_E96-48as_v4": 7.741000, + "Standard_E96-48as_v5": 11.359000, + "Standard_E96-48ds_v5": 8.847000, + "Standard_E96-48s_v5": 7.741000, + "Standard_E96_v5": 7.741000, + "Standard_E96a_v4": 7.741000, + "Standard_E96ads_v5": 8.095000, + "Standard_E96as_v4": 7.741000, + "Standard_E96as_v5": 6.943000, + "Standard_E96bds_v5": 10.260000, + "Standard_E96bs_v5": 9.155000, + "Standard_E96d_v5": 8.847000, + "Standard_E96ds_v5": 8.847000, + "Standard_E96iads_v5": 8.854000, + "Standard_E96ias_v4": 7.741000, + "Standard_E96ias_v5": 7.637000, + "Standard_E96s_v5": 7.741000, + "Standard_F1": 0.054900, + "Standard_F16": 0.878000, + "Standard_F16s": 0.878000, + "Standard_F16s_v2": 0.680000, + "Standard_F1s": 0.054900, + "Standard_F2": 0.110000, + "Standard_F2s": 0.110000, + "Standard_F2s_v2": 0.085000, + "Standard_F32s_v2": 1.360000, + "Standard_F4": 0.219000, + "Standard_F48s_v2": 2.040000, + "Standard_F4s": 0.219000, + "Standard_F4s_v2": 0.170000, + "Standard_F64s_v2": 2.720000, + "Standard_F72s_v2": 3.060000, + "Standard_F8": 0.439000, + "Standard_F8s": 0.439000, + "Standard_F8s_v2": 0.340000, + "Standard_L16as_v3": 1.597000, + "Standard_L16s_v3": 1.787000, + "Standard_L32as_v3": 3.195000, + "Standard_L32s_v3": 3.574000, + "Standard_L48as_v3": 4.792000, + "Standard_L48s_v3": 5.361000, + "Standard_L64as_v3": 6.390000, + "Standard_L64s_v3": 7.148000, + "Standard_L80as_v3": 7.987000, + "Standard_L80s_v3": 8.934000, + "Standard_L8as_v3": 0.799000, + "Standard_L8s_v3": 0.893000, + } + // westus + initialOnDemandPrices["westus"] = map[string]float64{ + "Basic_A0": 0.018000, + "Basic_A1": 0.031000, + "Basic_A2": 0.081000, + "Basic_A3": 0.188000, + "Basic_A4": 0.376000, + "DCadsv5 Type 1": 7.515000, + "DCasv5 Type 1": 6.222000, + "DCdsv3 Type1": 7.022000, + "DCsv2 Type 1": 0.986000, + "DCsv3 Type1": 5.914000, + "Dadsv5_Type1": 7.515000, + "Dasv4_Type1": 5.914000, + "Dasv4_Type2": 6.160000, + "Dasv5_Type1": 6.222000, + "Ddsv4_Type 1": 4.682000, + "Ddsv4_Type2": 5.559000, + "Ddsv5_Type1": 7.022000, + "Dsv3_Type3": 4.929000, + "Dsv3_Type4": 6.161000, + "Dsv4_Type1": 4.928000, + "Dsv4_Type2": 5.914000, + "Dsv5_Type1": 6.160000, + "ECadsv5 Type 1": 8.596000, + "ECasv5 Type 1": 7.318000, + "Eadsv5_Type1": 7.814000, + "Easv4_Type1": 7.392000, + "Easv4_Type2": 7.392000, + "Easv5_Type1": 6.653000, + "Ebdsv5-Type1": 6.653000, + "Ebsv5-Type1": 5.878000, + "Edsv4_Type 1": 5.702000, + "Edsv4_Type2": 6.772000, + "Edsv5_Type1": 8.554000, + "Esv3_Type3": 4.774000, + "Esv3_Type4": 6.468000, + "Esv4_Type1": 4.928000, + "Esv4_Type2": 6.468000, + "Esv5_Type1": 7.392000, + "Fsv2 Type3": 4.897000, + "Fsv2_Type2": 4.198000, + "Fsv2_Type4": 5.597000, + "Lasv3_Type1": 7.568000, + "Lsv2_Type1": 7.568000, + "Lsv3_Type1": 8.448000, + "Mdmsv2MedMem _Type1": 38.167000, + "Mdsv2MedMem_Type1": 19.076000, + "Ms_Type1": 19.069000, + "Msm_Type1": 38.172000, + "Msmv2MedMem Type1": 37.565000, + "Msmv2_Type1": 141.780000, + "Msv2MedMem Type1": 18.474000, + "Msv2_Type1": 70.899000, + "NVsv3_Type1": 5.016000, + "Standard_A0": 0.020000, + "Standard_A1": 0.060000, + "Standard_A1_v2": 0.043000, + "Standard_A2": 0.120000, + "Standard_A2_v2": 0.091000, + "Standard_A2m_v2": 0.149000, + "Standard_A3": 0.240000, + "Standard_A4": 0.480000, + "Standard_A4_v2": 0.191000, + "Standard_A4m_v2": 0.297000, + "Standard_A5": 0.250000, + "Standard_A6": 0.500000, + "Standard_A7": 1.000000, + "Standard_A8_v2": 0.400000, + "Standard_A8m_v2": 0.594000, + "Standard_B12ms": 0.595000, + "Standard_B16als_v2": 0.632000, + "Standard_B16as_v2": 0.787000, + "Standard_B16ls_v2": 0.776000, + "Standard_B16ms": 0.794000, + "Standard_B16pls_v2": 0.640000, + "Standard_B16ps_v2": 0.714000, + "Standard_B16s_v2": 0.794000, + "Standard_B1ls": 0.006200, + "Standard_B1ms": 0.024800, + "Standard_B1s": 0.012400, + "Standard_B20ms": 0.992000, + "Standard_B2als_v2": 0.053800, + "Standard_B2as_v2": 0.089200, + "Standard_B2ats_v2": 0.011200, + "Standard_B2ls_v2": 0.049600, + "Standard_B2ms": 0.099200, + "Standard_B2pls_v2": 0.049200, + "Standard_B2ps_v2": 0.089200, + "Standard_B2pts_v2": 0.010000, + "Standard_B2s": 0.049600, + "Standard_B2s_v2": 0.099200, + "Standard_B2ts_v2": 0.021600, + "Standard_B32als_v2": 1.411000, + "Standard_B32as_v2": 1.574000, + "Standard_B32ls_v2": 1.405000, + "Standard_B32s_v2": 1.587000, + "Standard_B4als_v2": 0.158000, + "Standard_B4as_v2": 0.197000, + "Standard_B4ls_v2": 0.176000, + "Standard_B4ms": 0.198000, + "Standard_B4pls_v2": 0.142000, + "Standard_B4ps_v2": 0.178000, + "Standard_B4s_v2": 0.198000, + "Standard_B8als_v2": 0.316000, + "Standard_B8as_v2": 0.394000, + "Standard_B8ls_v2": 0.388000, + "Standard_B8ms": 0.397000, + "Standard_B8pls_v2": 0.283000, + "Standard_B8ps_v2": 0.320000, + "Standard_B8s_v2": 0.434000, + "Standard_D1": 0.077000, + "Standard_D11": 0.193000, + "Standard_D11_v2": 0.185000, + "Standard_D11_v2_Promo": 0.185000, + "Standard_D12": 0.386000, + "Standard_D12_v2": 0.370000, + "Standard_D12_v2_Promo": 0.370000, + "Standard_D13": 0.771000, + "Standard_D13_v2": 0.741000, + "Standard_D13_v2_Promo": 0.741000, + "Standard_D14": 1.542000, + "Standard_D14_v2": 1.482000, + "Standard_D14_v2_Promo": 1.482000, + "Standard_D15_v2": 1.852000, + "Standard_D15i_v2": 1.852000, + "Standard_D16_v3": 0.936000, + "Standard_D16_v4": 0.896000, + "Standard_D16_v5": 0.896000, + "Standard_D16a_v4": 0.896000, + "Standard_D16ads_v5": 1.712000, + "Standard_D16as_v4": 0.896000, + "Standard_D16as_v5": 1.544000, + "Standard_D16d_v4": 1.064000, + "Standard_D16d_v5": 1.064000, + "Standard_D16ds_v4": 1.064000, + "Standard_D16ds_v5": 1.064000, + "Standard_D16lds_v5": 0.960000, + "Standard_D16ls_v5": 0.848000, + "Standard_D16pds_v5": 0.848000, + "Standard_D16plds_v5": 0.768000, + "Standard_D16pls_v5": 0.678000, + "Standard_D16ps_v5": 0.717000, + "Standard_D16s_v3": 0.936000, + "Standard_D16s_v4": 0.896000, + "Standard_D16s_v5": 0.896000, + "Standard_D1_v2": 0.070000, + "Standard_D2": 0.154000, + "Standard_D2_v2": 0.140000, + "Standard_D2_v2_Promo": 0.140000, + "Standard_D2_v3": 0.117000, + "Standard_D2_v4": 0.112000, + "Standard_D2_v5": 0.112000, + "Standard_D2a_v4": 0.112000, + "Standard_D2ads_v5": 0.214000, + "Standard_D2as_v4": 0.112000, + "Standard_D2as_v5": 0.101000, + "Standard_D2d_v4": 0.133000, + "Standard_D2d_v5": 0.133000, + "Standard_D2ds_v4": 0.133000, + "Standard_D2ds_v5": 0.133000, + "Standard_D2lds_v5": 0.120000, + "Standard_D2ls_v5": 0.106000, + "Standard_D2pds_v5": 0.106000, + "Standard_D2plds_v5": 0.096000, + "Standard_D2pls_v5": 0.084800, + "Standard_D2ps_v5": 0.089600, + "Standard_D2s_v3": 0.117000, + "Standard_D2s_v4": 0.112000, + "Standard_D2s_v5": 0.112000, + "Standard_D3": 0.308000, + "Standard_D32-16s_v3": 1.872000, + "Standard_D32-8s_v3": 1.872000, + "Standard_D32_v3": 1.872000, + "Standard_D32_v4": 1.792000, + "Standard_D32_v5": 1.792000, + "Standard_D32a_v4": 1.792000, + "Standard_D32ads_v5": 3.424000, + "Standard_D32as_v4": 1.792000, + "Standard_D32as_v5": 3.088000, + "Standard_D32d_v4": 2.128000, + "Standard_D32d_v5": 2.128000, + "Standard_D32ds_v4": 2.128000, + "Standard_D32ds_v5": 2.128000, + "Standard_D32lds_v5": 1.920000, + "Standard_D32ls_v5": 1.696000, + "Standard_D32pds_v5": 1.696000, + "Standard_D32plds_v5": 1.536000, + "Standard_D32pls_v5": 1.357000, + "Standard_D32ps_v5": 1.434000, + "Standard_D32s_v3": 1.872000, + "Standard_D32s_v4": 1.792000, + "Standard_D32s_v5": 1.792000, + "Standard_D3_v2": 0.279000, + "Standard_D3_v2_Promo": 0.279000, + "Standard_D4": 0.616000, + "Standard_D48_v3": 2.808000, + "Standard_D48_v4": 2.688000, + "Standard_D48_v5": 2.688000, + "Standard_D48a_v4": 2.688000, + "Standard_D48ads_v5": 2.928000, + "Standard_D48as_v4": 2.688000, + "Standard_D48as_v5": 4.632000, + "Standard_D48d_v4": 3.192000, + "Standard_D48d_v5": 3.192000, + "Standard_D48ds_v4": 3.192000, + "Standard_D48ds_v5": 3.192000, + "Standard_D48lds_v5": 2.880000, + "Standard_D48ls_v5": 2.544000, + "Standard_D48pds_v5": 2.544000, + "Standard_D48plds_v5": 2.304000, + "Standard_D48pls_v5": 2.035000, + "Standard_D48ps_v5": 2.150000, + "Standard_D48s_v3": 2.808000, + "Standard_D48s_v4": 2.688000, + "Standard_D48s_v5": 2.688000, + "Standard_D4_v2": 0.559000, + "Standard_D4_v2_Promo": 0.559000, + "Standard_D4_v3": 0.234000, + "Standard_D4_v4": 0.224000, + "Standard_D4_v5": 0.224000, + "Standard_D4a_v4": 0.224000, + "Standard_D4ads_v5": 0.428000, + "Standard_D4as_v4": 0.224000, + "Standard_D4as_v5": 0.386000, + "Standard_D4d_v4": 0.266000, + "Standard_D4d_v5": 0.266000, + "Standard_D4ds_v4": 0.266000, + "Standard_D4ds_v5": 0.266000, + "Standard_D4lds_v5": 0.240000, + "Standard_D4ls_v5": 0.212000, + "Standard_D4pds_v5": 0.212000, + "Standard_D4plds_v5": 0.192000, + "Standard_D4pls_v5": 0.170000, + "Standard_D4ps_v5": 0.179000, + "Standard_D4s_v3": 0.234000, + "Standard_D4s_v4": 0.224000, + "Standard_D4s_v5": 0.224000, + "Standard_D5_v2": 1.117000, + "Standard_D5_v2_Promo": 1.117000, + "Standard_D64-16s_v3": 3.744000, + "Standard_D64-32s_v3": 3.744000, + "Standard_D64_v3": 3.744000, + "Standard_D64_v4": 3.584000, + "Standard_D64_v5": 3.584000, + "Standard_D64a_v4": 3.584000, + "Standard_D64ads_v5": 3.904000, + "Standard_D64as_v4": 3.584000, + "Standard_D64as_v5": 6.176000, + "Standard_D64d_v4": 4.256000, + "Standard_D64d_v5": 4.256000, + "Standard_D64ds_v4": 4.256000, + "Standard_D64ds_v5": 4.256000, + "Standard_D64lds_v5": 3.840000, + "Standard_D64ls_v5": 3.392000, + "Standard_D64pds_v5": 3.392000, + "Standard_D64plds_v5": 3.072000, + "Standard_D64pls_v5": 2.714000, + "Standard_D64ps_v5": 2.867000, + "Standard_D64s_v3": 3.744000, + "Standard_D64s_v4": 3.584000, + "Standard_D64s_v5": 3.584000, + "Standard_D8_v3": 0.468000, + "Standard_D8_v4": 0.448000, + "Standard_D8_v5": 0.448000, + "Standard_D8a_v4": 0.448000, + "Standard_D8ads_v5": 0.488000, + "Standard_D8as_v4": 0.448000, + "Standard_D8as_v5": 0.772000, + "Standard_D8d_v4": 0.532000, + "Standard_D8d_v5": 0.532000, + "Standard_D8ds_v4": 0.532000, + "Standard_D8ds_v5": 0.532000, + "Standard_D8lds_v5": 0.480000, + "Standard_D8ls_v5": 0.424000, + "Standard_D8pds_v5": 0.424000, + "Standard_D8plds_v5": 0.384000, + "Standard_D8pls_v5": 0.339000, + "Standard_D8ps_v5": 0.358000, + "Standard_D8s_v3": 0.468000, + "Standard_D8s_v4": 0.448000, + "Standard_D8s_v5": 0.448000, + "Standard_D96_v5": 5.376000, + "Standard_D96a_v4": 5.376000, + "Standard_D96ads_v5": 10.272000, + "Standard_D96as_v4": 5.376000, + "Standard_D96as_v5": 4.848000, + "Standard_D96d_v5": 6.384000, + "Standard_D96ds_v5": 6.384000, + "Standard_D96lds_v5": 5.760000, + "Standard_D96ls_v5": 5.088000, + "Standard_D96s_v5": 5.376000, + "Standard_DC16ads_cc_v5": 0.976000, + "Standard_DC16ads_v5": 0.976000, + "Standard_DC16as_cc_v5": 0.808000, + "Standard_DC16as_v5": 0.808000, + "Standard_DC16ds_v3": 2.128000, + "Standard_DC16s_v3": 1.792000, + "Standard_DC1ds_v3": 0.133000, + "Standard_DC1s_v2": 0.112000, + "Standard_DC1s_v3": 0.112000, + "Standard_DC24ds_v3": 3.192000, + "Standard_DC24s_v3": 2.688000, + "Standard_DC2ads_v5": 0.122000, + "Standard_DC2as_v5": 0.101000, + "Standard_DC2ds_v3": 0.266000, + "Standard_DC2s_v2": 0.224000, + "Standard_DC2s_v3": 0.224000, + "Standard_DC32ads_cc_v5": 1.952000, + "Standard_DC32ads_v5": 1.952000, + "Standard_DC32as_cc_v5": 1.616000, + "Standard_DC32as_v5": 1.616000, + "Standard_DC32ds_v3": 4.256000, + "Standard_DC32s_v3": 3.584000, + "Standard_DC48ads_cc_v5": 2.928000, + "Standard_DC48ads_v5": 2.928000, + "Standard_DC48as_cc_v5": 2.424000, + "Standard_DC48as_v5": 2.424000, + "Standard_DC48ds_v3": 6.384000, + "Standard_DC48s_v3": 5.376000, + "Standard_DC4ads_cc_v5": 0.244000, + "Standard_DC4ads_v5": 0.244000, + "Standard_DC4as_cc_v5": 0.202000, + "Standard_DC4as_v5": 0.202000, + "Standard_DC4ds_v3": 0.532000, + "Standard_DC4s_v2": 0.448000, + "Standard_DC4s_v3": 0.448000, + "Standard_DC64ads_cc_v5": 3.904000, + "Standard_DC64ads_v5": 3.904000, + "Standard_DC64as_cc_v5": 3.232000, + "Standard_DC64as_v5": 3.232000, + "Standard_DC8_v2": 0.896000, + "Standard_DC8ads_cc_v5": 0.488000, + "Standard_DC8ads_v5": 0.488000, + "Standard_DC8as_cc_v5": 0.404000, + "Standard_DC8as_v5": 0.404000, + "Standard_DC8ds_v3": 1.064000, + "Standard_DC8s_v3": 0.896000, + "Standard_DC96ads_cc_v5": 5.856000, + "Standard_DC96ads_v5": 5.856000, + "Standard_DC96as_cc_v5": 4.848000, + "Standard_DC96as_v5": 4.848000, + "Standard_DS1": 0.077000, + "Standard_DS11": 0.193000, + "Standard_DS11-1_v2": 0.185000, + "Standard_DS11_v2": 0.185000, + "Standard_DS11_v2_Promo": 0.185000, + "Standard_DS12": 0.386000, + "Standard_DS12-1_v2": 0.370000, + "Standard_DS12-2_v2": 0.370000, + "Standard_DS12_v2": 0.370000, + "Standard_DS12_v2_Promo": 0.370000, + "Standard_DS13": 0.771000, + "Standard_DS13-2_v2": 0.741000, + "Standard_DS13-4_v2": 0.741000, + "Standard_DS13_v2": 0.741000, + "Standard_DS13_v2_Promo": 0.741000, + "Standard_DS14": 1.542000, + "Standard_DS14-4_v2": 1.482000, + "Standard_DS14-8_v2": 1.482000, + "Standard_DS14_v2": 1.482000, + "Standard_DS14_v2_Promo": 1.482000, + "Standard_DS15_v2": 1.852000, + "Standard_DS15i_v2": 1.852000, + "Standard_DS1_v2": 0.070000, + "Standard_DS2": 0.154000, + "Standard_DS2_v2": 0.140000, + "Standard_DS2_v2_Promo": 0.140000, + "Standard_DS3": 0.308000, + "Standard_DS3_v2": 0.279000, + "Standard_DS3_v2_Promo": 0.279000, + "Standard_DS4": 0.616000, + "Standard_DS4_v2": 0.559000, + "Standard_DS4_v2_Promo": 0.559000, + "Standard_DS5_v2": 1.117000, + "Standard_DS5_v2_Promo": 1.117000, + "Standard_E104i_v5": 8.008000, + "Standard_E104id_v5": 9.266000, + "Standard_E104ids_v5": 9.266000, + "Standard_E104is_v5": 8.008000, + "Standard_E112iads_v5": 9.117000, + "Standard_E112ias_v5": 7.762000, + "Standard_E112ibds_v5": 11.642000, + "Standard_E112ibs_v5": 10.287000, + "Standard_E16-4ads_v5": 1.920000, + "Standard_E16-4as_v4": 1.120000, + "Standard_E16-4as_v5": 1.008000, + "Standard_E16-4ds_v4": 1.296000, + "Standard_E16-4ds_v5": 1.296000, + "Standard_E16-4s_v3": 1.120000, + "Standard_E16-4s_v4": 1.120000, + "Standard_E16-4s_v5": 1.120000, + "Standard_E16-8ads_v5": 1.920000, + "Standard_E16-8as_v4": 1.120000, + "Standard_E16-8as_v5": 1.744000, + "Standard_E16-8ds_v4": 1.296000, + "Standard_E16-8ds_v5": 1.296000, + "Standard_E16-8s_v3": 1.120000, + "Standard_E16-8s_v4": 1.120000, + "Standard_E16-8s_v5": 1.120000, + "Standard_E16_v3": 1.120000, + "Standard_E16_v4": 1.120000, + "Standard_E16_v5": 1.120000, + "Standard_E16a_v4": 1.120000, + "Standard_E16ads_v5": 1.230000, + "Standard_E16as_v4": 1.120000, + "Standard_E16as_v5": 1.054000, + "Standard_E16bds_v5": 1.512000, + "Standard_E16bs_v5": 1.336000, + "Standard_E16d_v4": 1.296000, + "Standard_E16d_v5": 1.296000, + "Standard_E16ds_v4": 1.296000, + "Standard_E16ds_v5": 1.296000, + "Standard_E16pds_v5": 1.040000, + "Standard_E16ps_v5": 0.896000, + "Standard_E16s_v3": 1.120000, + "Standard_E16s_v4": 1.120000, + "Standard_E16s_v5": 1.120000, + "Standard_E20_v3": 1.480000, + "Standard_E20_v4": 1.400000, + "Standard_E20_v5": 1.400000, + "Standard_E20a_v4": 1.400000, + "Standard_E20ads_v5": 1.526000, + "Standard_E20as_v4": 1.400000, + "Standard_E20as_v5": 1.306000, + "Standard_E20d_v4": 1.620000, + "Standard_E20d_v5": 1.620000, + "Standard_E20ds_v4": 1.620000, + "Standard_E20ds_v5": 1.620000, + "Standard_E20pds_v5": 1.300000, + "Standard_E20ps_v5": 1.120000, + "Standard_E20s_v3": 1.480000, + "Standard_E20s_v4": 1.400000, + "Standard_E20s_v5": 1.400000, + "Standard_E2_v3": 0.148000, + "Standard_E2_v4": 0.140000, + "Standard_E2_v5": 0.140000, + "Standard_E2a_v4": 0.140000, + "Standard_E2ads_v5": 0.194000, + "Standard_E2as_v4": 0.140000, + "Standard_E2as_v5": 0.172000, + "Standard_E2bds_v5": 0.189000, + "Standard_E2bs_v5": 0.167000, + "Standard_E2d_v4": 0.162000, + "Standard_E2d_v5": 0.162000, + "Standard_E2ds_v4": 0.162000, + "Standard_E2ds_v5": 0.162000, + "Standard_E2pds_v5": 0.130000, + "Standard_E2ps_v5": 0.112000, + "Standard_E2s_v3": 0.148000, + "Standard_E2s_v4": 0.140000, + "Standard_E2s_v5": 0.140000, + "Standard_E32-16ads_v5": 3.840000, + "Standard_E32-16as_v4": 2.240000, + "Standard_E32-16as_v5": 2.016000, + "Standard_E32-16ds_v4": 2.592000, + "Standard_E32-16ds_v5": 2.592000, + "Standard_E32-16s_v3": 2.240000, + "Standard_E32-16s_v4": 2.240000, + "Standard_E32-16s_v5": 2.240000, + "Standard_E32-8ads_v5": 3.840000, + "Standard_E32-8as_v4": 2.240000, + "Standard_E32-8as_v5": 3.488000, + "Standard_E32-8ds_v4": 2.592000, + "Standard_E32-8ds_v5": 2.592000, + "Standard_E32-8s_v3": 2.240000, + "Standard_E32-8s_v4": 2.240000, + "Standard_E32-8s_v5": 2.240000, + "Standard_E32_v3": 2.240000, + "Standard_E32_v4": 2.240000, + "Standard_E32_v5": 2.240000, + "Standard_E32a_v4": 2.240000, + "Standard_E32ads_v5": 2.414000, + "Standard_E32as_v4": 2.240000, + "Standard_E32as_v5": 2.062000, + "Standard_E32bds_v5": 3.024000, + "Standard_E32bs_v5": 2.672000, + "Standard_E32d_v4": 2.592000, + "Standard_E32d_v5": 2.592000, + "Standard_E32ds_v4": 2.592000, + "Standard_E32ds_v5": 2.592000, + "Standard_E32pds_v5": 2.080000, + "Standard_E32ps_v5": 1.792000, + "Standard_E32s_v3": 2.240000, + "Standard_E32s_v4": 2.240000, + "Standard_E32s_v5": 2.240000, + "Standard_E4-2ads_v5": 0.296000, + "Standard_E4-2as_v4": 0.280000, + "Standard_E4-2as_v5": 0.252000, + "Standard_E4-2ds_v4": 0.324000, + "Standard_E4-2ds_v5": 0.324000, + "Standard_E4-2s_v3": 0.296000, + "Standard_E4-2s_v4": 0.280000, + "Standard_E4-2s_v5": 0.280000, + "Standard_E48_v3": 3.360000, + "Standard_E48_v4": 3.360000, + "Standard_E48_v5": 3.360000, + "Standard_E48a_v4": 3.360000, + "Standard_E48ads_v5": 3.598000, + "Standard_E48as_v4": 3.360000, + "Standard_E48as_v5": 3.070000, + "Standard_E48bds_v5": 4.536000, + "Standard_E48bs_v5": 4.008000, + "Standard_E48d_v4": 3.888000, + "Standard_E48d_v5": 3.888000, + "Standard_E48ds_v4": 3.888000, + "Standard_E48ds_v5": 3.888000, + "Standard_E48s_v3": 3.360000, + "Standard_E48s_v4": 3.360000, + "Standard_E48s_v5": 3.360000, + "Standard_E4_v3": 0.296000, + "Standard_E4_v4": 0.280000, + "Standard_E4_v5": 0.280000, + "Standard_E4a_v4": 0.280000, + "Standard_E4ads_v5": 0.342000, + "Standard_E4as_v4": 0.280000, + "Standard_E4as_v5": 0.252000, + "Standard_E4bds_v5": 0.378000, + "Standard_E4bs_v5": 0.334000, + "Standard_E4d_v4": 0.324000, + "Standard_E4d_v5": 0.324000, + "Standard_E4ds_v4": 0.324000, + "Standard_E4ds_v5": 0.324000, + "Standard_E4pds_v5": 0.260000, + "Standard_E4ps_v5": 0.224000, + "Standard_E4s_v3": 0.296000, + "Standard_E4s_v4": 0.280000, + "Standard_E4s_v5": 0.280000, + "Standard_E64-16ads_v5": 4.736000, + "Standard_E64-16as_v4": 4.480000, + "Standard_E64-16as_v5": 4.032000, + "Standard_E64-16ds_v4": 5.184000, + "Standard_E64-16ds_v5": 5.184000, + "Standard_E64-16s_v3": 4.032000, + "Standard_E64-16s_v4": 4.480000, + "Standard_E64-16s_v5": 4.480000, + "Standard_E64-32ads_v5": 4.736000, + "Standard_E64-32as_v4": 4.480000, + "Standard_E64-32as_v5": 6.976000, + "Standard_E64-32ds_v4": 5.184000, + "Standard_E64-32ds_v5": 5.184000, + "Standard_E64-32s_v3": 4.032000, + "Standard_E64-32s_v4": 4.480000, + "Standard_E64-32s_v5": 4.480000, + "Standard_E64_v3": 4.032000, + "Standard_E64_v4": 4.480000, + "Standard_E64_v5": 4.480000, + "Standard_E64a_v4": 4.480000, + "Standard_E64ads_v5": 4.736000, + "Standard_E64as_v4": 4.480000, + "Standard_E64as_v5": 4.078000, + "Standard_E64bds_v5": 6.048000, + "Standard_E64bs_v5": 5.344000, + "Standard_E64d_v4": 5.184000, + "Standard_E64d_v5": 5.184000, + "Standard_E64ds_v4": 5.184000, + "Standard_E64ds_v5": 5.184000, + "Standard_E64i_v3": 4.032000, + "Standard_E64is_v3": 4.032000, + "Standard_E64s_v3": 4.032000, + "Standard_E64s_v4": 4.480000, + "Standard_E64s_v5": 4.480000, + "Standard_E8-2ads_v5": 0.960000, + "Standard_E8-2as_v4": 0.560000, + "Standard_E8-2as_v5": 0.872000, + "Standard_E8-2ds_v4": 0.648000, + "Standard_E8-2ds_v5": 0.648000, + "Standard_E8-2s_v3": 0.560000, + "Standard_E8-2s_v4": 0.560000, + "Standard_E8-2s_v5": 0.560000, + "Standard_E8-4ads_v5": 0.592000, + "Standard_E8-4as_v4": 0.560000, + "Standard_E8-4as_v5": 0.872000, + "Standard_E8-4ds_v4": 0.648000, + "Standard_E8-4ds_v5": 0.648000, + "Standard_E8-4s_v3": 0.560000, + "Standard_E8-4s_v4": 0.560000, + "Standard_E8-4s_v5": 0.560000, + "Standard_E80ids_v4": 6.480000, + "Standard_E80is_v4": 5.599000, + "Standard_E8_v3": 0.560000, + "Standard_E8_v4": 0.560000, + "Standard_E8_v5": 0.560000, + "Standard_E8a_v4": 0.560000, + "Standard_E8ads_v5": 0.592000, + "Standard_E8as_v4": 0.560000, + "Standard_E8as_v5": 0.504000, + "Standard_E8bds_v5": 0.756000, + "Standard_E8bs_v5": 0.668000, + "Standard_E8d_v4": 0.648000, + "Standard_E8d_v5": 0.648000, + "Standard_E8ds_v4": 0.648000, + "Standard_E8ds_v5": 0.648000, + "Standard_E8pds_v5": 0.520000, + "Standard_E8ps_v5": 0.448000, + "Standard_E8s_v3": 0.560000, + "Standard_E8s_v4": 0.560000, + "Standard_E8s_v5": 0.560000, + "Standard_E96-24ads_v5": 11.520000, + "Standard_E96-24as_v4": 6.720000, + "Standard_E96-24as_v5": 10.464000, + "Standard_E96-24ds_v5": 7.776000, + "Standard_E96-24s_v5": 6.720000, + "Standard_E96-48ads_v5": 11.520000, + "Standard_E96-48as_v4": 6.720000, + "Standard_E96-48as_v5": 6.048000, + "Standard_E96-48ds_v5": 7.776000, + "Standard_E96-48s_v5": 6.720000, + "Standard_E96_v5": 6.720000, + "Standard_E96a_v4": 6.720000, + "Standard_E96ads_v5": 7.150000, + "Standard_E96as_v4": 6.720000, + "Standard_E96as_v5": 6.048000, + "Standard_E96bds_v5": 9.072000, + "Standard_E96bs_v5": 8.016000, + "Standard_E96d_v5": 7.776000, + "Standard_E96ds_v5": 7.776000, + "Standard_E96iads_v5": 7.814000, + "Standard_E96ias_v5": 6.653000, + "Standard_E96s_v5": 6.720000, + "Standard_EC16ads_cc_v5": 1.184000, + "Standard_EC16ads_v5": 1.184000, + "Standard_EC16as_cc_v5": 1.008000, + "Standard_EC16as_v5": 1.008000, + "Standard_EC20ads_cc_v5": 1.480000, + "Standard_EC20ads_v5": 1.480000, + "Standard_EC20as_cc_v5": 1.260000, + "Standard_EC20as_v5": 1.260000, + "Standard_EC2ads_v5": 0.148000, + "Standard_EC2as_v5": 0.126000, + "Standard_EC32ads_cc_v5": 2.368000, + "Standard_EC32ads_v5": 2.368000, + "Standard_EC32as_cc_v5": 2.016000, + "Standard_EC32as_v5": 2.016000, + "Standard_EC48ads_cc_v5": 3.552000, + "Standard_EC48ads_v5": 3.552000, + "Standard_EC48as_cc_v5": 3.024000, + "Standard_EC48as_v5": 3.024000, + "Standard_EC4ads_cc_v5": 0.296000, + "Standard_EC4ads_v5": 0.296000, + "Standard_EC4as_cc_v5": 0.252000, + "Standard_EC4as_v5": 0.252000, + "Standard_EC64ads_cc_v5": 4.736000, + "Standard_EC64ads_v5": 4.736000, + "Standard_EC64as_cc_v5": 4.032000, + "Standard_EC64as_v5": 4.032000, + "Standard_EC8ads_cc_v5": 0.592000, + "Standard_EC8ads_v5": 0.592000, + "Standard_EC8as_cc_v5": 0.504000, + "Standard_EC8as_v5": 0.504000, + "Standard_EC96ads_cc_v5": 7.104000, + "Standard_EC96ads_v5": 7.104000, + "Standard_EC96as_cc_v5": 6.048000, + "Standard_EC96as_v5": 6.048000, + "Standard_EC96iads_v5": 7.814000, + "Standard_EC96ias_v5": 6.653000, + "Standard_F1": 0.062000, + "Standard_F16": 0.996000, + "Standard_F16s": 0.996000, + "Standard_F16s_v2": 0.848000, + "Standard_F1s": 0.062000, + "Standard_F2": 0.124000, + "Standard_F2s": 0.124000, + "Standard_F2s_v2": 0.106000, + "Standard_F32s_v2": 1.696000, + "Standard_F4": 0.249000, + "Standard_F48s_v2": 2.544000, + "Standard_F4s": 0.249000, + "Standard_F4s_v2": 0.212000, + "Standard_F64s_v2": 3.392000, + "Standard_F72s_v2": 3.816000, + "Standard_F8": 0.498000, + "Standard_F8s": 0.498000, + "Standard_F8s_v2": 0.424000, + "Standard_G1": 0.610000, + "Standard_G2": 1.220000, + "Standard_G3": 2.440000, + "Standard_G4": 4.880000, + "Standard_G5": 8.690000, + "Standard_GS1": 0.610000, + "Standard_GS2": 1.220000, + "Standard_GS3": 2.440000, + "Standard_GS4": 4.880000, + "Standard_GS4-4": 4.880000, + "Standard_GS4-8": 4.880000, + "Standard_GS5": 8.690000, + "Standard_GS5-16": 8.690000, + "Standard_GS5-8": 8.690000, + "Standard_H16": 1.941000, + "Standard_H16m": 2.601000, + "Standard_H16mr": 2.861000, + "Standard_H16r": 2.136000, + "Standard_H8": 0.971000, + "Standard_H8m": 1.301000, + "Standard_L16as_v3": 1.376000, + "Standard_L16s": 1.376000, + "Standard_L16s_v2": 1.376000, + "Standard_L16s_v3": 1.536000, + "Standard_L32as_v3": 2.752000, + "Standard_L32s": 2.752000, + "Standard_L32s_v2": 2.752000, + "Standard_L32s_v3": 3.072000, + "Standard_L48as_v3": 4.128000, + "Standard_L48s_v2": 4.128000, + "Standard_L48s_v3": 4.608000, + "Standard_L4s": 0.344000, + "Standard_L64as_v3": 5.504000, + "Standard_L64s_v2": 5.504000, + "Standard_L64s_v3": 6.144000, + "Standard_L80as_v3": 6.880000, + "Standard_L80s_v2": 6.880000, + "Standard_L80s_v3": 7.680000, + "Standard_L88is_v2": 7.568000, + "Standard_L8as_v3": 0.688000, + "Standard_L8s": 0.688000, + "Standard_L8s_v2": 0.688000, + "Standard_L8s_v3": 0.768000, + "Standard_L96s_v2": 8.256000, + "Standard_M128": 17.339000, + "Standard_M128-32ms": 34.694000, + "Standard_M128-64ms": 34.694000, + "Standard_M128dms_v2": 34.697000, + "Standard_M128ds_v2": 17.342000, + "Standard_M128m": 34.694000, + "Standard_M128ms": 34.694000, + "Standard_M128ms_v2": 34.150000, + "Standard_M128s": 17.339000, + "Standard_M128s_v2": 16.795000, + "Standard_M16-4ms": 3.995000, + "Standard_M16-8ms": 3.995000, + "Standard_M16ms": 3.995000, + "Standard_M16s": 3.103000, + "Standard_M192idms_v2": 41.683000, + "Standard_M192ids_v2": 20.841000, + "Standard_M192ims_v2": 41.136000, + "Standard_M192is_v2": 20.294000, + "Standard_M208-104ms_v2": 58.006000, + "Standard_M208-104s_v2": 29.003000, + "Standard_M208-52ms_v2": 58.006000, + "Standard_M208-52s_v2": 29.003000, + "Standard_M208ms_v2": 58.006000, + "Standard_M208s_v2": 29.003000, + "Standard_M32-16ms": 7.990000, + "Standard_M32-8ms": 7.990000, + "Standard_M32dms_v2": 7.990000, + "Standard_M32ls": 3.201000, + "Standard_M32ms": 7.990000, + "Standard_M32ms_v2": 7.853000, + "Standard_M32s": 6.206000, + "Standard_M32ts": 3.016000, + "Standard_M416-104ms_v2": 128.900000, + "Standard_M416-104s_v2": 64.454000, + "Standard_M416-208ms_v2": 128.900000, + "Standard_M416-208s_v2": 64.454000, + "Standard_M416is_v2": 64.454000, + "Standard_M416ms_v2": 128.900000, + "Standard_M416s_8_v2": 66.110000, + "Standard_M416s_v2": 64.454000, + "Standard_M64": 8.670000, + "Standard_M64-16ms": 17.347000, + "Standard_M64-32ms": 17.347000, + "Standard_M64dms_v2": 13.442000, + "Standard_M64ds_v2": 8.670000, + "Standard_M64ls": 6.403000, + "Standard_M64m": 17.347000, + "Standard_M64ms": 17.347000, + "Standard_M64ms_v2": 13.168000, + "Standard_M64s": 8.670000, + "Standard_M64s_v2": 8.396000, + "Standard_M8-2ms": 1.997000, + "Standard_M8-4ms": 1.997000, + "Standard_M8ms": 1.997000, + "Standard_NC12s_v3": 7.956000, + "Standard_NC16as_T4_v3": 1.445000, + "Standard_NC24ads_A100_v4": 4.775000, + "Standard_NC24rs_v3": 17.503000, + "Standard_NC24s_v3": 15.912000, + "Standard_NC48ads_A100_v4": 9.550000, + "Standard_NC4as_T4_v3": 0.631000, + "Standard_NC64as_T4_v3": 5.222000, + "Standard_NC6s_v3": 3.978000, + "Standard_NC8as_T4_v3": 0.902000, + "Standard_NC96ads_A100_v4": 19.100000, + "Standard_ND96ams_A100_v4": 42.601000, + "Standard_ND96amsr_A100_v4": 42.601000, + "Standard_NV12ads_A10_v5": 1.180000, + "Standard_NV12s_v2": 1.534000, + "Standard_NV12s_v3": 1.140000, + "Standard_NV18ads_A10_v5": 2.080000, + "Standard_NV24s_v2": 3.069000, + "Standard_NV24s_v3": 2.280000, + "Standard_NV36adms_A10_v5": 5.876000, + "Standard_NV36ads_A10_v5": 4.160000, + "Standard_NV48s_v3": 4.560000, + "Standard_NV6ads_A10_v5": 0.590000, + "Standard_NV6s_v2": 0.767000, + "Standard_NV72ads_A10_v5": 8.476000, + } + // westus2 + initialOnDemandPrices["westus2"] = map[string]float64{ + "Basic_A0": 0.018000, + "Basic_A1": 0.023000, + "Basic_A2": 0.068000, + "Basic_A3": 0.176000, + "Basic_A4": 0.352000, + "DCdsv3 Type1": 5.966000, + "DCsv2 Type 1": 0.845000, + "DCsv3 Type1": 5.069000, + "Dadsv5_Type1": 6.345000, + "Dasv4_Type1": 5.069000, + "Dasv4_Type2": 5.280000, + "Dasv5_Type1": 5.298000, + "Ddsv4_Type 1": 3.978000, + "Ddsv4_Type2": 4.723000, + "Ddsv5_Type1": 5.966000, + "Dsv3_Type3": 4.225000, + "Dsv3_Type4": 5.281000, + "Dsv4_Type1": 4.224000, + "Dsv4_Type2": 5.069000, + "Dsv5_Type1": 5.280000, + "Eadsv5_Type1": 6.917000, + "Easv4_Type1": 6.653000, + "Easv4_Type2": 6.653000, + "Easv5_Type1": 5.966000, + "Ebdsv5-Type1": 5.878000, + "Ebsv5-Type1": 5.245000, + "Edsv4_Type 1": 5.069000, + "Edsv4_Type2": 6.019000, + "Edsv5_Type1": 7.603000, + "Esv3_Type3": 4.297000, + "Esv3_Type4": 5.822000, + "Esv4_Type1": 4.435000, + "Esv4_Type2": 5.821000, + "Esv5_Type1": 6.653000, + "FXmds Type1": 4.910000, + "Fsv2 Type3": 3.927000, + "Fsv2_Type2": 3.366000, + "Fsv2_Type4": 4.468000, + "Lasv3_Type1": 6.864000, + "Lsv2_Type1": 6.864000, + "Lsv3_Type1": 7.678000, + "Mdmsv2MedMem _Type1": 29.359000, + "Mdsv2MedMem_Type1": 14.674000, + "Ms_Type1": 14.669000, + "Msm_Type1": 29.363000, + "Msmv2MedMem Type1": 28.896000, + "Msmv2_Type1": 109.060000, + "Msv2MedMem Type1": 14.211000, + "Msv2_Type1": 54.538000, + "NDasrA100v4_Type1": 29.917000, + "NVasv4_Type1": 7.176000, + "NVsv3_Type1": 5.016000, + "Standard_A0": 0.020000, + "Standard_A1": 0.051000, + "Standard_A1_v2": 0.036000, + "Standard_A2": 0.101000, + "Standard_A2_v2": 0.076000, + "Standard_A2m_v2": 0.099000, + "Standard_A3": 0.202000, + "Standard_A4": 0.405000, + "Standard_A4_v2": 0.159000, + "Standard_A4m_v2": 0.208000, + "Standard_A5": 0.220000, + "Standard_A6": 0.440000, + "Standard_A7": 0.880000, + "Standard_A8_v2": 0.333000, + "Standard_A8m_v2": 0.437000, + "Standard_B12ms": 0.499000, + "Standard_B16als_v2": 0.533000, + "Standard_B16as_v2": 0.675000, + "Standard_B16ls_v2": 0.663000, + "Standard_B16ms": 0.666000, + "Standard_B16pls_v2": 0.476000, + "Standard_B16ps_v2": 0.538000, + "Standard_B16s_v2": 0.739000, + "Standard_B1ls": 0.005200, + "Standard_B1ms": 0.020700, + "Standard_B1s": 0.010400, + "Standard_B20ms": 0.832000, + "Standard_B2als_v2": 0.046800, + "Standard_B2as_v2": 0.075200, + "Standard_B2ats_v2": 0.009400, + "Standard_B2ls_v2": 0.050800, + "Standard_B2ms": 0.083200, + "Standard_B2pls_v2": 0.033600, + "Standard_B2ps_v2": 0.067200, + "Standard_B2pts_v2": 0.008400, + "Standard_B2s": 0.041600, + "Standard_B2s_v2": 0.092400, + "Standard_B2ts_v2": 0.019600, + "Standard_B32als_v2": 1.213000, + "Standard_B32as_v2": 1.203000, + "Standard_B32ls_v2": 1.179000, + "Standard_B32s_v2": 1.478000, + "Standard_B4als_v2": 0.152000, + "Standard_B4as_v2": 0.169000, + "Standard_B4ls_v2": 0.147000, + "Standard_B4ms": 0.166000, + "Standard_B4pls_v2": 0.137000, + "Standard_B4ps_v2": 0.134000, + "Standard_B4s_v2": 0.185000, + "Standard_B8als_v2": 0.266000, + "Standard_B8as_v2": 0.301000, + "Standard_B8ls_v2": 0.332000, + "Standard_B8ms": 0.333000, + "Standard_B8pls_v2": 0.238000, + "Standard_B8ps_v2": 0.269000, + "Standard_B8s_v2": 0.370000, + "Standard_D11_v2": 0.149000, + "Standard_D11_v2_Promo": 0.149000, + "Standard_D12_v2": 0.299000, + "Standard_D12_v2_Promo": 0.299000, + "Standard_D13_v2": 0.598000, + "Standard_D13_v2_Promo": 0.598000, + "Standard_D14_v2": 1.196000, + "Standard_D14_v2_Promo": 1.196000, + "Standard_D15_v2": 1.495000, + "Standard_D15i_v2": 1.495000, + "Standard_D16_v3": 0.768000, + "Standard_D16_v4": 0.768000, + "Standard_D16_v5": 0.768000, + "Standard_D16a_v4": 0.768000, + "Standard_D16ads_v5": 0.824000, + "Standard_D16as_v4": 0.768000, + "Standard_D16as_v5": 0.688000, + "Standard_D16d_v4": 0.904000, + "Standard_D16d_v5": 0.904000, + "Standard_D16ds_v4": 0.904000, + "Standard_D16ds_v5": 0.904000, + "Standard_D16lds_v5": 0.768000, + "Standard_D16ls_v5": 0.680000, + "Standard_D16pds_v5": 0.723000, + "Standard_D16plds_v5": 0.614000, + "Standard_D16pls_v5": 0.544000, + "Standard_D16ps_v5": 0.616000, + "Standard_D16s_v3": 0.768000, + "Standard_D16s_v4": 0.768000, + "Standard_D16s_v5": 0.768000, + "Standard_D1_v2": 0.057000, + "Standard_D2_v2": 0.114000, + "Standard_D2_v2_Promo": 0.114000, + "Standard_D2_v3": 0.096000, + "Standard_D2_v4": 0.096000, + "Standard_D2_v5": 0.096000, + "Standard_D2a_v4": 0.096000, + "Standard_D2ads_v5": 0.195000, + "Standard_D2as_v4": 0.096000, + "Standard_D2as_v5": 0.178000, + "Standard_D2d_v4": 0.113000, + "Standard_D2d_v5": 0.113000, + "Standard_D2ds_v4": 0.113000, + "Standard_D2ds_v5": 0.113000, + "Standard_D2lds_v5": 0.096000, + "Standard_D2ls_v5": 0.085000, + "Standard_D2pds_v5": 0.090400, + "Standard_D2plds_v5": 0.076800, + "Standard_D2pls_v5": 0.068000, + "Standard_D2ps_v5": 0.077000, + "Standard_D2s_v3": 0.096000, + "Standard_D2s_v4": 0.096000, + "Standard_D2s_v5": 0.096000, + "Standard_D32-16s_v3": 1.536000, + "Standard_D32-8s_v3": 1.536000, + "Standard_D32_v3": 1.536000, + "Standard_D32_v4": 1.536000, + "Standard_D32_v5": 1.536000, + "Standard_D32a_v4": 1.536000, + "Standard_D32ads_v5": 3.120000, + "Standard_D32as_v4": 1.536000, + "Standard_D32as_v5": 2.848000, + "Standard_D32d_v4": 1.808000, + "Standard_D32d_v5": 1.808000, + "Standard_D32ds_v4": 1.808000, + "Standard_D32ds_v5": 1.808000, + "Standard_D32lds_v5": 1.536000, + "Standard_D32ls_v5": 1.360000, + "Standard_D32pds_v5": 1.446000, + "Standard_D32plds_v5": 1.229000, + "Standard_D32pls_v5": 1.088000, + "Standard_D32ps_v5": 1.232000, + "Standard_D32s_v3": 1.536000, + "Standard_D32s_v4": 1.536000, + "Standard_D32s_v5": 1.536000, + "Standard_D3_v2": 0.229000, + "Standard_D3_v2_Promo": 0.229000, + "Standard_D48_v3": 2.304000, + "Standard_D48_v4": 2.304000, + "Standard_D48_v5": 2.304000, + "Standard_D48a_v4": 2.304000, + "Standard_D48ads_v5": 2.472000, + "Standard_D48as_v4": 2.304000, + "Standard_D48as_v5": 4.272000, + "Standard_D48d_v4": 2.712000, + "Standard_D48d_v5": 2.712000, + "Standard_D48ds_v4": 2.712000, + "Standard_D48ds_v5": 2.712000, + "Standard_D48lds_v5": 2.304000, + "Standard_D48ls_v5": 2.040000, + "Standard_D48pds_v5": 2.170000, + "Standard_D48plds_v5": 1.843000, + "Standard_D48pls_v5": 1.632000, + "Standard_D48ps_v5": 1.848000, + "Standard_D48s_v3": 2.304000, + "Standard_D48s_v4": 2.304000, + "Standard_D48s_v5": 2.304000, + "Standard_D4_v2": 0.458000, + "Standard_D4_v2_Promo": 0.458000, + "Standard_D4_v3": 0.192000, + "Standard_D4_v4": 0.192000, + "Standard_D4_v5": 0.192000, + "Standard_D4a_v4": 0.192000, + "Standard_D4ads_v5": 0.206000, + "Standard_D4as_v4": 0.192000, + "Standard_D4as_v5": 0.172000, + "Standard_D4d_v4": 0.226000, + "Standard_D4d_v5": 0.226000, + "Standard_D4ds_v4": 0.226000, + "Standard_D4ds_v5": 0.226000, + "Standard_D4lds_v5": 0.192000, + "Standard_D4ls_v5": 0.170000, + "Standard_D4pds_v5": 0.181000, + "Standard_D4plds_v5": 0.154000, + "Standard_D4pls_v5": 0.136000, + "Standard_D4ps_v5": 0.154000, + "Standard_D4s_v3": 0.192000, + "Standard_D4s_v4": 0.192000, + "Standard_D4s_v5": 0.192000, + "Standard_D5_v2": 0.916000, + "Standard_D5_v2_Promo": 0.916000, + "Standard_D64-16s_v3": 3.072000, + "Standard_D64-32s_v3": 3.072000, + "Standard_D64_v3": 3.072000, + "Standard_D64_v4": 3.072000, + "Standard_D64_v5": 3.072000, + "Standard_D64a_v4": 3.072000, + "Standard_D64ads_v5": 3.296000, + "Standard_D64as_v4": 3.072000, + "Standard_D64as_v5": 5.696000, + "Standard_D64d_v4": 3.616000, + "Standard_D64d_v5": 3.616000, + "Standard_D64ds_v4": 3.616000, + "Standard_D64ds_v5": 3.616000, + "Standard_D64lds_v5": 3.072000, + "Standard_D64ls_v5": 2.720000, + "Standard_D64pds_v5": 2.893000, + "Standard_D64plds_v5": 2.458000, + "Standard_D64pls_v5": 2.176000, + "Standard_D64ps_v5": 2.464000, + "Standard_D64s_v3": 3.072000, + "Standard_D64s_v4": 3.072000, + "Standard_D64s_v5": 3.072000, + "Standard_D8_v3": 0.384000, + "Standard_D8_v4": 0.384000, + "Standard_D8_v5": 0.384000, + "Standard_D8a_v4": 0.384000, + "Standard_D8ads_v5": 0.412000, + "Standard_D8as_v4": 0.384000, + "Standard_D8as_v5": 0.344000, + "Standard_D8d_v4": 0.452000, + "Standard_D8d_v5": 0.452000, + "Standard_D8ds_v4": 0.452000, + "Standard_D8ds_v5": 0.452000, + "Standard_D8lds_v5": 0.384000, + "Standard_D8ls_v5": 0.340000, + "Standard_D8pds_v5": 0.362000, + "Standard_D8plds_v5": 0.307000, + "Standard_D8pls_v5": 0.272000, + "Standard_D8ps_v5": 0.308000, + "Standard_D8s_v3": 0.384000, + "Standard_D8s_v4": 0.384000, + "Standard_D8s_v5": 0.384000, + "Standard_D96_v5": 4.608000, + "Standard_D96a_v4": 4.608000, + "Standard_D96ads_v5": 4.944000, + "Standard_D96as_v4": 4.608000, + "Standard_D96as_v5": 4.128000, + "Standard_D96d_v5": 5.424000, + "Standard_D96ds_v5": 5.424000, + "Standard_D96lds_v5": 4.608000, + "Standard_D96ls_v5": 4.080000, + "Standard_D96s_v5": 4.608000, + "Standard_DC16ds_v3": 1.808000, + "Standard_DC16s_v3": 1.536000, + "Standard_DC1ds_v3": 0.113000, + "Standard_DC1s_v2": 0.096000, + "Standard_DC1s_v3": 0.096000, + "Standard_DC24ds_v3": 2.712000, + "Standard_DC24s_v3": 2.304000, + "Standard_DC2ds_v3": 0.226000, + "Standard_DC2s_v2": 0.192000, + "Standard_DC2s_v3": 0.192000, + "Standard_DC32ds_v3": 3.616000, + "Standard_DC32s_v3": 3.072000, + "Standard_DC48ds_v3": 5.424000, + "Standard_DC48s_v3": 4.608000, + "Standard_DC4ds_v3": 0.452000, + "Standard_DC4s_v2": 0.384000, + "Standard_DC4s_v3": 0.384000, + "Standard_DC8_v2": 0.768000, + "Standard_DC8ds_v3": 0.904000, + "Standard_DC8s_v3": 0.768000, + "Standard_DS11-1_v2": 0.149000, + "Standard_DS11_v2": 0.149000, + "Standard_DS11_v2_Promo": 0.149000, + "Standard_DS12-1_v2": 0.299000, + "Standard_DS12-2_v2": 0.299000, + "Standard_DS12_v2": 0.299000, + "Standard_DS12_v2_Promo": 0.299000, + "Standard_DS13-2_v2": 0.598000, + "Standard_DS13-4_v2": 0.598000, + "Standard_DS13_v2": 0.598000, + "Standard_DS13_v2_Promo": 0.598000, + "Standard_DS14-4_v2": 1.196000, + "Standard_DS14-8_v2": 1.196000, + "Standard_DS14_v2": 1.196000, + "Standard_DS14_v2_Promo": 1.196000, + "Standard_DS15_v2": 1.495000, + "Standard_DS15i_v2": 1.495000, + "Standard_DS1_v2": 0.057000, + "Standard_DS2_v2": 0.114000, + "Standard_DS2_v2_Promo": 0.114000, + "Standard_DS3_v2": 0.229000, + "Standard_DS3_v2_Promo": 0.229000, + "Standard_DS4_v2": 0.458000, + "Standard_DS4_v2_Promo": 0.458000, + "Standard_DS5_v2": 0.916000, + "Standard_DS5_v2_Promo": 0.916000, + "Standard_E104i_v5": 7.207000, + "Standard_E104id_v5": 8.237000, + "Standard_E104ids_v5": 8.237000, + "Standard_E104is_v5": 7.207000, + "Standard_E112iads_v5": 8.070000, + "Standard_E112ias_v5": 6.961000, + "Standard_E112ibds_v5": 10.287000, + "Standard_E112ibs_v5": 9.178000, + "Standard_E16-4ads_v5": 1.048000, + "Standard_E16-4as_v4": 1.008000, + "Standard_E16-4as_v5": 0.904000, + "Standard_E16-4ds_v4": 1.152000, + "Standard_E16-4ds_v5": 1.152000, + "Standard_E16-4s_v3": 1.008000, + "Standard_E16-4s_v4": 1.008000, + "Standard_E16-4s_v5": 1.008000, + "Standard_E16-8ads_v5": 1.784000, + "Standard_E16-8as_v4": 1.008000, + "Standard_E16-8as_v5": 0.904000, + "Standard_E16-8ds_v4": 1.152000, + "Standard_E16-8ds_v5": 1.152000, + "Standard_E16-8s_v3": 1.008000, + "Standard_E16-8s_v4": 1.008000, + "Standard_E16-8s_v5": 1.008000, + "Standard_E16_v3": 1.008000, + "Standard_E16_v4": 1.008000, + "Standard_E16_v5": 1.008000, + "Standard_E16a_v4": 1.008000, + "Standard_E16ads_v5": 1.094000, + "Standard_E16as_v4": 1.008000, + "Standard_E16as_v5": 0.904000, + "Standard_E16bds_v5": 1.336000, + "Standard_E16bs_v5": 1.192000, + "Standard_E16d_v4": 1.152000, + "Standard_E16d_v5": 1.152000, + "Standard_E16ds_v4": 1.152000, + "Standard_E16ds_v5": 1.152000, + "Standard_E16pds_v5": 0.922000, + "Standard_E16ps_v5": 0.806000, + "Standard_E16s_v3": 1.008000, + "Standard_E16s_v4": 1.008000, + "Standard_E16s_v5": 1.008000, + "Standard_E20_v3": 1.260000, + "Standard_E20_v4": 1.260000, + "Standard_E20_v5": 1.260000, + "Standard_E20a_v4": 1.260000, + "Standard_E20ads_v5": 1.356000, + "Standard_E20as_v4": 1.260000, + "Standard_E20as_v5": 1.130000, + "Standard_E20d_v4": 1.440000, + "Standard_E20d_v5": 1.440000, + "Standard_E20ds_v4": 1.440000, + "Standard_E20ds_v5": 1.440000, + "Standard_E20pds_v5": 1.152000, + "Standard_E20ps_v5": 1.008000, + "Standard_E20s_v3": 1.260000, + "Standard_E20s_v4": 1.260000, + "Standard_E20s_v5": 1.260000, + "Standard_E2_v3": 0.126000, + "Standard_E2_v4": 0.126000, + "Standard_E2_v5": 0.126000, + "Standard_E2a_v4": 0.126000, + "Standard_E2ads_v5": 0.131000, + "Standard_E2as_v4": 0.126000, + "Standard_E2as_v5": 0.159000, + "Standard_E2bds_v5": 0.167000, + "Standard_E2bs_v5": 0.149000, + "Standard_E2d_v4": 0.144000, + "Standard_E2d_v5": 0.144000, + "Standard_E2ds_v4": 0.144000, + "Standard_E2ds_v5": 0.144000, + "Standard_E2pds_v5": 0.115000, + "Standard_E2ps_v5": 0.101000, + "Standard_E2s_v3": 0.126000, + "Standard_E2s_v4": 0.126000, + "Standard_E2s_v5": 0.126000, + "Standard_E32-16ads_v5": 3.568000, + "Standard_E32-16as_v4": 2.016000, + "Standard_E32-16as_v5": 1.808000, + "Standard_E32-16ds_v4": 2.304000, + "Standard_E32-16ds_v5": 2.304000, + "Standard_E32-16s_v3": 2.016000, + "Standard_E32-16s_v4": 2.016000, + "Standard_E32-16s_v5": 2.016000, + "Standard_E32-8ads_v5": 2.096000, + "Standard_E32-8as_v4": 2.016000, + "Standard_E32-8as_v5": 1.808000, + "Standard_E32-8ds_v4": 2.304000, + "Standard_E32-8ds_v5": 2.304000, + "Standard_E32-8s_v3": 2.016000, + "Standard_E32-8s_v4": 2.016000, + "Standard_E32-8s_v5": 2.016000, + "Standard_E32_v3": 2.016000, + "Standard_E32_v4": 2.016000, + "Standard_E32_v5": 2.016000, + "Standard_E32a_v4": 2.016000, + "Standard_E32ads_v5": 2.096000, + "Standard_E32as_v4": 2.016000, + "Standard_E32as_v5": 1.854000, + "Standard_E32bds_v5": 2.672000, + "Standard_E32bs_v5": 2.384000, + "Standard_E32d_v4": 2.304000, + "Standard_E32d_v5": 2.304000, + "Standard_E32ds_v4": 2.304000, + "Standard_E32ds_v5": 2.304000, + "Standard_E32pds_v5": 1.843000, + "Standard_E32ps_v5": 1.613000, + "Standard_E32s_v3": 2.016000, + "Standard_E32s_v4": 2.016000, + "Standard_E32s_v5": 2.016000, + "Standard_E4-2ads_v5": 0.446000, + "Standard_E4-2as_v4": 0.252000, + "Standard_E4-2as_v5": 0.410000, + "Standard_E4-2ds_v4": 0.288000, + "Standard_E4-2ds_v5": 0.288000, + "Standard_E4-2s_v3": 0.252000, + "Standard_E4-2s_v4": 0.252000, + "Standard_E4-2s_v5": 0.252000, + "Standard_E48_v3": 3.024000, + "Standard_E48_v4": 3.024000, + "Standard_E48_v5": 3.024000, + "Standard_E48a_v4": 3.024000, + "Standard_E48ads_v5": 3.190000, + "Standard_E48as_v4": 3.024000, + "Standard_E48as_v5": 2.758000, + "Standard_E48bds_v5": 4.008000, + "Standard_E48bs_v5": 3.576000, + "Standard_E48d_v4": 3.456000, + "Standard_E48d_v5": 3.456000, + "Standard_E48ds_v4": 3.456000, + "Standard_E48ds_v5": 3.456000, + "Standard_E48s_v3": 3.024000, + "Standard_E48s_v4": 3.024000, + "Standard_E48s_v5": 3.024000, + "Standard_E4_v3": 0.252000, + "Standard_E4_v4": 0.252000, + "Standard_E4_v5": 0.252000, + "Standard_E4a_v4": 0.252000, + "Standard_E4ads_v5": 0.262000, + "Standard_E4as_v4": 0.252000, + "Standard_E4as_v5": 0.272000, + "Standard_E4bds_v5": 0.334000, + "Standard_E4bs_v5": 0.298000, + "Standard_E4d_v4": 0.288000, + "Standard_E4d_v5": 0.288000, + "Standard_E4ds_v4": 0.288000, + "Standard_E4ds_v5": 0.288000, + "Standard_E4pds_v5": 0.230000, + "Standard_E4ps_v5": 0.202000, + "Standard_E4s_v3": 0.252000, + "Standard_E4s_v4": 0.252000, + "Standard_E4s_v5": 0.252000, + "Standard_E64-16ads_v5": 4.192000, + "Standard_E64-16as_v4": 4.032000, + "Standard_E64-16as_v5": 3.616000, + "Standard_E64-16ds_v4": 4.608000, + "Standard_E64-16ds_v5": 4.608000, + "Standard_E64-16s_v3": 3.629000, + "Standard_E64-16s_v4": 4.032000, + "Standard_E64-16s_v5": 4.032000, + "Standard_E64-32ads_v5": 7.136000, + "Standard_E64-32as_v4": 4.032000, + "Standard_E64-32as_v5": 6.560000, + "Standard_E64-32ds_v4": 4.608000, + "Standard_E64-32ds_v5": 4.608000, + "Standard_E64-32s_v3": 3.629000, + "Standard_E64-32s_v4": 4.032000, + "Standard_E64-32s_v5": 4.032000, + "Standard_E64_v3": 3.629000, + "Standard_E64_v4": 4.032000, + "Standard_E64_v5": 4.032000, + "Standard_E64a_v4": 4.032000, + "Standard_E64ads_v5": 4.238000, + "Standard_E64as_v4": 4.032000, + "Standard_E64as_v5": 3.616000, + "Standard_E64bds_v5": 5.344000, + "Standard_E64bs_v5": 4.768000, + "Standard_E64d_v4": 4.608000, + "Standard_E64d_v5": 4.608000, + "Standard_E64ds_v4": 4.608000, + "Standard_E64ds_v5": 4.608000, + "Standard_E64i_v3": 3.629000, + "Standard_E64is_v3": 3.629000, + "Standard_E64s_v3": 3.629000, + "Standard_E64s_v4": 4.032000, + "Standard_E64s_v5": 4.032000, + "Standard_E8-2ads_v5": 0.892000, + "Standard_E8-2as_v4": 0.504000, + "Standard_E8-2as_v5": 0.820000, + "Standard_E8-2ds_v4": 0.576000, + "Standard_E8-2ds_v5": 0.576000, + "Standard_E8-2s_v3": 0.504000, + "Standard_E8-2s_v4": 0.504000, + "Standard_E8-2s_v5": 0.504000, + "Standard_E8-4ads_v5": 0.892000, + "Standard_E8-4as_v4": 0.504000, + "Standard_E8-4as_v5": 0.452000, + "Standard_E8-4ds_v4": 0.576000, + "Standard_E8-4ds_v5": 0.576000, + "Standard_E8-4s_v3": 0.504000, + "Standard_E8-4s_v4": 0.504000, + "Standard_E8-4s_v5": 0.504000, + "Standard_E80ids_v4": 5.760000, + "Standard_E80is_v4": 5.040000, + "Standard_E8_v3": 0.504000, + "Standard_E8_v4": 0.504000, + "Standard_E8_v5": 0.504000, + "Standard_E8a_v4": 0.504000, + "Standard_E8ads_v5": 0.570000, + "Standard_E8as_v4": 0.504000, + "Standard_E8as_v5": 0.498000, + "Standard_E8bds_v5": 0.668000, + "Standard_E8bs_v5": 0.596000, + "Standard_E8d_v4": 0.576000, + "Standard_E8d_v5": 0.576000, + "Standard_E8ds_v4": 0.576000, + "Standard_E8ds_v5": 0.576000, + "Standard_E8pds_v5": 0.461000, + "Standard_E8ps_v5": 0.403000, + "Standard_E8s_v3": 0.504000, + "Standard_E8s_v4": 0.504000, + "Standard_E8s_v5": 0.504000, + "Standard_E96-24ads_v5": 10.704000, + "Standard_E96-24as_v4": 6.048000, + "Standard_E96-24as_v5": 5.424000, + "Standard_E96-24ds_v5": 6.912000, + "Standard_E96-24s_v5": 6.048000, + "Standard_E96-48ads_v5": 6.288000, + "Standard_E96-48as_v4": 6.048000, + "Standard_E96-48as_v5": 5.424000, + "Standard_E96-48ds_v5": 6.912000, + "Standard_E96-48s_v5": 6.048000, + "Standard_E96_v5": 6.048000, + "Standard_E96a_v4": 6.048000, + "Standard_E96ads_v5": 6.288000, + "Standard_E96as_v4": 6.048000, + "Standard_E96as_v5": 5.470000, + "Standard_E96bds_v5": 8.016000, + "Standard_E96bs_v5": 7.152000, + "Standard_E96d_v5": 6.912000, + "Standard_E96ds_v5": 6.912000, + "Standard_E96iads_v5": 6.917000, + "Standard_E96ias_v5": 5.966000, + "Standard_E96s_v5": 6.048000, + "Standard_F1": 0.049700, + "Standard_F16": 0.796000, + "Standard_F16s": 0.796000, + "Standard_F16s_v2": 0.677000, + "Standard_F1s": 0.049700, + "Standard_F2": 0.099000, + "Standard_F2s": 0.099000, + "Standard_F2s_v2": 0.084600, + "Standard_F32s_v2": 1.353000, + "Standard_F4": 0.199000, + "Standard_F48s_v2": 2.030000, + "Standard_F4s": 0.199000, + "Standard_F4s_v2": 0.169000, + "Standard_F64s_v2": 2.706000, + "Standard_F72s_v2": 3.045000, + "Standard_F8": 0.398000, + "Standard_F8s": 0.398000, + "Standard_F8s_v2": 0.338000, + "Standard_FX12mds": 1.116000, + "Standard_FX24mds": 2.232000, + "Standard_FX36mds": 3.348000, + "Standard_FX48mds": 4.464000, + "Standard_FX4mds": 0.372000, + "Standard_G1": 0.490000, + "Standard_G2": 0.981000, + "Standard_G3": 1.961000, + "Standard_G4": 3.922000, + "Standard_G5": 7.820000, + "Standard_GS1": 0.490000, + "Standard_GS2": 0.981000, + "Standard_GS3": 1.961000, + "Standard_GS4": 3.922000, + "Standard_GS4-4": 3.922000, + "Standard_GS4-8": 3.922000, + "Standard_GS5": 7.820000, + "Standard_GS5-16": 7.820000, + "Standard_GS5-8": 7.820000, + "Standard_H16": 1.591000, + "Standard_H16m": 2.132000, + "Standard_H16mr": 2.345000, + "Standard_H16r": 1.750000, + "Standard_H8": 0.796000, + "Standard_H8m": 1.066000, + "Standard_HB120-16rs_v2": 3.600000, + "Standard_HB120-32rs_v2": 3.600000, + "Standard_HB120-64rs_v2": 3.600000, + "Standard_HB120-96rs_v2": 3.600000, + "Standard_HB120rs_v2": 3.600000, + "Standard_HC44-16rs": 3.168000, + "Standard_HC44-32rs": 3.168000, + "Standard_HC44rs": 3.168000, + "Standard_L16as_v3": 1.248000, + "Standard_L16s": 1.248000, + "Standard_L16s_v2": 1.248000, + "Standard_L16s_v3": 1.396000, + "Standard_L32as_v3": 2.496000, + "Standard_L32s": 2.496000, + "Standard_L32s_v2": 2.496000, + "Standard_L32s_v3": 2.792000, + "Standard_L48as_v3": 3.744000, + "Standard_L48s_v2": 3.744000, + "Standard_L48s_v3": 4.188000, + "Standard_L4s": 0.312000, + "Standard_L64as_v3": 4.992000, + "Standard_L64s_v2": 4.992000, + "Standard_L64s_v3": 5.584000, + "Standard_L80as_v3": 6.240000, + "Standard_L80s_v2": 6.240000, + "Standard_L80s_v3": 6.980000, + "Standard_L88is_v2": 6.864000, + "Standard_L8as_v3": 0.624000, + "Standard_L8s": 0.624000, + "Standard_L8s_v2": 0.624000, + "Standard_L8s_v3": 0.698000, + "Standard_M128": 13.338000, + "Standard_M128-32ms": 26.688000, + "Standard_M128-64ms": 26.688000, + "Standard_M128dms_v2": 26.690000, + "Standard_M128ds_v2": 13.340000, + "Standard_M128m": 26.688000, + "Standard_M128ms": 26.688000, + "Standard_M128ms_v2": 26.269000, + "Standard_M128s": 13.338000, + "Standard_M128s_v2": 12.919000, + "Standard_M16-4ms": 3.073000, + "Standard_M16-8ms": 3.073000, + "Standard_M16ms": 3.073000, + "Standard_M16s": 2.387000, + "Standard_M192idms_v2": 32.064000, + "Standard_M192ids_v2": 16.032000, + "Standard_M192ims_v2": 31.643000, + "Standard_M192is_v2": 15.611000, + "Standard_M208-104ms_v2": 44.620000, + "Standard_M208-104s_v2": 22.310000, + "Standard_M208-52ms_v2": 44.620000, + "Standard_M208-52s_v2": 22.310000, + "Standard_M208ms_v2": 44.620000, + "Standard_M208s_v2": 22.310000, + "Standard_M32-16ms": 6.146000, + "Standard_M32-8ms": 6.146000, + "Standard_M32dms_v2": 6.146000, + "Standard_M32ls": 2.873000, + "Standard_M32ms": 6.146000, + "Standard_M32ms_v2": 6.041000, + "Standard_M32s": 3.335000, + "Standard_M32ts": 2.707000, + "Standard_M416-104ms_v2": 99.150000, + "Standard_M416-104s_v2": 49.580000, + "Standard_M416-208ms_v2": 99.150000, + "Standard_M416-208s_v2": 49.580000, + "Standard_M416is_v2": 49.580000, + "Standard_M416ms_v2": 99.150000, + "Standard_M416s_8_v2": 66.110000, + "Standard_M416s_v2": 49.580000, + "Standard_M64": 6.669000, + "Standard_M64-16ms": 10.337000, + "Standard_M64-32ms": 10.337000, + "Standard_M64dms_v2": 10.340000, + "Standard_M64ds_v2": 6.669000, + "Standard_M64ls": 5.415000, + "Standard_M64m": 10.337000, + "Standard_M64ms": 10.337000, + "Standard_M64ms_v2": 10.130000, + "Standard_M64s": 6.669000, + "Standard_M64s_v2": 6.459000, + "Standard_M8-2ms": 1.536500, + "Standard_M8-4ms": 1.536500, + "Standard_M8ms": 1.536500, + "Standard_NC12": 1.800000, + "Standard_NC12s_v2": 4.140000, + "Standard_NC12s_v3": 6.120000, + "Standard_NC16as_T4_v3": 1.204000, + "Standard_NC24": 3.600000, + "Standard_NC24ads_A100_v4": 3.673000, + "Standard_NC24r": 3.960000, + "Standard_NC24rs_v2": 9.108000, + "Standard_NC24rs_v3": 13.460000, + "Standard_NC24s_v2": 8.280000, + "Standard_NC24s_v3": 12.240000, + "Standard_NC48ads_A100_v4": 7.346000, + "Standard_NC4as_T4_v3": 0.526000, + "Standard_NC6": 0.900000, + "Standard_NC64as_T4_v3": 4.352000, + "Standard_NC6s_v2": 2.070000, + "Standard_NC6s_v3": 3.060000, + "Standard_NC8as_T4_v3": 0.752000, + "Standard_NC96ads_A100_v4": 14.692000, + "Standard_ND12s": 4.140000, + "Standard_ND24rs": 9.108000, + "Standard_ND24s": 8.280000, + "Standard_ND40rs_v2": 22.032000, + "Standard_ND6s": 2.070000, + "Standard_ND96amsr_A100_v4": 32.770000, + "Standard_ND96asr_A100_v4": 27.197000, + "Standard_ND96asr_v4": 27.197000, + "Standard_NP10s": 1.650000, + "Standard_NP20s": 3.300000, + "Standard_NP40s": 6.600000, + "Standard_NV12": 2.185000, + "Standard_NV12ads_A10_v5": 0.908000, + "Standard_NV12s_v3": 1.140000, + "Standard_NV16as_v4": 0.932000, + "Standard_NV18ads_A10_v5": 1.600000, + "Standard_NV24": 4.370000, + "Standard_NV24s_v3": 2.280000, + "Standard_NV32as_v4": 1.864000, + "Standard_NV36adms_A10_v5": 4.520000, + "Standard_NV36ads_A10_v5": 3.200000, + "Standard_NV48s_v3": 4.560000, + "Standard_NV4as_v4": 0.233000, + "Standard_NV6": 1.092000, + "Standard_NV6ads_A10_v5": 0.454000, + "Standard_NV72ads_A10_v5": 6.520000, + "Standard_NV8as_v4": 0.466000, + "Standard_PB12s": 1.660000, + "Standard_PB24s": 3.320000, + "Standard_PB6s": 0.830000, + } + // westus3 + initialOnDemandPrices["westus3"] = map[string]float64{ + "Dadsv5_Type1": 6.345000, + "Dasv4_Type1": 5.280000, + "Dasv4_Type2": 5.280000, + "Dasv5_Type1": 5.298000, + "Ddsv4_Type 1": 3.978000, + "Ddsv4_Type2": 4.723000, + "Ddsv5_Type1": 5.966000, + "Dsv3_Type3": 4.225000, + "Dsv3_Type4": 5.281000, + "Dsv4_Type1": 4.224000, + "Dsv4_Type2": 5.069000, + "Dsv5_Type1": 5.280000, + "Eadsv5_Type1": 6.917000, + "Easv4_Type1": 6.653000, + "Easv4_Type2": 6.653000, + "Easv5_Type1": 5.966000, + "Ebdsv5-Type1": 5.878000, + "Ebsv5-Type1": 5.245000, + "Edsv4_Type 1": 5.069000, + "Edsv4_Type2": 6.019000, + "Edsv5_Type1": 7.603000, + "Esv3_Type3": 5.545000, + "Esv3_Type4": 5.822000, + "Esv4_Type1": 4.435000, + "Esv4_Type2": 5.821000, + "Esv5_Type1": 6.653000, + "FXmds Type1": 5.570000, + "Fsv2 Type3": 3.927000, + "Fsv2_Type2": 3.366000, + "Fsv2_Type4": 4.468000, + "Lasv3_Type1": 6.864000, + "Lsv2_Type1": 6.864000, + "Lsv3_Type1": 7.678000, + "Mdmsv2MedMem _Type1": 29.359000, + "Mdsv2MedMem_Type1": 14.674000, + "Msmv2MedMem Type1": 28.896000, + "Msmv2_Type1": 109.060000, + "Msv2MedMem Type1": 14.211000, + "Msv2_Type1": 54.538000, + "NDamsrA100v4_Type1": 36.047000, + "NDasrA100v4_Type1": 29.917000, + "NVasv4_Type1": 7.176000, + "Standard_A1_v2": 0.036000, + "Standard_A2_v2": 0.076000, + "Standard_A2m_v2": 0.099000, + "Standard_A4_v2": 0.159000, + "Standard_A4m_v2": 0.208000, + "Standard_A8_v2": 0.333000, + "Standard_A8m_v2": 0.437000, + "Standard_B12ms": 0.499000, + "Standard_B16als_v2": 0.533000, + "Standard_B16as_v2": 0.602000, + "Standard_B16ls_v2": 0.663000, + "Standard_B16ms": 0.666000, + "Standard_B16pls_v2": 0.425000, + "Standard_B16ps_v2": 0.554000, + "Standard_B16s_v2": 0.666000, + "Standard_B1ls": 0.005200, + "Standard_B1ms": 0.020700, + "Standard_B1s": 0.010400, + "Standard_B20ms": 0.832000, + "Standard_B2als_v2": 0.046800, + "Standard_B2as_v2": 0.075200, + "Standard_B2ats_v2": 0.018600, + "Standard_B2ls_v2": 0.050800, + "Standard_B2ms": 0.083200, + "Standard_B2pls_v2": 0.039200, + "Standard_B2ps_v2": 0.069200, + "Standard_B2pts_v2": 0.016700, + "Standard_B2s": 0.041600, + "Standard_B2s_v2": 0.092400, + "Standard_B2ts_v2": 0.010400, + "Standard_B32als_v2": 1.065000, + "Standard_B32as_v2": 1.203000, + "Standard_B32ls_v2": 1.326000, + "Standard_B32s_v2": 1.331000, + "Standard_B4als_v2": 0.133000, + "Standard_B4as_v2": 0.169000, + "Standard_B4ls_v2": 0.147000, + "Standard_B4ms": 0.166000, + "Standard_B4pls_v2": 0.106000, + "Standard_B4ps_v2": 0.138000, + "Standard_B4s_v2": 0.166000, + "Standard_B8als_v2": 0.266000, + "Standard_B8as_v2": 0.338000, + "Standard_B8ls_v2": 0.295000, + "Standard_B8ms": 0.333000, + "Standard_B8pls_v2": 0.213000, + "Standard_B8ps_v2": 0.277000, + "Standard_B8s_v2": 0.370000, + "Standard_D1": 0.077000, + "Standard_D11": 0.192000, + "Standard_D11_v2": 0.149000, + "Standard_D12": 0.385000, + "Standard_D12_v2": 0.299000, + "Standard_D13": 0.770000, + "Standard_D13_v2": 0.598000, + "Standard_D14": 1.540000, + "Standard_D14_v2": 1.196000, + "Standard_D15_v2": 1.495000, + "Standard_D15i_v2": 1.495000, + "Standard_D16_v3": 0.768000, + "Standard_D16_v4": 0.768000, + "Standard_D16_v5": 0.768000, + "Standard_D16a_v4": 0.768000, + "Standard_D16ads_v5": 0.824000, + "Standard_D16as_v4": 0.768000, + "Standard_D16as_v5": 0.688000, + "Standard_D16as_v5_Promo": 0.717000, + "Standard_D16d_v4": 0.904000, + "Standard_D16d_v5": 0.904000, + "Standard_D16ds_v4": 0.904000, + "Standard_D16ds_v5": 0.904000, + "Standard_D16lds_v5": 0.768000, + "Standard_D16ls_v5": 0.680000, + "Standard_D16pds_v5": 0.723000, + "Standard_D16plds_v5": 0.614000, + "Standard_D16pls_v5": 0.544000, + "Standard_D16ps_v5": 0.616000, + "Standard_D16s_v3": 0.768000, + "Standard_D16s_v4": 0.768000, + "Standard_D16s_v5": 0.768000, + "Standard_D1_v2": 0.057000, + "Standard_D2": 0.154000, + "Standard_D2_v2": 0.114000, + "Standard_D2_v3": 0.096000, + "Standard_D2_v4": 0.096000, + "Standard_D2_v5": 0.096000, + "Standard_D2a_v4": 0.096000, + "Standard_D2ads_v5": 0.103000, + "Standard_D2as_v4": 0.096000, + "Standard_D2as_v5": 0.086000, + "Standard_D2as_v5_Promo": 0.089600, + "Standard_D2d_v4": 0.113000, + "Standard_D2d_v5": 0.113000, + "Standard_D2ds_v4": 0.113000, + "Standard_D2ds_v5": 0.113000, + "Standard_D2lds_v5": 0.096000, + "Standard_D2ls_v5": 0.085000, + "Standard_D2pds_v5": 0.090400, + "Standard_D2plds_v5": 0.076800, + "Standard_D2pls_v5": 0.068000, + "Standard_D2ps_v5": 0.077000, + "Standard_D2s_v3": 0.096000, + "Standard_D2s_v4": 0.096000, + "Standard_D2s_v5": 0.096000, + "Standard_D3": 0.308000, + "Standard_D32-16s_v3": 1.536000, + "Standard_D32-8s_v3": 1.536000, + "Standard_D32_v3": 1.536000, + "Standard_D32_v4": 1.536000, + "Standard_D32_v5": 1.536000, + "Standard_D32a_v4": 1.536000, + "Standard_D32ads_v5": 3.120000, + "Standard_D32as_v4": 1.536000, + "Standard_D32as_v5": 2.848000, + "Standard_D32as_v5_Promo": 1.434000, + "Standard_D32d_v4": 1.808000, + "Standard_D32d_v5": 1.808000, + "Standard_D32ds_v4": 1.808000, + "Standard_D32ds_v5": 1.808000, + "Standard_D32lds_v5": 1.536000, + "Standard_D32ls_v5": 1.360000, + "Standard_D32pds_v5": 1.446000, + "Standard_D32plds_v5": 1.229000, + "Standard_D32pls_v5": 1.088000, + "Standard_D32ps_v5": 1.232000, + "Standard_D32s_v3": 1.536000, + "Standard_D32s_v4": 1.536000, + "Standard_D32s_v5": 1.536000, + "Standard_D3_v2": 0.229000, + "Standard_D4": 0.616000, + "Standard_D48_v3": 2.304000, + "Standard_D48_v4": 2.304000, + "Standard_D48_v5": 2.304000, + "Standard_D48a_v4": 2.304000, + "Standard_D48ads_v5": 4.680000, + "Standard_D48as_v4": 2.304000, + "Standard_D48as_v5": 4.272000, + "Standard_D48as_v5_Promo": 2.150000, + "Standard_D48d_v4": 2.712000, + "Standard_D48d_v5": 2.712000, + "Standard_D48ds_v4": 2.712000, + "Standard_D48ds_v5": 2.712000, + "Standard_D48lds_v5": 2.304000, + "Standard_D48ls_v5": 2.040000, + "Standard_D48pds_v5": 2.170000, + "Standard_D48plds_v5": 1.843000, + "Standard_D48pls_v5": 1.632000, + "Standard_D48ps_v5": 1.848000, + "Standard_D48s_v3": 2.304000, + "Standard_D48s_v4": 2.304000, + "Standard_D48s_v5": 2.304000, + "Standard_D4_v2": 0.458000, + "Standard_D4_v3": 0.192000, + "Standard_D4_v4": 0.192000, + "Standard_D4_v5": 0.192000, + "Standard_D4a_v4": 0.192000, + "Standard_D4ads_v5": 0.390000, + "Standard_D4as_v4": 0.192000, + "Standard_D4as_v5": 0.356000, + "Standard_D4as_v5_Promo": 0.179000, + "Standard_D4d_v4": 0.226000, + "Standard_D4d_v5": 0.226000, + "Standard_D4ds_v4": 0.226000, + "Standard_D4ds_v5": 0.226000, + "Standard_D4lds_v5": 0.192000, + "Standard_D4ls_v5": 0.170000, + "Standard_D4pds_v5": 0.181000, + "Standard_D4plds_v5": 0.154000, + "Standard_D4pls_v5": 0.136000, + "Standard_D4ps_v5": 0.154000, + "Standard_D4s_v3": 0.192000, + "Standard_D4s_v4": 0.192000, + "Standard_D4s_v5": 0.192000, + "Standard_D5_v2": 0.916000, + "Standard_D64-16s_v3": 3.072000, + "Standard_D64-32s_v3": 3.072000, + "Standard_D64_v3": 3.072000, + "Standard_D64_v4": 3.072000, + "Standard_D64_v5": 3.072000, + "Standard_D64a_v4": 3.072000, + "Standard_D64ads_v5": 3.296000, + "Standard_D64as_v4": 3.072000, + "Standard_D64as_v5": 2.752000, + "Standard_D64as_v5_Promo": 2.867000, + "Standard_D64d_v4": 3.616000, + "Standard_D64d_v5": 3.616000, + "Standard_D64ds_v4": 3.616000, + "Standard_D64ds_v5": 3.616000, + "Standard_D64lds_v5": 3.072000, + "Standard_D64ls_v5": 2.720000, + "Standard_D64pds_v5": 2.893000, + "Standard_D64plds_v5": 2.458000, + "Standard_D64pls_v5": 2.176000, + "Standard_D64ps_v5": 2.464000, + "Standard_D64s_v3": 3.072000, + "Standard_D64s_v4": 3.072000, + "Standard_D64s_v5": 3.072000, + "Standard_D8_v3": 0.384000, + "Standard_D8_v4": 0.384000, + "Standard_D8_v5": 0.384000, + "Standard_D8a_v4": 0.384000, + "Standard_D8ads_v5": 0.780000, + "Standard_D8as_v4": 0.384000, + "Standard_D8as_v5": 0.712000, + "Standard_D8as_v5_Promo": 0.358000, + "Standard_D8d_v4": 0.452000, + "Standard_D8d_v5": 0.452000, + "Standard_D8ds_v4": 0.452000, + "Standard_D8ds_v5": 0.452000, + "Standard_D8lds_v5": 0.384000, + "Standard_D8ls_v5": 0.340000, + "Standard_D8pds_v5": 0.362000, + "Standard_D8plds_v5": 0.307000, + "Standard_D8pls_v5": 0.272000, + "Standard_D8ps_v5": 0.308000, + "Standard_D8s_v3": 0.384000, + "Standard_D8s_v4": 0.384000, + "Standard_D8s_v5": 0.384000, + "Standard_D96_v5": 4.608000, + "Standard_D96a_v4": 4.608000, + "Standard_D96ads_v5": 9.360000, + "Standard_D96as_v4": 4.608000, + "Standard_D96as_v5": 4.128000, + "Standard_D96as_v5_Promo": 4.301000, + "Standard_D96d_v5": 5.424000, + "Standard_D96ds_v5": 5.424000, + "Standard_D96lds_v5": 4.608000, + "Standard_D96ls_v5": 4.080000, + "Standard_D96s_v5": 4.608000, + "Standard_DS1": 0.077000, + "Standard_DS11": 0.192000, + "Standard_DS11-1_v2": 0.149000, + "Standard_DS11_v2": 0.149000, + "Standard_DS11_v2_Promo": 0.149000, + "Standard_DS12": 0.385000, + "Standard_DS12-1_v2": 0.299000, + "Standard_DS12-2_v2": 0.299000, + "Standard_DS12_v2": 0.299000, + "Standard_DS12_v2_Promo": 0.299000, + "Standard_DS13": 0.770000, + "Standard_DS13-2_v2": 0.598000, + "Standard_DS13-4_v2": 0.598000, + "Standard_DS13_v2": 0.598000, + "Standard_DS13_v2_Promo": 0.598000, + "Standard_DS14": 1.540000, + "Standard_DS14-4_v2": 1.196000, + "Standard_DS14-8_v2": 1.196000, + "Standard_DS14_v2": 1.196000, + "Standard_DS14_v2_Promo": 1.196000, + "Standard_DS15_v2": 1.495000, + "Standard_DS15i_v2": 1.495000, + "Standard_DS1_v2": 0.057000, + "Standard_DS2": 0.154000, + "Standard_DS2_v2": 0.114000, + "Standard_DS2_v2_Promo": 0.114000, + "Standard_DS3": 0.308000, + "Standard_DS3_v2": 0.229000, + "Standard_DS3_v2_Promo": 0.229000, + "Standard_DS4": 0.616000, + "Standard_DS4_v2": 0.458000, + "Standard_DS4_v2_Promo": 0.458000, + "Standard_DS5_v2": 0.916000, + "Standard_DS5_v2_Promo": 0.916000, + "Standard_E104i_v5": 7.207000, + "Standard_E104id_v5": 8.237000, + "Standard_E104ids_v5": 8.237000, + "Standard_E104is_v5": 7.207000, + "Standard_E112iads_v5": 8.070000, + "Standard_E112ias_v5": 6.961000, + "Standard_E112ibds_v5": 10.287000, + "Standard_E112ibs_v5": 9.178000, + "Standard_E16-4ads_v5": 1.784000, + "Standard_E16-4as_v4": 1.008000, + "Standard_E16-4as_v5": 1.640000, + "Standard_E16-4ds_v4": 1.152000, + "Standard_E16-4ds_v5": 1.152000, + "Standard_E16-4s_v3": 1.008000, + "Standard_E16-4s_v4": 1.008000, + "Standard_E16-4s_v5": 1.008000, + "Standard_E16-8ads_v5": 1.784000, + "Standard_E16-8as_v4": 1.008000, + "Standard_E16-8as_v5": 0.904000, + "Standard_E16-8ds_v4": 1.152000, + "Standard_E16-8ds_v5": 1.152000, + "Standard_E16-8s_v3": 1.008000, + "Standard_E16-8s_v4": 1.008000, + "Standard_E16-8s_v5": 1.008000, + "Standard_E16_v3": 1.008000, + "Standard_E16_v4": 1.008000, + "Standard_E16_v5": 1.008000, + "Standard_E16a_v4": 1.008000, + "Standard_E16ads_v5": 1.094000, + "Standard_E16as_v4": 1.008000, + "Standard_E16as_v5": 0.904000, + "Standard_E16bds_v5": 1.336000, + "Standard_E16bs_v5": 1.192000, + "Standard_E16d_v4": 1.152000, + "Standard_E16d_v5": 1.152000, + "Standard_E16ds_v4": 1.152000, + "Standard_E16ds_v5": 1.152000, + "Standard_E16pds_v5": 0.922000, + "Standard_E16ps_v5": 0.806000, + "Standard_E16s_v3": 1.008000, + "Standard_E16s_v4": 1.008000, + "Standard_E16s_v5": 1.008000, + "Standard_E20_v3": 1.260000, + "Standard_E20_v4": 1.260000, + "Standard_E20_v5": 1.260000, + "Standard_E20a_v4": 1.260000, + "Standard_E20ads_v5": 1.310000, + "Standard_E20as_v4": 1.260000, + "Standard_E20as_v5": 1.176000, + "Standard_E20d_v4": 1.440000, + "Standard_E20d_v5": 1.440000, + "Standard_E20ds_v4": 1.440000, + "Standard_E20ds_v5": 1.440000, + "Standard_E20pds_v5": 1.152000, + "Standard_E20ps_v5": 1.008000, + "Standard_E20s_v3": 1.260000, + "Standard_E20s_v4": 1.260000, + "Standard_E20s_v5": 1.260000, + "Standard_E2_v3": 0.126000, + "Standard_E2_v4": 0.126000, + "Standard_E2_v5": 0.126000, + "Standard_E2a_v4": 0.126000, + "Standard_E2ads_v5": 0.177000, + "Standard_E2as_v4": 0.126000, + "Standard_E2as_v5": 0.113000, + "Standard_E2bds_v5": 0.167000, + "Standard_E2bs_v5": 0.149000, + "Standard_E2d_v4": 0.144000, + "Standard_E2d_v5": 0.144000, + "Standard_E2ds_v4": 0.144000, + "Standard_E2ds_v5": 0.144000, + "Standard_E2pds_v5": 0.115000, + "Standard_E2ps_v5": 0.101000, + "Standard_E2s_v3": 0.126000, + "Standard_E2s_v4": 0.126000, + "Standard_E2s_v5": 0.126000, + "Standard_E32-16ads_v5": 2.096000, + "Standard_E32-16as_v4": 2.016000, + "Standard_E32-16as_v5": 3.280000, + "Standard_E32-16ds_v4": 2.304000, + "Standard_E32-16ds_v5": 2.304000, + "Standard_E32-16s_v3": 2.016000, + "Standard_E32-16s_v4": 2.016000, + "Standard_E32-16s_v5": 2.016000, + "Standard_E32-8ads_v5": 3.568000, + "Standard_E32-8as_v4": 2.016000, + "Standard_E32-8as_v5": 3.280000, + "Standard_E32-8ds_v4": 2.304000, + "Standard_E32-8ds_v5": 2.304000, + "Standard_E32-8s_v3": 2.016000, + "Standard_E32-8s_v4": 2.016000, + "Standard_E32-8s_v5": 2.016000, + "Standard_E32_v3": 2.016000, + "Standard_E32_v4": 2.016000, + "Standard_E32_v5": 2.016000, + "Standard_E32a_v4": 2.016000, + "Standard_E32ads_v5": 2.142000, + "Standard_E32as_v4": 2.016000, + "Standard_E32as_v5": 1.808000, + "Standard_E32bds_v5": 2.672000, + "Standard_E32bs_v5": 2.384000, + "Standard_E32d_v4": 2.304000, + "Standard_E32d_v5": 2.304000, + "Standard_E32ds_v4": 2.304000, + "Standard_E32ds_v5": 2.304000, + "Standard_E32pds_v5": 1.843000, + "Standard_E32ps_v5": 1.613000, + "Standard_E32s_v3": 2.016000, + "Standard_E32s_v4": 2.016000, + "Standard_E32s_v5": 2.016000, + "Standard_E4-2ads_v5": 0.262000, + "Standard_E4-2as_v4": 0.252000, + "Standard_E4-2as_v5": 0.226000, + "Standard_E4-2ds_v4": 0.288000, + "Standard_E4-2ds_v5": 0.288000, + "Standard_E4-2s_v3": 0.252000, + "Standard_E4-2s_v4": 0.252000, + "Standard_E4-2s_v5": 0.252000, + "Standard_E48_v3": 3.024000, + "Standard_E48_v4": 3.024000, + "Standard_E48_v5": 3.024000, + "Standard_E48a_v4": 3.024000, + "Standard_E48ads_v5": 3.190000, + "Standard_E48as_v4": 3.024000, + "Standard_E48as_v5": 2.712000, + "Standard_E48bds_v5": 4.008000, + "Standard_E48bs_v5": 3.576000, + "Standard_E48d_v4": 3.456000, + "Standard_E48d_v5": 3.456000, + "Standard_E48ds_v4": 3.456000, + "Standard_E48ds_v5": 3.456000, + "Standard_E48s_v3": 3.024000, + "Standard_E48s_v4": 3.024000, + "Standard_E48s_v5": 3.024000, + "Standard_E4_v3": 0.252000, + "Standard_E4_v4": 0.252000, + "Standard_E4_v5": 0.252000, + "Standard_E4a_v4": 0.252000, + "Standard_E4ads_v5": 0.262000, + "Standard_E4as_v4": 0.252000, + "Standard_E4as_v5": 0.226000, + "Standard_E4bds_v5": 0.334000, + "Standard_E4bs_v5": 0.298000, + "Standard_E4d_v4": 0.288000, + "Standard_E4d_v5": 0.288000, + "Standard_E4ds_v4": 0.288000, + "Standard_E4ds_v5": 0.288000, + "Standard_E4pds_v5": 0.230000, + "Standard_E4ps_v5": 0.202000, + "Standard_E4s_v3": 0.252000, + "Standard_E4s_v4": 0.252000, + "Standard_E4s_v5": 0.252000, + "Standard_E64-16ads_v5": 7.136000, + "Standard_E64-16as_v4": 4.032000, + "Standard_E64-16as_v5": 3.616000, + "Standard_E64-16ds_v4": 4.608000, + "Standard_E64-16ds_v5": 4.608000, + "Standard_E64-16s_v3": 3.629000, + "Standard_E64-16s_v4": 4.032000, + "Standard_E64-16s_v5": 4.032000, + "Standard_E64-32ads_v5": 4.192000, + "Standard_E64-32as_v4": 4.032000, + "Standard_E64-32as_v5": 6.560000, + "Standard_E64-32ds_v4": 4.608000, + "Standard_E64-32ds_v5": 4.608000, + "Standard_E64-32s_v3": 3.629000, + "Standard_E64-32s_v4": 4.032000, + "Standard_E64-32s_v5": 4.032000, + "Standard_E64_v3": 3.629000, + "Standard_E64_v4": 4.032000, + "Standard_E64_v5": 4.032000, + "Standard_E64a_v4": 4.032000, + "Standard_E64ads_v5": 4.238000, + "Standard_E64as_v4": 4.032000, + "Standard_E64as_v5": 3.662000, + "Standard_E64bds_v5": 5.344000, + "Standard_E64bs_v5": 4.768000, + "Standard_E64d_v4": 4.608000, + "Standard_E64d_v5": 4.608000, + "Standard_E64ds_v4": 4.608000, + "Standard_E64ds_v5": 4.608000, + "Standard_E64i_v3": 3.629000, + "Standard_E64is_v3": 3.629000, + "Standard_E64s_v3": 3.629000, + "Standard_E64s_v4": 4.032000, + "Standard_E64s_v5": 4.032000, + "Standard_E8-2ads_v5": 0.524000, + "Standard_E8-2as_v4": 0.504000, + "Standard_E8-2as_v5": 0.452000, + "Standard_E8-2ds_v4": 0.576000, + "Standard_E8-2ds_v5": 0.576000, + "Standard_E8-2s_v3": 0.504000, + "Standard_E8-2s_v4": 0.504000, + "Standard_E8-2s_v5": 0.504000, + "Standard_E8-4ads_v5": 0.892000, + "Standard_E8-4as_v4": 0.504000, + "Standard_E8-4as_v5": 0.452000, + "Standard_E8-4ds_v4": 0.576000, + "Standard_E8-4ds_v5": 0.576000, + "Standard_E8-4s_v3": 0.504000, + "Standard_E8-4s_v4": 0.504000, + "Standard_E8-4s_v5": 0.504000, + "Standard_E80ids_v4": 5.760000, + "Standard_E80is_v4": 5.040000, + "Standard_E8_v3": 0.504000, + "Standard_E8_v4": 0.504000, + "Standard_E8_v5": 0.504000, + "Standard_E8a_v4": 0.504000, + "Standard_E8ads_v5": 0.570000, + "Standard_E8as_v4": 0.504000, + "Standard_E8as_v5": 0.498000, + "Standard_E8bds_v5": 0.668000, + "Standard_E8bs_v5": 0.596000, + "Standard_E8d_v4": 0.576000, + "Standard_E8d_v5": 0.576000, + "Standard_E8ds_v4": 0.576000, + "Standard_E8ds_v5": 0.576000, + "Standard_E8pds_v5": 0.461000, + "Standard_E8ps_v5": 0.403000, + "Standard_E8s_v3": 0.504000, + "Standard_E8s_v4": 0.504000, + "Standard_E8s_v5": 0.504000, + "Standard_E96-24ads_v5": 6.288000, + "Standard_E96-24as_v4": 6.048000, + "Standard_E96-24as_v5": 9.840000, + "Standard_E96-24ds_v5": 6.912000, + "Standard_E96-24s_v5": 6.048000, + "Standard_E96-48ads_v5": 6.288000, + "Standard_E96-48as_v4": 6.048000, + "Standard_E96-48as_v5": 5.424000, + "Standard_E96-48ds_v5": 6.912000, + "Standard_E96-48s_v5": 6.048000, + "Standard_E96_v5": 6.048000, + "Standard_E96a_v4": 6.048000, + "Standard_E96ads_v5": 6.288000, + "Standard_E96as_v4": 6.048000, + "Standard_E96as_v5": 5.470000, + "Standard_E96bds_v5": 8.016000, + "Standard_E96bs_v5": 7.152000, + "Standard_E96d_v5": 6.912000, + "Standard_E96ds_v5": 6.912000, + "Standard_E96iads_v5": 6.917000, + "Standard_E96ias_v4": 6.048000, + "Standard_E96ias_v5": 5.966000, + "Standard_E96s_v5": 6.048000, + "Standard_F1": 0.049700, + "Standard_F16": 0.796000, + "Standard_F16s": 0.796000, + "Standard_F16s_v2": 0.677000, + "Standard_F1s": 0.049700, + "Standard_F2": 0.099000, + "Standard_F2s": 0.099000, + "Standard_F2s_v2": 0.084600, + "Standard_F32s_v2": 1.353000, + "Standard_F4": 0.199000, + "Standard_F48s_v2": 2.030000, + "Standard_F4s": 0.199000, + "Standard_F4s_v2": 0.169000, + "Standard_F64s_v2": 2.706000, + "Standard_F72s_v2": 3.045000, + "Standard_F8": 0.398000, + "Standard_F8s": 0.398000, + "Standard_F8s_v2": 0.338000, + "Standard_FX12mds": 1.266000, + "Standard_FX24mds": 2.532000, + "Standard_FX36mds": 3.798000, + "Standard_FX48mds": 5.064000, + "Standard_FX4mds": 0.422000, + "Standard_HB120-16rs_v3": 3.600000, + "Standard_HB120-32rs_v3": 3.600000, + "Standard_HB120-64rs_v3": 3.600000, + "Standard_HB120-96rs_v3": 3.600000, + "Standard_HB120rs_v3": 3.600000, + "Standard_HB176-144rs_v4": 7.200000, + "Standard_HB176-24rs_v4": 7.200000, + "Standard_HB176-48rs_v4": 7.200000, + "Standard_HB176-96rs_v4": 7.200000, + "Standard_HB176rs_v4": 7.200000, + "Standard_HX176-144rs": 8.640000, + "Standard_HX176-24rs": 8.640000, + "Standard_HX176-48rs": 8.640000, + "Standard_HX176-96rs": 8.640000, + "Standard_HX176rs": 8.640000, + "Standard_L16as_v3": 1.248000, + "Standard_L16s_v2": 1.248000, + "Standard_L16s_v3": 1.396000, + "Standard_L32as_v3": 2.496000, + "Standard_L32s_v2": 2.496000, + "Standard_L32s_v3": 2.792000, + "Standard_L48as_v3": 3.744000, + "Standard_L48s_v2": 3.744000, + "Standard_L48s_v3": 4.188000, + "Standard_L64as_v3": 4.992000, + "Standard_L64s_v2": 4.992000, + "Standard_L64s_v3": 5.584000, + "Standard_L80as_v3": 6.240000, + "Standard_L80s_v2": 6.240000, + "Standard_L80s_v3": 6.980000, + "Standard_L88is_v2": 6.864000, + "Standard_L8as_v3": 0.624000, + "Standard_L8s_v2": 0.624000, + "Standard_L8s_v3": 0.698000, + "Standard_M128": 13.338000, + "Standard_M128-32ms": 26.688000, + "Standard_M128-64ms": 26.688000, + "Standard_M128dms_v2": 26.690000, + "Standard_M128ds_v2": 13.340000, + "Standard_M128m": 26.688000, + "Standard_M128ms": 26.688000, + "Standard_M128ms_v2": 26.269000, + "Standard_M128s": 13.338000, + "Standard_M128s_v2": 12.919000, + "Standard_M16-4ms": 3.073000, + "Standard_M16-8ms": 3.073000, + "Standard_M16ms": 3.073000, + "Standard_M16s": 2.387000, + "Standard_M192idms_v2": 32.064000, + "Standard_M192ids_v2": 16.032000, + "Standard_M192ims_v2": 31.643000, + "Standard_M192is_v2": 15.611000, + "Standard_M208-104ms_v2": 44.620000, + "Standard_M208-104s_v2": 22.310000, + "Standard_M208-52ms_v2": 44.620000, + "Standard_M208-52s_v2": 22.310000, + "Standard_M208ms_v2": 44.620000, + "Standard_M208s_v2": 22.310000, + "Standard_M32-16ms": 6.146000, + "Standard_M32-8ms": 6.146000, + "Standard_M32dms_v2": 6.146000, + "Standard_M32ls": 2.873000, + "Standard_M32ms": 6.146000, + "Standard_M32ms_v2": 6.041000, + "Standard_M32s": 3.335000, + "Standard_M32ts": 2.707000, + "Standard_M416-104ms_v2": 99.150000, + "Standard_M416-104s_v2": 49.580000, + "Standard_M416-208ms_v2": 99.150000, + "Standard_M416-208s_v2": 49.580000, + "Standard_M416is_v2": 49.580000, + "Standard_M416ms_v2": 99.150000, + "Standard_M416s_8_v2": 66.110000, + "Standard_M416s_v2": 49.580000, + "Standard_M64": 6.669000, + "Standard_M64-16ms": 10.337000, + "Standard_M64-32ms": 10.337000, + "Standard_M64dms_v2": 10.340000, + "Standard_M64ds_v2": 6.669000, + "Standard_M64ls": 5.415000, + "Standard_M64m": 10.337000, + "Standard_M64ms": 10.337000, + "Standard_M64ms_v2": 10.130000, + "Standard_M64s": 6.669000, + "Standard_M64s_v2": 6.459000, + "Standard_M8-2ms": 1.536000, + "Standard_M8-4ms": 1.536000, + "Standard_M8ms": 1.536000, + "Standard_NC12s_v3": 6.120000, + "Standard_NC16ads_A10_v4": 2.200000, + "Standard_NC16as_T4_v3": 1.204000, + "Standard_NC24ads_A100_v4": 3.673000, + "Standard_NC24rs_v3": 13.460000, + "Standard_NC24s_v3": 12.240000, + "Standard_NC32ads_A10_v4": 4.400000, + "Standard_NC48ads_A100_v4": 7.346000, + "Standard_NC4as_T4_v3": 0.526000, + "Standard_NC64as_T4_v3": 4.352000, + "Standard_NC6s_v3": 3.060000, + "Standard_NC8ads_A10_v4": 1.100000, + "Standard_NC8as_T4_v3": 0.752000, + "Standard_NC96ads_A100_v4": 14.692000, + "Standard_ND96ams_A100_v4": 32.770000, + "Standard_ND96amsr_A100_v4": 32.770000, + "Standard_ND96asr_A100_v4": 27.197000, + "Standard_NG16ads_V620_v1": 2.190000, + "Standard_NG32adms_V620": 5.694000, + "Standard_NG32ads_V620_v1": 4.380000, + "Standard_NG8ads_V620_v1": 1.095000, + "Standard_NV12ads_A10_v5": 0.908000, + "Standard_NV16as_v4": 0.932000, + "Standard_NV18ads_A10_v5": 1.600000, + "Standard_NV32as_v4": 1.864000, + "Standard_NV36adms_A10_v5": 4.520000, + "Standard_NV36ads_A10_v5": 3.200000, + "Standard_NV4as_v4": 0.233000, + "Standard_NV6ads_A10_v5": 0.454000, + "Standard_NV72ads_A10_v5": 6.520000, + "Standard_NV8as_v4": 0.466000, + } +} diff --git a/pkg/test/aksnodeclass.go b/pkg/test/aksnodeclass.go new file mode 100644 index 000000000..98b34d6e9 --- /dev/null +++ b/pkg/test/aksnodeclass.go @@ -0,0 +1,45 @@ +/* +Portions Copyright (c) Microsoft Corporation. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package test + +import ( + "context" + "fmt" + + "github.com/imdario/mergo" + + "github.com/aws/karpenter-core/pkg/test" + + "github.com/Azure/karpenter/pkg/apis/v1alpha2" +) + +func AKSNodeClass(overrides ...v1alpha2.AKSNodeClass) *v1alpha2.AKSNodeClass { + options := v1alpha2.AKSNodeClass{} + for _, override := range overrides { + if err := mergo.Merge(&options, override, mergo.WithOverride); err != nil { + panic(fmt.Sprintf("Failed to merge settings: %s", err)) + } + } + + nc := &v1alpha2.AKSNodeClass{ + ObjectMeta: test.ObjectMeta(options.ObjectMeta), + Spec: options.Spec, + Status: options.Status, + } + nc.SetDefaults(context.Background()) + return nc +} diff --git a/pkg/test/environment.go b/pkg/test/environment.go new file mode 100644 index 000000000..14384ac0d --- /dev/null +++ b/pkg/test/environment.go @@ -0,0 +1,168 @@ +/* +Portions Copyright (c) Microsoft Corporation. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package test + +import ( + "context" + + azurecache "github.com/Azure/karpenter/pkg/cache" + "github.com/Azure/karpenter/pkg/fake" + "github.com/Azure/karpenter/pkg/providers/imagefamily" + "github.com/Azure/karpenter/pkg/providers/instance" + "github.com/Azure/karpenter/pkg/providers/instancetype" + "github.com/Azure/karpenter/pkg/providers/launchtemplate" + "github.com/Azure/karpenter/pkg/providers/loadbalancer" + "github.com/Azure/karpenter/pkg/providers/pricing" + "github.com/patrickmn/go-cache" + "knative.dev/pkg/ptr" + + coretest "github.com/aws/karpenter-core/pkg/test" +) + +var resourceGroup = "test-resourceGroup" + +type Environment struct { + // API + VirtualMachinesAPI *fake.VirtualMachinesAPI + AzureResourceGraphAPI *fake.AzureResourceGraphAPI + VirtualMachineExtensionsAPI *fake.VirtualMachineExtensionsAPI + NetworkInterfacesAPI *fake.NetworkInterfacesAPI + CommunityImageVersionsAPI *fake.CommunityGalleryImageVersionsAPI + MockSkuClientSignalton *fake.MockSkuClientSingleton + PricingAPI *fake.PricingAPI + LoadBalancersAPI *fake.LoadBalancersAPI + + // Cache + KubernetesVersionCache *cache.Cache + InstanceTypeCache *cache.Cache + LoadBalancerCache *cache.Cache + UnavailableOfferingsCache *azurecache.UnavailableOfferings + + // Providers + InstanceTypesProvider *instancetype.Provider + InstanceProvider *instance.Provider + PricingProvider *pricing.Provider + ImageProvider *imagefamily.Provider + ImageResolver *imagefamily.Resolver + LaunchTemplateProvider *launchtemplate.Provider + LoadBalancerProvider *loadbalancer.Provider +} + +func NewEnvironment(ctx context.Context, env *coretest.Environment) *Environment { + testSettings := Settings() + location := fake.Region + + // API + virtualMachinesAPI := &fake.VirtualMachinesAPI{} + azureResourceGraphAPI := &fake.AzureResourceGraphAPI{AzureResourceGraphBehavior: fake.AzureResourceGraphBehavior{VirtualMachinesAPI: virtualMachinesAPI, ResourceGroup: resourceGroup}} + virtualMachinesExtensionsAPI := &fake.VirtualMachineExtensionsAPI{} + networkInterfacesAPI := &fake.NetworkInterfacesAPI{} + pricingAPI := &fake.PricingAPI{} + skuClientSignalton := &fake.MockSkuClientSingleton{SKUClient: &fake.ResourceSKUsAPI{}} + coummunityImageVersionsAPI := &fake.CommunityGalleryImageVersionsAPI{} + loadBalancersAPI := &fake.LoadBalancersAPI{} + + // Cache + kubernetesVersionCache := cache.New(azurecache.KubernetesVersionTTL, azurecache.DefaultCleanupInterval) + instanceTypeCache := cache.New(instancetype.InstanceTypesCacheTTL, azurecache.DefaultCleanupInterval) + loadBalancerCache := cache.New(loadbalancer.LoadBalancersCacheTTL, azurecache.DefaultCleanupInterval) + unavailableOfferingsCache := azurecache.NewUnavailableOfferings() + + // Providers + pricingProvider := pricing.NewProvider(ctx, pricingAPI, location, make(chan struct{})) + imageFamilyProvider := imagefamily.NewProvider(env.KubernetesInterface, kubernetesVersionCache, coummunityImageVersionsAPI, location) + imageFamilyResolver := imagefamily.New(env.Client, imageFamilyProvider) + instanceTypesProvider := instancetype.NewProvider(location, instanceTypeCache, skuClientSignalton, pricingProvider, unavailableOfferingsCache) + launchTemplateProvider := launchtemplate.NewProvider( + ctx, + imageFamilyResolver, + imageFamilyProvider, + ptr.String("ca-bundle"), + testSettings.ClusterEndpoint, + "test-tenant", + "test-subscription", + "test-userAssignedIdentity", + resourceGroup, + location, + ) + loadBalancerProvider := loadbalancer.NewProvider( + loadBalancersAPI, + loadBalancerCache, + "test-resourceGroup", + ) + azClient := instance.NewAZClientFromAPI( + virtualMachinesAPI, + azureResourceGraphAPI, + virtualMachinesExtensionsAPI, + networkInterfacesAPI, + loadBalancersAPI, + coummunityImageVersionsAPI, + skuClientSignalton, + ) + instanceProvider := instance.NewProvider( + ctx, + azClient, + instanceTypesProvider, + launchTemplateProvider, + loadBalancerProvider, + unavailableOfferingsCache, + location, // region + resourceGroup, // resourceGroup + "", // subnet + "", // subscriptionID + ) + + return &Environment{ + VirtualMachinesAPI: virtualMachinesAPI, + AzureResourceGraphAPI: azureResourceGraphAPI, + VirtualMachineExtensionsAPI: virtualMachinesExtensionsAPI, + NetworkInterfacesAPI: networkInterfacesAPI, + LoadBalancersAPI: loadBalancersAPI, + MockSkuClientSignalton: skuClientSignalton, + PricingAPI: pricingAPI, + + KubernetesVersionCache: kubernetesVersionCache, + InstanceTypeCache: instanceTypeCache, + UnavailableOfferingsCache: unavailableOfferingsCache, + LoadBalancerCache: loadBalancerCache, + + InstanceTypesProvider: instanceTypesProvider, + InstanceProvider: instanceProvider, + PricingProvider: pricingProvider, + ImageProvider: imageFamilyProvider, + ImageResolver: imageFamilyResolver, + LaunchTemplateProvider: launchTemplateProvider, + LoadBalancerProvider: loadBalancerProvider, + } +} + +func (env *Environment) Reset() { + env.VirtualMachinesAPI.Reset() + env.AzureResourceGraphAPI.Reset() + env.VirtualMachineExtensionsAPI.Reset() + env.NetworkInterfacesAPI.Reset() + env.LoadBalancersAPI.Reset() + env.CommunityImageVersionsAPI.Reset() + env.MockSkuClientSignalton.Reset() + env.PricingAPI.Reset() + env.PricingProvider.Reset() + + env.KubernetesVersionCache.Flush() + env.InstanceTypeCache.Flush() + env.UnavailableOfferingsCache.Flush() + env.LoadBalancerCache.Flush() +} diff --git a/pkg/test/loadbalancer.go b/pkg/test/loadbalancer.go new file mode 100644 index 000000000..980246ac8 --- /dev/null +++ b/pkg/test/loadbalancer.go @@ -0,0 +1,52 @@ +/* +Portions Copyright (c) Microsoft Corporation. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package test + +import ( + "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/network/armnetwork" + "github.com/Azure/karpenter/pkg/fake" + "github.com/Azure/karpenter/pkg/providers/loadbalancer" + "github.com/samber/lo" +) + +func MakeStandardLoadBalancer(resourceGroup string, lbName string, includeOutbound bool) armnetwork.LoadBalancer { + lbID := fake.MakeLoadBalancerID(resourceGroup, lbName) + + result := armnetwork.LoadBalancer{ + ID: &lbID, + Name: &lbName, + Properties: &armnetwork.LoadBalancerPropertiesFormat{ + BackendAddressPools: []*armnetwork.BackendAddressPool{ + { + ID: lo.ToPtr(fake.MakeBackendAddressPoolID(resourceGroup, lbName, loadbalancer.SLBInboundBackendPoolName)), + Name: lo.ToPtr(loadbalancer.SLBInboundBackendPoolName), + Properties: &armnetwork.BackendAddressPoolPropertiesFormat{}, + }, + }, + }, + } + + if includeOutbound { + result.Properties.BackendAddressPools = append(result.Properties.BackendAddressPools, &armnetwork.BackendAddressPool{ + ID: lo.ToPtr(fake.MakeBackendAddressPoolID(resourceGroup, lbName, loadbalancer.SLBOutboundBackendPoolName)), + Name: lo.ToPtr(loadbalancer.SLBOutboundBackendPoolName), + Properties: &armnetwork.BackendAddressPoolPropertiesFormat{}, + }) + } + + return result +} diff --git a/pkg/test/nodepool.go b/pkg/test/nodepool.go new file mode 100644 index 000000000..9e9dfcb39 --- /dev/null +++ b/pkg/test/nodepool.go @@ -0,0 +1,30 @@ +/* +Portions Copyright (c) Microsoft Corporation. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package test + +import ( + "context" + + corev1beta1 "github.com/aws/karpenter-core/pkg/apis/v1beta1" + "github.com/aws/karpenter-core/pkg/test" +) + +func NodePool(options corev1beta1.NodePool) *corev1beta1.NodePool { + nodePool := test.NodePool(options) + nodePool.SetDefaults(context.Background()) + return nodePool +} diff --git a/pkg/test/settings.go b/pkg/test/settings.go new file mode 100644 index 000000000..04068e2ff --- /dev/null +++ b/pkg/test/settings.go @@ -0,0 +1,60 @@ +/* +Portions Copyright (c) Microsoft Corporation. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package test + +import ( + "fmt" + + "github.com/imdario/mergo" + "github.com/samber/lo" + + azsettings "github.com/Azure/karpenter/pkg/apis/settings" +) + +type SettingOptions struct { + ClusterName *string + ClusterEndpoint *string + ClusterID *string + KubeletClientTLSBootstrapToken *string + SSHPublicKey *string + NetworkPlugin *string + NetworkPolicy *string + VMMemoryOverheadPercent *float64 + NodeIdentities []string + Tags map[string]string +} + +func Settings(overrides ...SettingOptions) *azsettings.Settings { + options := SettingOptions{} + for _, override := range overrides { + if err := mergo.Merge(&options, override, mergo.WithOverride); err != nil { + panic(fmt.Sprintf("Failed to merge settings: %s", err)) + } + } + return &azsettings.Settings{ + ClusterName: lo.FromPtrOr(options.ClusterName, "test-cluster"), + ClusterEndpoint: lo.FromPtrOr(options.ClusterEndpoint, "https://test-cluster"), + ClusterID: lo.FromPtrOr(options.ClusterID, "00000000"), + KubeletClientTLSBootstrapToken: lo.FromPtrOr(options.KubeletClientTLSBootstrapToken, "test-token"), + SSHPublicKey: lo.FromPtrOr(options.SSHPublicKey, "test-ssh-public-key"), + NetworkPlugin: lo.FromPtrOr(options.NetworkPlugin, "kubenet"), + NetworkPolicy: lo.FromPtrOr(options.NetworkPolicy, ""), + VMMemoryOverheadPercent: lo.FromPtrOr(options.VMMemoryOverheadPercent, 0.075), + NodeIdentities: options.NodeIdentities, + Tags: options.Tags, + } +} diff --git a/pkg/utils/gpu.go b/pkg/utils/gpu.go new file mode 100644 index 000000000..4741aa382 --- /dev/null +++ b/pkg/utils/gpu.go @@ -0,0 +1,168 @@ +/* +Portions Copyright (c) Microsoft Corporation. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package utils + +import ( + "strings" +) + +// TODO: Get these from agentbaker +const ( + Nvidia470CudaDriverVersion = "cuda-470.82.01" + Nvidia510CudaDriverVersion = "cuda-510.47.03" + Nvidia525CudaDriverVersion = "cuda-525.85.12" + Nvidia510GridDriverVersion = "grid-510.73.08" +) + +var ( + /* If a new GPU sku becomes available, add a key to this map, but only if you have a confirmation + that we have an agreement with NVIDIA for this specific gpu. + */ + NvidiaEnabledSKUs = map[string]bool{ + // M60 + "standard_nv6": true, + "standard_nv12": true, + "standard_nv12s_v3": true, + "standard_nv24": true, + "standard_nv24s_v3": true, + "standard_nv24r": true, + "standard_nv48s_v3": true, + // P40 + "standard_nd6s": true, + "standard_nd12s": true, + "standard_nd24s": true, + "standard_nd24rs": true, + // P100 + "standard_nc6s_v2": true, + "standard_nc12s_v2": true, + "standard_nc24s_v2": true, + "standard_nc24rs_v2": true, + // V100 + "standard_nc6s_v3": true, + "standard_nc12s_v3": true, + "standard_nc24s_v3": true, + "standard_nc24rs_v3": true, + "standard_nd40s_v3": true, + "standard_nd40rs_v2": true, + // T4 + "standard_nc4as_t4_v3": true, + "standard_nc8as_t4_v3": true, + "standard_nc16as_t4_v3": true, + "standard_nc64as_t4_v3": true, + // A100 40GB + "standard_nd96asr_v4": true, + "standard_nd112asr_a100_v4": true, + "standard_nd120asr_a100_v4": true, + // A100 80GB + "standard_nd96amsr_a100_v4": true, + "standard_nd112amsr_a100_v4": true, + "standard_nd120amsr_a100_v4": true, + // A100 PCIE 80GB + "standard_nc24ads_a100_v4": true, + "standard_nc48ads_a100_v4": true, + "standard_nc96ads_a100_v4": true, + "standard_ncads_a100_v4": true, + // A10 + "standard_nc8ads_a10_v4": true, + "standard_nc16ads_a10_v4": true, + "standard_nc32ads_a10_v4": true, + // A10, GRID only + "standard_nv6ads_a10_v5": true, + "standard_nv12ads_a10_v5": true, + "standard_nv18ads_a10_v5": true, + "standard_nv36ads_a10_v5": true, + "standard_nv36adms_a10_v5": true, + "standard_nv72ads_a10_v5": true, + // A100 + "standard_nd96ams_v4": true, + "standard_nd96ams_a100_v4": true, + } + + // List of GPU SKUs currently enabled and validated for Mariner. Will expand the support + // to cover other SKUs available in Azure + MarinerNvidiaEnabledSKUs = map[string]bool{ + // V100 + "standard_nc6s_v3": true, + "standard_nc12s_v3": true, + "standard_nc24s_v3": true, + "standard_nc24rs_v3": true, + "standard_nd40s_v3": true, + "standard_nd40rs_v2": true, + // T4 + "standard_nc4as_t4_v3": true, + "standard_nc8as_t4_v3": true, + "standard_nc16as_t4_v3": true, + "standard_nc64as_t4_v3": true, + } +) + +// IsNvidiaEnabledSKU determines if an VM SKU has nvidia driver support +func IsNvidiaEnabledSKU(vmSize string) bool { + // Trim the optional _Promo suffix. + vmSize = strings.ToLower(vmSize) + vmSize = strings.TrimSuffix(vmSize, "_promo") + return NvidiaEnabledSKUs[vmSize] +} + +// IsNvidiaEnabledSKU determines if an VM SKU has nvidia driver support +func IsMarinerEnabledGPUSKU(vmSize string) bool { + // Trim the optional _Promo suffix. + vmSize = strings.ToLower(vmSize) + vmSize = strings.TrimSuffix(vmSize, "_promo") + return MarinerNvidiaEnabledSKUs[vmSize] +} + +// NV series GPUs target graphics workloads vs NC which targets compute. +// they typically use GRID, not CUDA drivers, and will fail to install CUDA drivers. +// NVv1 seems to run with CUDA, NVv5 requires GRID. +// NVv3 is untested on AKS, NVv4 is AMD so n/a, and NVv2 no longer seems to exist (?). +func GetGPUDriverVersion(size string) string { + if useGridDrivers(size) { + return Nvidia510GridDriverVersion + } + if isStandardNCv1(size) { + return Nvidia470CudaDriverVersion + } + return Nvidia525CudaDriverVersion +} + +func isStandardNCv1(size string) bool { + tmp := strings.ToLower(size) + return strings.HasPrefix(tmp, "standard_nc") && !strings.Contains(tmp, "_v") +} + +func useGridDrivers(size string) bool { + return ConvergedGPUDriverSizes[strings.ToLower(size)] +} + +/* ConvergedGPUDriverSizes : these sizes use a "converged" driver to support both cuda/grid workloads. +how do you figure this out? ask HPC or find out by trial and error. +installing vanilla cuda drivers will fail to install with opaque errors. +see https://github.com/Azure/azhpc-extensions/blob/daaefd78df6f27012caf30f3b54c3bd6dc437652/NvidiaGPU/resources.json +*/ +//nolint:gochecknoglobals +var ConvergedGPUDriverSizes = map[string]bool{ + "standard_nv6ads_a10_v5": true, + "standard_nv12ads_a10_v5": true, + "standard_nv18ads_a10_v5": true, + "standard_nv36ads_a10_v5": true, + "standard_nv72ads_a10_v5": true, + "standard_nv36adms_a10_v5": true, + "standard_nc8ads_a10_v4": true, + "standard_nc16ads_a10_v4": true, + "standard_nc32ads_a10_v4": true, +} diff --git a/pkg/utils/gpu_test.go b/pkg/utils/gpu_test.go new file mode 100644 index 000000000..7db3eb50f --- /dev/null +++ b/pkg/utils/gpu_test.go @@ -0,0 +1,70 @@ +/* +Portions Copyright (c) Microsoft Corporation. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package utils + +import ( + "testing" +) + +func TestIsNvidiaEnabledSKU(t *testing.T) { + tests := []struct { + name string + input string + output bool + }{ + {"Valid SKU", "standard_nc6s_v2", true}, + {"Valid SKU with Promo", "standard_nc6s_v2_promo", true}, + {"Non-Existent SKU", "non_existent_sku", false}, + {"Uppercase SKU", "STANDARD_NC6s_v2", true}, + {"Mixed Case SKU with Promo", "Standard_Nc6s_v2_Promo", true}, + {"Not supported SKU", "standard_d2_v2", false}, + {"Empty SKU", "", false}, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + result := IsNvidiaEnabledSKU(test.input) + if result != test.output { + t.Errorf("Expected %v, but got %v", test.output, result) + } + }) + } +} + +func TestIsMarinerEnabledGPUSKU(t *testing.T) { + tests := []struct { + name string + input string + output bool + }{ + {"Valid Mariner SKU", "standard_nc6s_v3", true}, + {"Valid Mariner SKU with Promo", "standard_nc6s_v3_promo", true}, + {"Non-Existent Mariner SKU", "non_existent_sku", false}, + {"Uppercase Mariner SKU", "STANDARD_NC6S_V3", true}, + {"Mixed Case Mariner SKU with Promo", "Standard_Nc6s_V3_Promo", true}, + {"Empty Mariner SKU", "", false}, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + result := IsMarinerEnabledGPUSKU(test.input) + if result != test.output { + t.Errorf("Expected %v, but got %v", test.output, result) + } + }) + } +} diff --git a/pkg/utils/opts/armopts.go b/pkg/utils/opts/armopts.go new file mode 100644 index 000000000..68e5834d9 --- /dev/null +++ b/pkg/utils/opts/armopts.go @@ -0,0 +1,64 @@ +/* +Portions Copyright (c) Microsoft Corporation. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package opts + +import ( + "net/http" + "time" + + "github.com/Azure/azure-sdk-for-go/sdk/azcore/arm" + "github.com/Azure/azure-sdk-for-go/sdk/azcore/policy" + "github.com/Azure/karpenter/pkg/auth" +) + +func DefaultArmOpts() *arm.ClientOptions { + opts := &arm.ClientOptions{} + opts.Telemetry = DefaultTelemetryOpts() + opts.Retry = DefaultRetryOpts() + opts.Transport = defaultHTTPClient + return opts +} + +func DefaultRetryOpts() policy.RetryOptions { + return policy.RetryOptions{ + MaxRetries: 20, + // Note the default retry behavior is exponential backoff + RetryDelay: time.Second * 5, + // TODO: bsoghigian: Investigate if we want to leverage some of the status codes other than the defaults. + // the defaults are // StatusCodes specifies the HTTP status codes that indicate the operation should be retried. + // A nil slice will use the following values. + // http.StatusRequestTimeout 408 + // http.StatusTooManyRequests 429 + // http.StatusInternalServerError 500 + // http.StatusBadGateway 502 + // http.StatusServiceUnavailable 503 + // http.StatusGatewayTimeout 504 + // Specifying values will replace the default values. + // Specifying an empty slice will disable retries for HTTP status codes. + // StatusCodes: nil, + } +} + +func DefaultHTTPClient() *http.Client { + return defaultHTTPClient +} + +func DefaultTelemetryOpts() policy.TelemetryOptions { + return policy.TelemetryOptions{ + ApplicationID: auth.GetUserAgentExtension(), + } +} diff --git a/pkg/utils/opts/init_http_client.go b/pkg/utils/opts/init_http_client.go new file mode 100644 index 000000000..1057d9a1a --- /dev/null +++ b/pkg/utils/opts/init_http_client.go @@ -0,0 +1,52 @@ +/* +Portions Copyright (c) Microsoft Corporation. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package opts + +import ( + "crypto/tls" + "net" + "net/http" + "time" + + "github.com/Azure/go-armbalancer" +) + +var defaultHTTPClient *http.Client + +func init() { + defaultHTTPClient = &http.Client{ + // For Now using the defaults recommended by Track 2 + Transport: armbalancer.New(armbalancer.Options{ + PoolSize: 100, + Transport: &http.Transport{ + Proxy: http.ProxyFromEnvironment, + DialContext: (&net.Dialer{ + Timeout: 30 * time.Second, + KeepAlive: 30 * time.Second, + }).DialContext, + ForceAttemptHTTP2: true, + MaxIdleConns: 100, + IdleConnTimeout: 90 * time.Second, + TLSHandshakeTimeout: 10 * time.Second, + ExpectContinueTimeout: 1 * time.Second, + TLSClientConfig: &tls.Config{ + MinVersion: tls.VersionTLS12, + }, + }, + }), + } +} diff --git a/pkg/utils/project/project.go b/pkg/utils/project/project.go new file mode 100644 index 000000000..fd92f36a5 --- /dev/null +++ b/pkg/utils/project/project.go @@ -0,0 +1,21 @@ +/* +Portions Copyright (c) Microsoft Corporation. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package project + +// Version is the karpenter app version injected during compilation +// when using the Makefile +var Version = "unspecified" diff --git a/pkg/utils/utils.go b/pkg/utils/utils.go new file mode 100644 index 000000000..bd6ea31de --- /dev/null +++ b/pkg/utils/utils.go @@ -0,0 +1,60 @@ +/* +Portions Copyright (c) Microsoft Corporation. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package utils + +import ( + "context" + "fmt" + "regexp" + + "knative.dev/pkg/logging" + "sigs.k8s.io/cloud-provider-azure/pkg/provider" +) + +// GetVMName parses the provider ID stored on the node to get the vmName +// associated with a node +func GetVMName(providerID string) (string, error) { + // standalone VMs have providerID in the format: azure:///subscriptions//resourceGroups//providers/Microsoft.Compute/virtualMachines/ + r := regexp.MustCompile(`azure:///subscriptions/.*/resourceGroups/.*/providers/Microsoft.Compute/virtualMachines/(?P.*)`) + matches := r.FindStringSubmatch(providerID) + if matches == nil { + return "", fmt.Errorf("parsing vm name %s", providerID) + } + for i, name := range r.SubexpNames() { + if name == "InstanceID" { + return matches[i], nil + } + } + return "", fmt.Errorf("parsing vm name %s", providerID) +} + +func ResourceIDToProviderID(ctx context.Context, id string) string { + providerID := fmt.Sprintf("azure://%s", id) + // for historical reasons Azure providerID has the resource group name in lower case + providerIDLowerRG, err := provider.ConvertResourceGroupNameToLower(providerID) + if err != nil { + logging.FromContext(ctx).Warnf("Failed to convert resource group name to lower case in providerID %s: %v", providerID, err) + // fallback to original providerID + return providerID + } + return providerIDLowerRG +} + +func MkVMID(resourceGroupName string, vmName string) string { + const idFormat = "/subscriptions/subscriptionID/resourceGroups/%s/providers/Microsoft.Compute/virtualMachines/%s" + return fmt.Sprintf(idFormat, resourceGroupName, vmName) +} diff --git a/pkg/webhooks/webhooks.go b/pkg/webhooks/webhooks.go new file mode 100644 index 000000000..d905ce72f --- /dev/null +++ b/pkg/webhooks/webhooks.go @@ -0,0 +1,63 @@ +/* +Portions Copyright (c) Microsoft Corporation. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package webhooks + +import ( + "context" + + "k8s.io/apimachinery/pkg/runtime/schema" + "knative.dev/pkg/configmap" + "knative.dev/pkg/controller" + knativeinjection "knative.dev/pkg/injection" + "knative.dev/pkg/webhook/resourcesemantics" + "knative.dev/pkg/webhook/resourcesemantics/defaulting" + "knative.dev/pkg/webhook/resourcesemantics/validation" + + "github.com/Azure/karpenter/pkg/apis/v1alpha2" +) + +func NewWebhooks() []knativeinjection.ControllerConstructor { + return []knativeinjection.ControllerConstructor{ + NewCRDDefaultingWebhook, + NewCRDValidationWebhook, + } +} + +func NewCRDDefaultingWebhook(ctx context.Context, _ configmap.Watcher) *controller.Impl { + return defaulting.NewAdmissionController(ctx, + "defaulting.webhook.karpenter.azure.com", + "/default/karpenter.azure.com", + Resources, + func(ctx context.Context) context.Context { return ctx }, + true, + ) +} + +func NewCRDValidationWebhook(ctx context.Context, _ configmap.Watcher) *controller.Impl { + return validation.NewAdmissionController(ctx, + "validation.webhook.karpenter.azure.com", + "/validate/karpenter.azure.com", + Resources, + func(ctx context.Context) context.Context { return ctx }, + true, + ) +} + +var Resources = map[schema.GroupVersionKind]resourcesemantics.GenericCRD{ + v1alpha2.SchemeGroupVersion.WithKind("AKSNodeClass"): &v1alpha2.AKSNodeClass{}, + // corev1alpha5.SchemeGroupVersion.WithKind("Provisioner"): &v1alpha5.Provisioner{}, +} diff --git a/prometheus-values.yaml b/prometheus-values.yaml new file mode 100644 index 000000000..f1affe3fb --- /dev/null +++ b/prometheus-values.yaml @@ -0,0 +1,14 @@ +alertmanager: + persistentVolume: + enabled: false + +server: + fullnameOverride: prometheus-server + persistentVolume: + enabled: false + +extraScrapeConfigs: | + - job_name: karpenter + static_configs: + - targets: + - karpenter.karpenter:8080 diff --git a/skaffold.yaml b/skaffold.yaml new file mode 100644 index 000000000..cc0eb94c4 --- /dev/null +++ b/skaffold.yaml @@ -0,0 +1,77 @@ +apiVersion: skaffold/v3 +kind: Config +build: + artifacts: + - image: controller + ko: + main: github.com/Azure/karpenter/cmd/controller + dependencies: + paths: + - '**/*.go' + - '**/*.gtpl' + #flags: ['-tags', 'ccp'] +manifests: + helm: + releases: + - name: karpenter + chartPath: charts/karpenter + skipBuildDependencies: true + namespace: karpenter + createNamespace: true + setValueTemplates: + controller.image.repository: "{{.IMAGE_REPO_controller}}" + controller.image.tag: "{{.IMAGE_TAG_controller}}" + controller.image.digest: "{{.IMAGE_DIGEST_controller}}" + overrides: + # these go into the global config map + settings: + featureGates: + drift: true + azure: + clusterName: karpenter + clusterEndpoint: https://karpenter-tallaxes-azure-k-26fe00-j3imiwu5.hcp.westus2.staging.azmk8s.io:443 + kubeletClientTLSBootstrapToken: "" # TODO: get this from the cluster + # TODO: autogenerate + sshPublicKey: "" + networkPlugin: "azure" # TODO: get this from the cluster + networkPolicy: "" + replicas: 1 # for better debugging experience + controller: + # The maximum amount of time with no new ending pods that if exceeded ends the current batching window. If pods arrive + # faster than this time, the batching window will be extended up to the maxDuration. If they arrive slower, the pods + # will be batched separately. + batchIdleDuration: 1s # 60s is a good value for large runs (500+ nodes) + # The maximum length of a batch window. The longer this is, the more pods we can consider for provisioning at one + # time which usually results in fewer but larger nodes. + batchMaxDuration: 10s # 60s is a good value for large runs (500+ nodes) + env: + - name: ARM_SUBSCRIPTION_ID + value: "" + - name: LOCATION + value: westus2 + - name: ARM_USE_MANAGED_IDENTITY_EXTENSION + value: "true" + - name: ARM_USER_ASSIGNED_IDENTITY_ID + value: "" + - name: AZURE_NODE_RESOURCE_GROUP + value: "" + - name: AZURE_SUBNET_ID # the id of subnet to create network interfaces on + value: "" + - name: LEADER_ELECT # disable leader election for better debugging experience + value: "false" + - name: AZURE_VNET_NAME + value: "" + - name: AZURE_SUBNET_NAME + value: "" + # disable HTTP/2 to reduce ARM throttling on large-scale tests; + # with this in place write (and read) QPS can be increased too + #- name: GODEBUG + # value: http2client=0 + resources: + requests: + cpu: 200m + limits: + cpu: 500m + logLevel: debug +deploy: + helm: {} diff --git a/test/README.md b/test/README.md new file mode 100644 index 000000000..2ff33e120 --- /dev/null +++ b/test/README.md @@ -0,0 +1,7 @@ +# Testing Directory + +These test scenarios are designed to be used against a live AKS cluster running karpenter, and validate particular E2E scenarios. + +## File Directory +- `/suites`: Ginkgo test suites for particular scenarios live here. +- `/pkg`: Common code re-used across test suites lives here. diff --git a/test/pkg/debug/events.go b/test/pkg/debug/events.go new file mode 100644 index 000000000..89f546c5d --- /dev/null +++ b/test/pkg/debug/events.go @@ -0,0 +1,146 @@ +/* +Portions Copyright (c) Microsoft Corporation. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package debug + +import ( + "context" + "fmt" + "strings" + "time" + + "github.com/samber/lo" + "go.uber.org/multierr" + v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/fields" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +type EventClient struct { + start time.Time + kubeClient client.Client +} + +func NewEventClient(kubeClient client.Client) *EventClient { + return &EventClient{ + start: time.Now(), + kubeClient: kubeClient, + } +} + +func (c *EventClient) DumpEvents(ctx context.Context) error { + return multierr.Combine( + c.dumpKarpenterEvents(ctx), + c.dumpPodEvents(ctx), + c.dumpNodeEvents(ctx), + ) +} + +func (c *EventClient) dumpKarpenterEvents(ctx context.Context) error { + el := &v1.EventList{} + if err := c.kubeClient.List(ctx, el, client.InNamespace("karpenter")); err != nil { + return err + } + for k, v := range coallateEvents(filterTestEvents(el.Items, c.start)) { + fmt.Print(getEventInformation(k, v)) + } + return nil +} + +func (c *EventClient) dumpPodEvents(ctx context.Context) error { + el := &v1.EventList{} + if err := c.kubeClient.List(ctx, el, &client.ListOptions{ + FieldSelector: fields.SelectorFromSet(map[string]string{"involvedObject.kind": "Pod"}), + }); err != nil { + return err + } + events := lo.Filter(filterTestEvents(el.Items, c.start), func(e v1.Event, _ int) bool { + return e.InvolvedObject.Namespace != "kube-system" + }) + for k, v := range coallateEvents(events) { + fmt.Print(getEventInformation(k, v)) + } + return nil +} + +func (c *EventClient) dumpNodeEvents(ctx context.Context) error { + el := &v1.EventList{} + if err := c.kubeClient.List(ctx, el, &client.ListOptions{ + FieldSelector: fields.SelectorFromSet(map[string]string{"involvedObject.kind": "Node"}), + }); err != nil { + return err + } + for k, v := range coallateEvents(filterTestEvents(el.Items, c.start)) { + fmt.Print(getEventInformation(k, v)) + } + return nil +} + +func filterTestEvents(events []v1.Event, startTime time.Time) []v1.Event { + return lo.Filter(events, func(e v1.Event, _ int) bool { + if !e.EventTime.IsZero() { + if e.EventTime.BeforeTime(&metav1.Time{Time: startTime}) { + return false + } + } else if e.FirstTimestamp.Before(&metav1.Time{Time: startTime}) { + return false + } + return true + }) +} + +func coallateEvents(events []v1.Event) map[v1.ObjectReference]*v1.EventList { + eventMap := map[v1.ObjectReference]*v1.EventList{} + for i := range events { + elem := events[i] + objectKey := v1.ObjectReference{Kind: elem.InvolvedObject.Kind, Namespace: elem.InvolvedObject.Namespace, Name: elem.InvolvedObject.Name} + if _, ok := eventMap[objectKey]; !ok { + eventMap[objectKey] = &v1.EventList{} + } + eventMap[objectKey].Items = append(eventMap[objectKey].Items, elem) + } + return eventMap +} + +// Partially copied from +// https://github.com/kubernetes/kubernetes/blob/04ee339c7a4d36b4037ce3635993e2a9e395ebf3/staging/src/k8s.io/kubectl/pkg/describe/describe.go#L4232 +func getEventInformation(o v1.ObjectReference, el *v1.EventList) string { + sb := strings.Builder{} + sb.WriteString(fmt.Sprintf("------- %s/%s%s EVENTS -------\n", + strings.ToLower(o.Kind), lo.Ternary(o.Namespace != "", o.Namespace+"/", ""), o.Name)) + if len(el.Items) == 0 { + return sb.String() + } + for _, e := range el.Items { + source := e.Source.Component + if source == "" { + source = e.ReportingController + } + eventTime := e.EventTime + if eventTime.IsZero() { + eventTime = metav1.NewMicroTime(e.FirstTimestamp.Time) + } + sb.WriteString(fmt.Sprintf("time=%s type=%s reason=%s from=%s message=%s\n", + eventTime.Format(time.RFC3339), + e.Type, + e.Reason, + source, + strings.TrimSpace(e.Message)), + ) + } + return sb.String() +} diff --git a/test/pkg/debug/monitor.go b/test/pkg/debug/monitor.go new file mode 100644 index 000000000..1db4afa42 --- /dev/null +++ b/test/pkg/debug/monitor.go @@ -0,0 +1,92 @@ +/* +Portions Copyright (c) Microsoft Corporation. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package debug + +import ( + "context" + "sync" + + "github.com/go-logr/zapr" + "github.com/samber/lo" + "k8s.io/client-go/rest" + "knative.dev/pkg/logging" + controllerruntime "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + ctrl "sigs.k8s.io/controller-runtime/pkg/log" + "sigs.k8s.io/controller-runtime/pkg/manager" + "sigs.k8s.io/controller-runtime/pkg/metrics/server" + + "github.com/aws/karpenter-core/pkg/operator/controller" + "github.com/aws/karpenter-core/pkg/operator/scheme" +) + +type Monitor struct { + ctx context.Context + cancel context.CancelFunc + wg sync.WaitGroup + mgr manager.Manager +} + +func New(ctx context.Context, config *rest.Config, kubeClient client.Client) *Monitor { + logger := logging.FromContext(ctx) + ctrl.SetLogger(zapr.NewLogger(logger.Desugar())) + mgr := lo.Must(controllerruntime.NewManager(config, controllerruntime.Options{ + Scheme: scheme.Scheme, + BaseContext: func() context.Context { + ctx := context.Background() + ctx = logging.WithLogger(ctx, logger) + logger.WithOptions() + return ctx + }, + Metrics: server.Options{ + BindAddress: "0", + }, + })) + for _, c := range newControllers(kubeClient) { + lo.Must0(c.Builder(ctx, mgr).Complete(c), "failed to register controller") + } + ctx, cancel := context.WithCancel(ctx) // this context is only meant for monitor start/stop + return &Monitor{ + ctx: ctx, + cancel: cancel, + mgr: mgr, + } +} + +// MustStart starts the debug monitor +func (m *Monitor) MustStart() { + m.wg.Add(1) + go func() { + defer m.wg.Done() + lo.Must0(m.mgr.Start(m.ctx)) + }() +} + +// Stop stops the monitor +func (m *Monitor) Stop() { + m.cancel() + m.wg.Wait() +} + +func newControllers(kubeClient client.Client) []controller.Controller { + return []controller.Controller{ + // NewMachineController(kubeClient), + NewNodeClaimController(kubeClient), + NewNodeController(kubeClient), + NewPodController(kubeClient), + } +} diff --git a/test/pkg/debug/node.go b/test/pkg/debug/node.go new file mode 100644 index 000000000..5e6c4d483 --- /dev/null +++ b/test/pkg/debug/node.go @@ -0,0 +1,87 @@ +/* +Portions Copyright (c) Microsoft Corporation. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package debug + +import ( + "context" + "fmt" + "time" + + v1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/errors" + controllerruntime "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller" + "sigs.k8s.io/controller-runtime/pkg/event" + "sigs.k8s.io/controller-runtime/pkg/manager" + "sigs.k8s.io/controller-runtime/pkg/predicate" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + + "github.com/aws/karpenter-core/pkg/apis/v1alpha5" + corecontroller "github.com/aws/karpenter-core/pkg/operator/controller" + nodeutils "github.com/aws/karpenter-core/pkg/utils/node" +) + +type NodeController struct { + kubeClient client.Client +} + +func NewNodeController(kubeClient client.Client) *NodeController { + return &NodeController{ + kubeClient: kubeClient, + } +} + +func (c *NodeController) Name() string { + return "node" +} + +func (c *NodeController) Reconcile(ctx context.Context, req reconcile.Request) (reconcile.Result, error) { + n := &v1.Node{} + if err := c.kubeClient.Get(ctx, req.NamespacedName, n); err != nil { + if errors.IsNotFound(err) { + fmt.Printf("[DELETED %s] NODE %s\n", time.Now().Format(time.RFC3339), req.NamespacedName.String()) + } + return reconcile.Result{}, client.IgnoreNotFound(err) + } + fmt.Printf("[CREATED/UPDATED %s] NODE %s %s\n", time.Now().Format(time.RFC3339), req.NamespacedName.Name, c.GetInfo(ctx, n)) + return reconcile.Result{}, nil +} + +func (c *NodeController) GetInfo(ctx context.Context, n *v1.Node) string { + pods, _ := nodeutils.GetNodePods(ctx, c.kubeClient, n) + return fmt.Sprintf("ready=%s schedulable=%t initialized=%s pods=%d taints=%v", nodeutils.GetCondition(n, v1.NodeReady).Status, !n.Spec.Unschedulable, n.Labels[v1alpha5.LabelNodeInitialized], len(pods), n.Spec.Taints) +} + +func (c *NodeController) Builder(ctx context.Context, m manager.Manager) corecontroller.Builder { + return corecontroller.Adapt(controllerruntime. + NewControllerManagedBy(m). + For(&v1.Node{}). + WithEventFilter(predicate.And( + predicate.Funcs{ + UpdateFunc: func(e event.UpdateEvent) bool { + oldNode := e.ObjectOld.(*v1.Node) + newNode := e.ObjectNew.(*v1.Node) + return c.GetInfo(ctx, oldNode) != c.GetInfo(ctx, newNode) + }, + }, + predicate.NewPredicateFuncs(func(o client.Object) bool { + return o.GetLabels()[v1alpha5.ProvisionerNameLabelKey] != "" + }), + )). + WithOptions(controller.Options{MaxConcurrentReconciles: 10})) +} diff --git a/test/pkg/debug/nodeclaim.go b/test/pkg/debug/nodeclaim.go new file mode 100644 index 000000000..92cf4402f --- /dev/null +++ b/test/pkg/debug/nodeclaim.go @@ -0,0 +1,84 @@ +/* +Portions Copyright (c) Microsoft Corporation. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package debug + +import ( + "context" + "fmt" + "time" + + "k8s.io/apimachinery/pkg/api/errors" + controllerruntime "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller" + "sigs.k8s.io/controller-runtime/pkg/event" + "sigs.k8s.io/controller-runtime/pkg/manager" + "sigs.k8s.io/controller-runtime/pkg/predicate" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + + corev1beta1 "github.com/aws/karpenter-core/pkg/apis/v1beta1" + corecontroller "github.com/aws/karpenter-core/pkg/operator/controller" +) + +type NodeClaimController struct { + kubeClient client.Client +} + +func NewNodeClaimController(kubeClient client.Client) *NodeClaimController { + return &NodeClaimController{ + kubeClient: kubeClient, + } +} + +func (c *NodeClaimController) Name() string { + return "nodeclaim" +} + +func (c *NodeClaimController) Reconcile(ctx context.Context, req reconcile.Request) (reconcile.Result, error) { + nc := &corev1beta1.NodeClaim{} + if err := c.kubeClient.Get(ctx, req.NamespacedName, nc); err != nil { + if errors.IsNotFound(err) { + fmt.Printf("[DELETED %s] NODECLAIM %s\n", time.Now().Format(time.RFC3339), req.NamespacedName.String()) + } + return reconcile.Result{}, client.IgnoreNotFound(err) + } + fmt.Printf("[CREATED/UPDATED %s] NODECLAIM %s %s\n", time.Now().Format(time.RFC3339), req.NamespacedName.Name, c.GetInfo(nc)) + return reconcile.Result{}, nil +} + +func (c *NodeClaimController) GetInfo(nc *corev1beta1.NodeClaim) string { + return fmt.Sprintf("ready=%t launched=%t registered=%t initialized=%t", + nc.StatusConditions().IsHappy(), + nc.StatusConditions().GetCondition(corev1beta1.Launched).IsTrue(), + nc.StatusConditions().GetCondition(corev1beta1.Registered).IsTrue(), + nc.StatusConditions().GetCondition(corev1beta1.Initialized).IsTrue(), + ) +} + +func (c *NodeClaimController) Builder(_ context.Context, m manager.Manager) corecontroller.Builder { + return corecontroller.Adapt(controllerruntime. + NewControllerManagedBy(m). + For(&corev1beta1.NodeClaim{}). + WithEventFilter(predicate.Funcs{ + UpdateFunc: func(e event.UpdateEvent) bool { + oldNodeClaim := e.ObjectOld.(*corev1beta1.NodeClaim) + newNodeClaim := e.ObjectNew.(*corev1beta1.NodeClaim) + return c.GetInfo(oldNodeClaim) != c.GetInfo(newNodeClaim) + }, + }). + WithOptions(controller.Options{MaxConcurrentReconciles: 10})) +} diff --git a/test/pkg/debug/pod.go b/test/pkg/debug/pod.go new file mode 100644 index 000000000..10c541361 --- /dev/null +++ b/test/pkg/debug/pod.go @@ -0,0 +1,95 @@ +/* +Portions Copyright (c) Microsoft Corporation. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package debug + +import ( + "context" + "fmt" + "strings" + "time" + + "github.com/samber/lo" + v1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/errors" + controllerruntime "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller" + "sigs.k8s.io/controller-runtime/pkg/event" + "sigs.k8s.io/controller-runtime/pkg/manager" + "sigs.k8s.io/controller-runtime/pkg/predicate" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + + corecontroller "github.com/aws/karpenter-core/pkg/operator/controller" + "github.com/aws/karpenter-core/pkg/utils/pod" +) + +type PodController struct { + kubeClient client.Client +} + +func NewPodController(kubeClient client.Client) *PodController { + return &PodController{ + kubeClient: kubeClient, + } +} + +func (c *PodController) Name() string { + return "pod" +} + +func (c *PodController) Reconcile(ctx context.Context, req reconcile.Request) (reconcile.Result, error) { + p := &v1.Pod{} + if err := c.kubeClient.Get(ctx, req.NamespacedName, p); err != nil { + if errors.IsNotFound(err) { + fmt.Printf("[DELETED %s] POD %s\n", time.Now().Format(time.RFC3339), req.NamespacedName.String()) + } + return reconcile.Result{}, client.IgnoreNotFound(err) + } + fmt.Printf("[CREATED/UPDATED %s] POD %s %s\n", time.Now().Format(time.RFC3339), req.NamespacedName.String(), c.GetInfo(p)) + return reconcile.Result{}, nil +} + +func (c *PodController) GetInfo(p *v1.Pod) string { + var containerInfo strings.Builder + for _, c := range p.Status.ContainerStatuses { + if containerInfo.Len() > 0 { + _ = lo.Must(fmt.Fprintf(&containerInfo, ", ")) + } + _ = lo.Must(fmt.Fprintf(&containerInfo, "%s restarts=%d", c.Name, c.RestartCount)) + } + return fmt.Sprintf("provisionable=%v phase=%s nodename=%s owner=%#v [%s]", + pod.IsProvisionable(p), p.Status.Phase, p.Spec.NodeName, p.OwnerReferences, containerInfo.String()) +} + +func (c *PodController) Builder(_ context.Context, m manager.Manager) corecontroller.Builder { + return corecontroller.Adapt(controllerruntime. + NewControllerManagedBy(m). + For(&v1.Pod{}). + WithEventFilter(predicate.And( + predicate.Funcs{ + UpdateFunc: func(e event.UpdateEvent) bool { + oldPod := e.ObjectOld.(*v1.Pod) + newPod := e.ObjectNew.(*v1.Pod) + return c.GetInfo(oldPod) != c.GetInfo(newPod) + }, + }, + predicate.NewPredicateFuncs(func(o client.Object) bool { + return o.GetNamespace() != "kube-system" && o.GetNamespace() != "karpenter" + }), + )). + WithOptions(controller.Options{MaxConcurrentReconciles: 10})) +} diff --git a/test/pkg/debug/setup.go b/test/pkg/debug/setup.go new file mode 100644 index 000000000..b03671ebc --- /dev/null +++ b/test/pkg/debug/setup.go @@ -0,0 +1,57 @@ +/* +Portions Copyright (c) Microsoft Corporation. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package debug + +import ( + "context" + + "github.com/onsi/ginkgo/v2" + "github.com/samber/lo" + "k8s.io/client-go/rest" + "sigs.k8s.io/controller-runtime/pkg/client" + + . "github.com/onsi/gomega" //nolint:revive,stylecheck +) + +const ( + NoWatch = "NoWatch" + NoEvents = "NoEvents" +) + +var m *Monitor +var e *EventClient + +func BeforeEach(ctx context.Context, config *rest.Config, kubeClient client.Client) { + // If the test is labeled as NoWatch, then the node/pod monitor will just list at the beginning + // of the test rather than perform a watch during it + if !lo.Contains(ginkgo.CurrentSpecReport().Labels(), NoWatch) { + m = New(ctx, config, kubeClient) + m.MustStart() + } + if !lo.Contains(ginkgo.CurrentSpecReport().Labels(), NoEvents) { + e = NewEventClient(kubeClient) + } +} + +func AfterEach(ctx context.Context) { + if !lo.Contains(ginkgo.CurrentSpecReport().Labels(), NoWatch) { + m.Stop() + } + if !lo.Contains(ginkgo.CurrentSpecReport().Labels(), NoEvents) { + Expect(e.DumpEvents(ctx)).To(Succeed()) + } +} diff --git a/test/pkg/environment/azure/environment.go b/test/pkg/environment/azure/environment.go new file mode 100644 index 000000000..7ee679b70 --- /dev/null +++ b/test/pkg/environment/azure/environment.go @@ -0,0 +1,102 @@ +/* +Portions Copyright (c) Microsoft Corporation. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package azure + +import ( + "testing" + + v1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" + + corev1beta1 "github.com/aws/karpenter-core/pkg/apis/v1beta1" + coretest "github.com/aws/karpenter-core/pkg/test" + + "github.com/Azure/karpenter/pkg/apis/v1alpha2" + "github.com/Azure/karpenter/pkg/test" + "github.com/Azure/karpenter/test/pkg/environment/common" +) + +const WindowsDefaultImage = "mcr.microsoft.com/oss/kubernetes/pause:3.9" + +type Environment struct { + *common.Environment + Region string + + // TODO (charliedmcb): explore porting over/implementing an Azure equivalent version + // SQSProvider *interruption.SQSProvider +} + +func NewEnvironment(t *testing.T) *Environment { + env := common.NewEnvironment(t) + + return &Environment{ + Region: "westus2", + Environment: env, + + // SQSProvider: interruption.NewSQSProvider(sqs.New(session)), + } +} + +func (env *Environment) DefaultNodePool(nodeClass *v1alpha2.AKSNodeClass) *corev1beta1.NodePool { + nodePool := coretest.NodePool() + nodePool.Spec.Template.Spec.NodeClassRef = &corev1beta1.NodeClassReference{ + Name: nodeClass.Name, + } + nodePool.Spec.Template.Spec.Requirements = []v1.NodeSelectorRequirement{ + { + Key: v1.LabelOSStable, + Operator: v1.NodeSelectorOpIn, + Values: []string{string(v1.Linux)}, + }, + { + Key: corev1beta1.CapacityTypeLabelKey, + Operator: v1.NodeSelectorOpIn, + Values: []string{corev1beta1.CapacityTypeOnDemand}, + }, + // TODO: remove constraint when arm64 is supported + { + Key: v1.LabelArchStable, + Operator: v1.NodeSelectorOpIn, + Values: []string{corev1beta1.ArchitectureAmd64}, + }, + { + Key: v1alpha2.LabelSKUHyperVGeneration, + Operator: v1.NodeSelectorOpIn, + Values: []string{v1alpha2.HyperVGenerationV2}, + }, + { + Key: v1alpha2.LabelSKUFamily, + Operator: v1.NodeSelectorOpIn, + Values: []string{"D"}, + }, + } + nodePool.Spec.Disruption.ConsolidateAfter = &corev1beta1.NillableDuration{} + nodePool.Spec.Disruption.ExpireAfter.Duration = nil + nodePool.Spec.Limits = corev1beta1.Limits(v1.ResourceList{ + v1.ResourceCPU: resource.MustParse("100"), + v1.ResourceMemory: resource.MustParse("1000Gi"), + }) + return nodePool +} + +func (env *Environment) DefaultAKSNodeClass() *v1alpha2.AKSNodeClass { + nodeClass := test.AKSNodeClass() + + // TODO: override values for testing + + return nodeClass +} diff --git a/test/pkg/environment/azure/setup.go b/test/pkg/environment/azure/setup.go new file mode 100644 index 000000000..6161bd962 --- /dev/null +++ b/test/pkg/environment/azure/setup.go @@ -0,0 +1,52 @@ +/* +Portions Copyright (c) Microsoft Corporation. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package azure + +import ( + //nolint:revive,stylecheck + v1 "k8s.io/api/core/v1" + "sigs.k8s.io/controller-runtime/pkg/client" + + "github.com/Azure/karpenter/pkg/apis/v1alpha2" +) + +var persistedSettings []v1.EnvVar +var persistedSettingsLegacy = &v1.ConfigMap{} + +var ( + CleanableObjects = []client.Object{ + &v1alpha2.AKSNodeClass{}, + } +) + +func (env *Environment) BeforeEach() { + persistedSettings = env.ExpectSettings() + persistedSettingsLegacy = env.ExpectSettingsLegacy() + env.Environment.BeforeEach() +} + +func (env *Environment) Cleanup() { + env.Environment.Cleanup() + env.Environment.CleanupObjects(CleanableObjects...) +} + +func (env *Environment) AfterEach() { + env.Environment.AfterEach() + // Ensure we reset settings after collecting the controller logs + env.ExpectSettingsReplaced(persistedSettings...) + env.ExpectSettingsReplacedLegacy(persistedSettingsLegacy.Data) +} diff --git a/test/pkg/environment/common/environment.go b/test/pkg/environment/common/environment.go new file mode 100644 index 000000000..7dca7aae3 --- /dev/null +++ b/test/pkg/environment/common/environment.go @@ -0,0 +1,138 @@ +/* +Portions Copyright (c) Microsoft Corporation. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package common + +import ( + "context" + "fmt" + "log" + "os" + "strconv" + "testing" + "time" + + "github.com/onsi/gomega" + "github.com/samber/lo" + v1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/client-go/kubernetes" + clientgoscheme "k8s.io/client-go/kubernetes/scheme" + "k8s.io/client-go/rest" + loggingtesting "knative.dev/pkg/logging/testing" + "knative.dev/pkg/system" + controllerruntime "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/cache" + "sigs.k8s.io/controller-runtime/pkg/client" + + "github.com/Azure/karpenter/pkg/apis" + coreapis "github.com/aws/karpenter-core/pkg/apis" + corev1beta1 "github.com/aws/karpenter-core/pkg/apis/v1beta1" + "github.com/aws/karpenter-core/pkg/operator" + "github.com/aws/karpenter-core/pkg/operator/injection" +) + +type ContextKey string + +const ( + GitRefContextKey = ContextKey("gitRef") +) + +type Environment struct { + context.Context + cancel context.CancelFunc + + Client client.Client + Config *rest.Config + KubeClient kubernetes.Interface + Monitor *Monitor + + StartingNodeCount int +} + +func NewEnvironment(t *testing.T) *Environment { + ctx := loggingtesting.TestContextWithLogger(t) + ctx, cancel := context.WithCancel(ctx) + config := NewConfig() + client := NewClient(ctx, config) + + lo.Must0(os.Setenv(system.NamespaceEnvKey, "karpenter")) + kubernetesInterface := kubernetes.NewForConfigOrDie(config) + ctx = injection.WithSettingsOrDie(ctx, kubernetesInterface, apis.Settings...) + if val, ok := os.LookupEnv("GIT_REF"); ok { + ctx = context.WithValue(ctx, GitRefContextKey, val) + } + + gomega.SetDefaultEventuallyTimeout(5 * time.Minute) + gomega.SetDefaultEventuallyPollingInterval(1 * time.Second) + return &Environment{ + Context: ctx, + cancel: cancel, + Config: config, + Client: client, + KubeClient: kubernetes.NewForConfigOrDie(config), + Monitor: NewMonitor(ctx, client), + } +} + +func (env *Environment) Stop() { + env.cancel() +} + +func NewConfig() *rest.Config { + config := controllerruntime.GetConfigOrDie() + config.UserAgent = fmt.Sprintf("testing-%s", operator.Version) + config.QPS = 1e6 + config.Burst = 1e6 + return config +} + +func NewClient(ctx context.Context, config *rest.Config) client.Client { + scheme := runtime.NewScheme() + lo.Must0(clientgoscheme.AddToScheme(scheme)) + lo.Must0(apis.AddToScheme(scheme)) + lo.Must0(coreapis.AddToScheme(scheme)) + + cache := lo.Must(cache.New(config, cache.Options{Scheme: scheme})) + lo.Must0(cache.IndexField(ctx, &v1.Pod{}, "spec.nodeName", func(o client.Object) []string { + pod := o.(*v1.Pod) + return []string{pod.Spec.NodeName} + })) + lo.Must0(cache.IndexField(ctx, &v1.Event{}, "involvedObject.kind", func(o client.Object) []string { + evt := o.(*v1.Event) + return []string{evt.InvolvedObject.Kind} + })) + lo.Must0(cache.IndexField(ctx, &v1.Node{}, "spec.unschedulable", func(o client.Object) []string { + node := o.(*v1.Node) + return []string{strconv.FormatBool(node.Spec.Unschedulable)} + })) + lo.Must0(cache.IndexField(ctx, &v1.Node{}, "spec.taints[*].karpenter.sh/disruption", func(o client.Object) []string { + node := o.(*v1.Node) + t, _ := lo.Find(node.Spec.Taints, func(t v1.Taint) bool { + return t.Key == corev1beta1.DisruptionTaintKey + }) + return []string{t.Value} + })) + c := lo.Must(client.New(config, client.Options{Scheme: scheme, Cache: &client.CacheOptions{Reader: cache}})) + + go func() { + lo.Must0(cache.Start(ctx)) + }() + if !cache.WaitForCacheSync(ctx) { + log.Fatalf("cache failed to sync") + } + return c +} diff --git a/test/pkg/environment/common/expectations.go b/test/pkg/environment/common/expectations.go new file mode 100644 index 000000000..ba728b20f --- /dev/null +++ b/test/pkg/environment/common/expectations.go @@ -0,0 +1,812 @@ +/* +Portions Copyright (c) Microsoft Corporation. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package common + +import ( + "bytes" + "encoding/base64" + "fmt" + "io" + "strings" + "time" + + . "github.com/onsi/ginkgo/v2" //nolint:revive,stylecheck + . "github.com/onsi/gomega" //nolint:revive,stylecheck + "github.com/samber/lo" + appsv1 "k8s.io/api/apps/v1" + coordinationv1 "k8s.io/api/coordination/v1" + v1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/equality" + "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/labels" + "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/sets" + "k8s.io/client-go/transport" + "knative.dev/pkg/logging" + "knative.dev/pkg/ptr" + "sigs.k8s.io/controller-runtime/pkg/client" + + "github.com/aws/karpenter-core/pkg/apis/v1alpha5" + corev1beta1 "github.com/aws/karpenter-core/pkg/apis/v1beta1" + pscheduling "github.com/aws/karpenter-core/pkg/controllers/provisioning/scheduling" + "github.com/aws/karpenter-core/pkg/scheduling" + "github.com/aws/karpenter-core/pkg/test" + nodepoolutil "github.com/aws/karpenter-core/pkg/utils/nodepool" +) + +func (env *Environment) ExpectCreated(objects ...client.Object) { + GinkgoHelper() + for _, object := range objects { + Eventually(func(g Gomega) { + object.SetLabels(lo.Assign(object.GetLabels(), map[string]string{ + test.DiscoveryLabel: "unspecified", + })) + g.Expect(env.Client.Create(env, object)).To(Succeed()) + }).WithTimeout(time.Second * 10).Should(Succeed()) + } +} + +func (env *Environment) ExpectDeleted(objects ...client.Object) { + GinkgoHelper() + for _, object := range objects { + Eventually(func(g Gomega) { + g.Expect(client.IgnoreNotFound(env.Client.Delete(env, object, client.PropagationPolicy(metav1.DeletePropagationForeground), &client.DeleteOptions{GracePeriodSeconds: ptr.Int64(0)}))).To(Succeed()) + }).WithTimeout(time.Second * 10).Should(Succeed()) + } +} + +func (env *Environment) ExpectUpdated(objects ...client.Object) { + GinkgoHelper() + for _, o := range objects { + Eventually(func(g Gomega) { + current := o.DeepCopyObject().(client.Object) + g.Expect(env.Client.Get(env.Context, client.ObjectKeyFromObject(current), current)).To(Succeed()) + o.SetResourceVersion(current.GetResourceVersion()) + g.Expect(env.Client.Update(env.Context, o)).To(Succeed()) + }).WithTimeout(time.Second * 10).Should(Succeed()) + } +} + +func (env *Environment) ExpectCreatedOrUpdated(objects ...client.Object) { + GinkgoHelper() + for _, o := range objects { + current := o.DeepCopyObject().(client.Object) + err := env.Client.Get(env, client.ObjectKeyFromObject(current), current) + if err != nil { + if errors.IsNotFound(err) { + env.ExpectCreated(o) + } else { + Fail(fmt.Sprintf("Getting object %s, %v", client.ObjectKeyFromObject(o), err)) + } + } else { + env.ExpectUpdated(o) + } + } +} + +func (env *Environment) ExpectSettings() (res []v1.EnvVar) { + GinkgoHelper() + + d := &appsv1.Deployment{} + Expect(env.Client.Get(env.Context, types.NamespacedName{Namespace: "karpenter", Name: "karpenter"}, d)).To(Succeed()) + Expect(d.Spec.Template.Spec.Containers).To(HaveLen(1)) + return lo.Map(d.Spec.Template.Spec.Containers[0].Env, func(v v1.EnvVar, _ int) v1.EnvVar { + return *v.DeepCopy() + }) +} + +func (env *Environment) ExpectSettingsReplaced(vars ...v1.EnvVar) { + GinkgoHelper() + + d := &appsv1.Deployment{} + Expect(env.Client.Get(env.Context, types.NamespacedName{Namespace: "karpenter", Name: "karpenter"}, d)).To(Succeed()) + Expect(d.Spec.Template.Spec.Containers).To(HaveLen(1)) + + stored := d.DeepCopy() + d.Spec.Template.Spec.Containers[0].Env = vars + + if !equality.Semantic.DeepEqual(d, stored) { + By("replacing environment variables for karpenter deployment") + Expect(env.Client.Patch(env.Context, d, client.MergeFrom(stored))).To(Succeed()) + env.EventuallyExpectKarpenterRestarted() + } +} + +func (env *Environment) ExpectSettingsOverridden(vars ...v1.EnvVar) { + GinkgoHelper() + + d := &appsv1.Deployment{} + Expect(env.Client.Get(env.Context, types.NamespacedName{Namespace: "karpenter", Name: "karpenter"}, d)).To(Succeed()) + Expect(d.Spec.Template.Spec.Containers).To(HaveLen(1)) + + stored := d.DeepCopy() + for _, v := range vars { + if _, i, ok := lo.FindIndexOf(d.Spec.Template.Spec.Containers[0].Env, func(e v1.EnvVar) bool { + return e.Name == v.Name + }); ok { + d.Spec.Template.Spec.Containers[0].Env[i] = v + } else { + d.Spec.Template.Spec.Containers[0].Env = append(d.Spec.Template.Spec.Containers[0].Env, v) + } + } + if !equality.Semantic.DeepEqual(d, stored) { + By("overriding environment variables for karpenter deployment") + Expect(env.Client.Patch(env.Context, d, client.MergeFrom(stored))).To(Succeed()) + env.EventuallyExpectKarpenterRestarted() + } +} + +func (env *Environment) ExpectSettingsRemoved(vars ...v1.EnvVar) { + GinkgoHelper() + + varNames := sets.New[string](lo.Map(vars, func(v v1.EnvVar, _ int) string { return v.Name })...) + + d := &appsv1.Deployment{} + Expect(env.Client.Get(env.Context, types.NamespacedName{Namespace: "karpenter", Name: "karpenter"}, d)).To(Succeed()) + Expect(d.Spec.Template.Spec.Containers).To(HaveLen(1)) + + stored := d.DeepCopy() + d.Spec.Template.Spec.Containers[0].Env = lo.Reject(d.Spec.Template.Spec.Containers[0].Env, func(v v1.EnvVar, _ int) bool { + return varNames.Has(v.Name) + }) + if !equality.Semantic.DeepEqual(d, stored) { + By("removing environment variables for karpenter deployment") + Expect(env.Client.Patch(env.Context, d, client.MergeFrom(stored))).To(Succeed()) + env.EventuallyExpectKarpenterRestarted() + } +} + +// ExpectSettingsLegacy gets the karpenter-global-settings ConfigMap +func (env *Environment) ExpectSettingsLegacy() *v1.ConfigMap { + GinkgoHelper() + return env.ExpectConfigMapExists(types.NamespacedName{Namespace: "karpenter", Name: "karpenter-global-settings"}) +} + +// ExpectSettingsReplacedLegacy performs a full replace of the settings, replacing the existing data +// with the data passed through +func (env *Environment) ExpectSettingsReplacedLegacy(data ...map[string]string) { + GinkgoHelper() + if env.ExpectConfigMapDataReplaced(types.NamespacedName{Namespace: "karpenter", Name: "karpenter-global-settings"}, data...) { + env.EventuallyExpectKarpenterRestarted() + } +} + +// ExpectSettingsOverriddenLegacy overrides specific values specified through data. It only overrides +// or inserts the specific values specified and does not upsert any of the existing data +func (env *Environment) ExpectSettingsOverriddenLegacy(data ...map[string]string) { + GinkgoHelper() + if env.ExpectConfigMapDataOverridden(types.NamespacedName{Namespace: "karpenter", Name: "karpenter-global-settings"}, data...) { + env.EventuallyExpectKarpenterRestarted() + } +} + +func (env *Environment) ExpectConfigMapExists(key types.NamespacedName) *v1.ConfigMap { + GinkgoHelper() + cm := &v1.ConfigMap{} + Expect(env.Client.Get(env, key, cm)).To(Succeed()) + return cm +} + +func (env *Environment) ExpectConfigMapDataReplaced(key types.NamespacedName, data ...map[string]string) (changed bool) { + GinkgoHelper() + cm := &v1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: key.Name, + Namespace: key.Namespace, + }, + } + err := env.Client.Get(env, key, cm) + Expect(client.IgnoreNotFound(err)).ToNot(HaveOccurred()) + + stored := cm.DeepCopy() + cm.Data = lo.Assign(data...) // Completely replace the data + + // If the data hasn't changed, we can just return and not update anything + if equality.Semantic.DeepEqual(stored, cm) { + return false + } + // Update the configMap to update the settings + env.ExpectCreatedOrUpdated(cm) + return true +} + +func (env *Environment) ExpectConfigMapDataOverridden(key types.NamespacedName, data ...map[string]string) (changed bool) { + GinkgoHelper() + cm := &v1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: key.Name, + Namespace: key.Namespace, + }, + } + err := env.Client.Get(env, key, cm) + Expect(client.IgnoreNotFound(err)).ToNot(HaveOccurred()) + + stored := cm.DeepCopy() + cm.Data = lo.Assign(append([]map[string]string{cm.Data}, data...)...) + + // If the data hasn't changed, we can just return and not update anything + if equality.Semantic.DeepEqual(stored, cm) { + return false + } + // Update the configMap to update the settings + env.ExpectCreatedOrUpdated(cm) + return true +} + +func (env *Environment) ExpectPodENIEnabled() { + GinkgoHelper() + env.ExpectDaemonSetEnvironmentVariableUpdated(types.NamespacedName{Namespace: "kube-system", Name: "aws-node"}, + "ENABLE_POD_ENI", "true", "aws-node") +} + +func (env *Environment) ExpectPodENIDisabled() { + GinkgoHelper() + env.ExpectDaemonSetEnvironmentVariableUpdated(types.NamespacedName{Namespace: "kube-system", Name: "aws-node"}, + "ENABLE_POD_ENI", "false", "aws-node") +} + +func (env *Environment) ExpectPrefixDelegationEnabled() { + GinkgoHelper() + env.ExpectDaemonSetEnvironmentVariableUpdated(types.NamespacedName{Namespace: "kube-system", Name: "aws-node"}, + "ENABLE_PREFIX_DELEGATION", "true", "aws-node") +} + +func (env *Environment) ExpectPrefixDelegationDisabled() { + GinkgoHelper() + env.ExpectDaemonSetEnvironmentVariableUpdated(types.NamespacedName{Namespace: "kube-system", Name: "aws-node"}, + "ENABLE_PREFIX_DELEGATION", "false", "aws-node") +} + +func (env *Environment) ExpectExists(obj client.Object) { + GinkgoHelper() + Eventually(func(g Gomega) { + g.Expect(env.Client.Get(env, client.ObjectKeyFromObject(obj), obj)).To(Succeed()) + }).WithTimeout(time.Second * 5).Should(Succeed()) +} + +func (env *Environment) EventuallyExpectHealthy(pods ...*v1.Pod) { + GinkgoHelper() + env.EventuallyExpectHealthyWithTimeout(-1, pods...) +} + +func (env *Environment) EventuallyExpectHealthyWithTimeout(timeout time.Duration, pods ...*v1.Pod) { + GinkgoHelper() + for _, pod := range pods { + Eventually(func(g Gomega) { + g.Expect(env.Client.Get(env, client.ObjectKeyFromObject(pod), pod)).To(Succeed()) + g.Expect(pod.Status.Conditions).To(ContainElement(And( + HaveField("Type", Equal(v1.PodReady)), + HaveField("Status", Equal(v1.ConditionTrue)), + ))) + }).WithTimeout(timeout).Should(Succeed()) + } +} + +func (env *Environment) EventuallyExpectKarpenterRestarted() { + GinkgoHelper() + By("rolling out the new karpenter deployment") + env.EventuallyExpectRollout("karpenter", "karpenter") + env.ExpectKarpenterLeaseOwnerChanged() +} + +func (env *Environment) ExpectKarpenterLeaseOwnerChanged() { + GinkgoHelper() + + By("waiting for a new karpenter pod to hold the lease") + pods := env.ExpectKarpenterPods() + Eventually(func(g Gomega) { + name := env.ExpectActiveKarpenterPodName() + g.Expect(lo.ContainsBy(pods, func(p *v1.Pod) bool { + return p.Name == name + })).To(BeTrue()) + }).Should(Succeed()) +} + +func (env *Environment) EventuallyExpectRollout(name, namespace string) { + GinkgoHelper() + By("restarting the deployment") + deploy := &appsv1.Deployment{} + Expect(env.Client.Get(env.Context, types.NamespacedName{Name: name, Namespace: namespace}, deploy)).To(Succeed()) + + stored := deploy.DeepCopy() + restartedAtAnnotation := map[string]string{ + "kubectl.kubernetes.io/restartedAt": time.Now().Format(time.RFC3339), + } + deploy.Spec.Template.Annotations = lo.Assign(deploy.Spec.Template.Annotations, restartedAtAnnotation) + Expect(env.Client.Patch(env.Context, deploy, client.MergeFrom(stored))).To(Succeed()) + + By("waiting for the newly generated deployment to rollout") + Eventually(func(g Gomega) { + podList := &v1.PodList{} + g.Expect(env.Client.List(env.Context, podList, client.InNamespace(namespace))).To(Succeed()) + pods := lo.Filter(podList.Items, func(p v1.Pod, _ int) bool { + return p.Annotations["kubectl.kubernetes.io/restartedAt"] == restartedAtAnnotation["kubectl.kubernetes.io/restartedAt"] + }) + g.Expect(len(pods)).To(BeNumerically("==", lo.FromPtr(deploy.Spec.Replicas))) + for _, pod := range pods { + g.Expect(pod.Status.Conditions).To(ContainElement(And( + HaveField("Type", Equal(v1.PodReady)), + HaveField("Status", Equal(v1.ConditionTrue)), + ))) + g.Expect(pod.Status.Phase).To(Equal(v1.PodRunning)) + } + }).Should(Succeed()) +} + +func (env *Environment) ExpectKarpenterPods() []*v1.Pod { + GinkgoHelper() + podList := &v1.PodList{} + Expect(env.Client.List(env.Context, podList, client.MatchingLabels{ + "app.kubernetes.io/instance": "karpenter", + })).To(Succeed()) + return lo.Map(podList.Items, func(p v1.Pod, _ int) *v1.Pod { return &p }) +} + +func (env *Environment) ExpectActiveKarpenterPodName() string { + GinkgoHelper() + lease := &coordinationv1.Lease{} + Expect(env.Client.Get(env.Context, types.NamespacedName{Name: "karpenter-leader-election", Namespace: "karpenter"}, lease)).To(Succeed()) + + // Holder identity for lease is always in the format "_ + holderArr := strings.Split(lo.FromPtr(lease.Spec.HolderIdentity), "_") + Expect(len(holderArr)).To(BeNumerically(">", 0)) + + return holderArr[0] +} + +func (env *Environment) ExpectActiveKarpenterPod() *v1.Pod { + GinkgoHelper() + podName := env.ExpectActiveKarpenterPodName() + + pod := &v1.Pod{} + Expect(env.Client.Get(env.Context, types.NamespacedName{Name: podName, Namespace: "karpenter"}, pod)).To(Succeed()) + return pod +} + +func (env *Environment) EventuallyExpectPendingPodCount(selector labels.Selector, numPods int) { + GinkgoHelper() + Eventually(func(g Gomega) { + g.Expect(env.Monitor.PendingPodsCount(selector)).To(Equal(numPods)) + }).Should(Succeed()) +} + +func (env *Environment) EventuallyExpectHealthyPodCount(selector labels.Selector, numPods int) { + By(fmt.Sprintf("waiting for %d pods matching selector %s to be ready", numPods, selector.String())) + GinkgoHelper() + env.EventuallyExpectHealthyPodCountWithTimeout(-1, selector, numPods) +} + +func (env *Environment) EventuallyExpectHealthyPodCountWithTimeout(timeout time.Duration, selector labels.Selector, numPods int) { + GinkgoHelper() + Eventually(func(g Gomega) { + g.Expect(env.Monitor.RunningPodsCount(selector)).To(Equal(numPods)) + }).WithTimeout(timeout).Should(Succeed()) +} + +func (env *Environment) ExpectPodsMatchingSelector(selector labels.Selector) []*v1.Pod { + GinkgoHelper() + + podList := &v1.PodList{} + Expect(env.Client.List(env.Context, podList, client.MatchingLabelsSelector{Selector: selector})).To(Succeed()) + return lo.ToSlicePtr(podList.Items) +} + +func (env *Environment) EventuallyExpectUniqueNodeNames(selector labels.Selector, uniqueNames int) { + GinkgoHelper() + + Eventually(func(g Gomega) { + pods := env.Monitor.RunningPods(selector) + nodeNames := sets.NewString() + for _, pod := range pods { + nodeNames.Insert(pod.Spec.NodeName) + } + g.Expect(len(nodeNames)).To(BeNumerically("==", uniqueNames)) + }).Should(Succeed()) +} + +func (env *Environment) eventuallyExpectScaleDown() { + GinkgoHelper() + Eventually(func(g Gomega) { + // expect the current node count to be what it was when the test started + g.Expect(env.Monitor.NodeCount()).To(Equal(env.StartingNodeCount)) + }).Should(Succeed(), fmt.Sprintf("expected scale down to %d nodes, had %d", env.StartingNodeCount, env.Monitor.NodeCount())) +} + +func (env *Environment) EventuallyExpectNotFound(objects ...client.Object) { + GinkgoHelper() + env.EventuallyExpectNotFoundAssertion(objects...).Should(Succeed()) +} + +func (env *Environment) EventuallyExpectNotFoundAssertion(objects ...client.Object) AsyncAssertion { + return Eventually(func(g Gomega) { + for _, object := range objects { + err := env.Client.Get(env, client.ObjectKeyFromObject(object), object) + g.Expect(errors.IsNotFound(err)).To(BeTrue()) + } + }) +} + +func (env *Environment) ExpectCreatedNodeCount(comparator string, count int) []*v1.Node { + GinkgoHelper() + createdNodes := env.Monitor.CreatedNodes() + Expect(len(createdNodes)).To(BeNumerically(comparator, count), + fmt.Sprintf("expected %d created nodes, had %d (%v)", count, len(createdNodes), NodeNames(createdNodes))) + return createdNodes +} + +func MachineNames(machines []*v1alpha5.Machine) []string { + return lo.Map(machines, func(m *v1alpha5.Machine, index int) string { + return m.Name + }) +} + +func NodeNames(nodes []*v1.Node) []string { + return lo.Map(nodes, func(n *v1.Node, index int) string { + return n.Name + }) +} + +func (env *Environment) ConsistentlyExpectNodeCount(comparator string, count int, duration string) []*v1.Node { + GinkgoHelper() + By(fmt.Sprintf("expecting nodes to be %s to %d for %s", comparator, count, duration)) + nodeList := &v1.NodeList{} + Consistently(func(g Gomega) { + g.Expect(env.Client.List(env, nodeList, client.HasLabels{test.DiscoveryLabel})).To(Succeed()) + g.Expect(len(nodeList.Items)).To(BeNumerically(comparator, count), + fmt.Sprintf("expected %d nodes, had %d (%v) for %s", count, len(nodeList.Items), NodeNames(lo.ToSlicePtr(nodeList.Items)), duration)) + }, duration).Should(Succeed()) + return lo.ToSlicePtr(nodeList.Items) +} + +func (env *Environment) ConsistentlyExpectMachineCount(comparator string, count int, duration string) []*v1alpha5.Machine { + GinkgoHelper() + By(fmt.Sprintf("expecting machines to be %s to %d for %s", comparator, count, duration)) + machineList := &v1alpha5.MachineList{} + Consistently(func(g Gomega) { + g.Expect(env.Client.List(env, machineList, client.HasLabels{test.DiscoveryLabel})).To(Succeed()) + g.Expect(len(machineList.Items)).To(BeNumerically(comparator, count), + fmt.Sprintf("expected %d machines, had %d (%v) for %s", count, len(machineList.Items), MachineNames(lo.ToSlicePtr(machineList.Items)), duration)) + }, duration).Should(Succeed()) + return lo.ToSlicePtr(machineList.Items) +} + +func (env *Environment) EventuallyExpectCordonedNodeCountLegacy(comparator string, count int) []*v1.Node { + GinkgoHelper() + By(fmt.Sprintf("waiting for cordoned nodes to be %s to %d", comparator, count)) + nodeList := &v1.NodeList{} + Eventually(func(g Gomega) { + g.Expect(env.Client.List(env, nodeList, client.MatchingFields{"spec.unschedulable": "true"})).To(Succeed()) + g.Expect(len(nodeList.Items)).To(BeNumerically(comparator, count), + fmt.Sprintf("expected %d cordoned nodes, had %d (%v)", count, len(nodeList.Items), NodeNames(lo.ToSlicePtr(nodeList.Items)))) + }).Should(Succeed()) + return lo.ToSlicePtr(nodeList.Items) +} + +func (env *Environment) EventuallyExpectNodesUncordonedLegacyWithTimeout(timeout time.Duration, nodes ...*v1.Node) { + GinkgoHelper() + By(fmt.Sprintf("waiting for %d nodes to be uncordoned", len(nodes))) + nodeList := &v1.NodeList{} + Eventually(func(g Gomega) { + g.Expect(env.Client.List(env, nodeList, client.MatchingFields{"spec.unschedulable": "true"})).To(Succeed()) + cordonedNodeNames := lo.Map(nodeList.Items, func(n v1.Node, _ int) string { return n.Name }) + g.Expect(cordonedNodeNames).ToNot(ContainElements(lo.Map(nodes, func(n *v1.Node, _ int) interface{} { return n.Name })...)) + }).WithTimeout(timeout).Should(Succeed()) +} + +func (env *Environment) EventuallyExpectCordonedNodeCount(comparator string, count int) []*v1.Node { + GinkgoHelper() + By(fmt.Sprintf("waiting for cordoned nodes to be %s to %d", comparator, count)) + nodeList := &v1.NodeList{} + Eventually(func(g Gomega) { + g.Expect(env.Client.List(env, nodeList, client.MatchingFields{"spec.taints[*].karpenter.sh/disruption": "disrupting"})).To(Succeed()) + g.Expect(len(nodeList.Items)).To(BeNumerically(comparator, count), + fmt.Sprintf("expected %d cordoned nodes, had %d (%v)", count, len(nodeList.Items), NodeNames(lo.ToSlicePtr(nodeList.Items)))) + }).Should(Succeed()) + return lo.ToSlicePtr(nodeList.Items) +} + +func (env *Environment) EventuallyExpectNodesUncordonedWithTimeout(timeout time.Duration, nodes ...*v1.Node) { + GinkgoHelper() + By(fmt.Sprintf("waiting for %d nodes to be uncordoned", len(nodes))) + nodeList := &v1.NodeList{} + Eventually(func(g Gomega) { + g.Expect(env.Client.List(env, nodeList, client.MatchingFields{"spec.taints[*].karpenter.sh/disruption": "disrupting"})).To(Succeed()) + cordonedNodeNames := lo.Map(nodeList.Items, func(n v1.Node, _ int) string { return n.Name }) + g.Expect(cordonedNodeNames).ToNot(ContainElements(lo.Map(nodes, func(n *v1.Node, _ int) interface{} { return n.Name })...)) + }).WithTimeout(timeout).Should(Succeed()) +} + +func (env *Environment) EventuallyExpectNodeCount(comparator string, count int) []*v1.Node { + GinkgoHelper() + By(fmt.Sprintf("waiting for nodes to be %s to %d", comparator, count)) + nodeList := &v1.NodeList{} + Eventually(func(g Gomega) { + g.Expect(env.Client.List(env, nodeList, client.HasLabels{test.DiscoveryLabel})).To(Succeed()) + g.Expect(len(nodeList.Items)).To(BeNumerically(comparator, count), + fmt.Sprintf("expected %d nodes, had %d (%v)", count, len(nodeList.Items), NodeNames(lo.ToSlicePtr(nodeList.Items)))) + }).Should(Succeed()) + return lo.ToSlicePtr(nodeList.Items) +} + +func (env *Environment) EventuallyExpectMachineCount(comparator string, count int) []*v1alpha5.Machine { + GinkgoHelper() + By(fmt.Sprintf("waiting for machines to be %s to %d", comparator, count)) + machineList := &v1alpha5.MachineList{} + Eventually(func(g Gomega) { + g.Expect(env.Client.List(env, machineList, client.HasLabels{test.DiscoveryLabel})).To(Succeed()) + g.Expect(len(machineList.Items)).To(BeNumerically(comparator, count), + fmt.Sprintf("expected %d machines, had %d (%v)", count, len(machineList.Items), MachineNames(lo.ToSlicePtr(machineList.Items)))) + }).Should(Succeed()) + return lo.ToSlicePtr(machineList.Items) +} + +func (env *Environment) EventuallyExpectNodeCountWithSelector(comparator string, count int, selector labels.Selector) []*v1.Node { + GinkgoHelper() + By(fmt.Sprintf("waiting for nodes with selector %v to be %s to %d", selector, comparator, count)) + nodeList := &v1.NodeList{} + Eventually(func(g Gomega) { + g.Expect(env.Client.List(env, nodeList, client.HasLabels{test.DiscoveryLabel}, client.MatchingLabelsSelector{Selector: selector})).To(Succeed()) + g.Expect(len(nodeList.Items)).To(BeNumerically(comparator, count), + fmt.Sprintf("expected %d nodes, had %d (%v)", count, len(nodeList.Items), NodeNames(lo.ToSlicePtr(nodeList.Items)))) + }).Should(Succeed()) + return lo.ToSlicePtr(nodeList.Items) +} + +func (env *Environment) EventuallyExpectCreatedNodeCount(comparator string, count int) []*v1.Node { + GinkgoHelper() + By(fmt.Sprintf("waiting for created nodes to be %s to %d", comparator, count)) + var createdNodes []*v1.Node + Eventually(func(g Gomega) { + createdNodes = env.Monitor.CreatedNodes() + g.Expect(len(createdNodes)).To(BeNumerically(comparator, count), + fmt.Sprintf("expected %d created nodes, had %d (%v)", count, len(createdNodes), NodeNames(createdNodes))) + }).Should(Succeed()) + return createdNodes +} + +func (env *Environment) EventuallyExpectDeletedNodeCount(comparator string, count int) []*v1.Node { + GinkgoHelper() + By(fmt.Sprintf("waiting for deleted nodes to be %s to %d", comparator, count)) + var deletedNodes []*v1.Node + Eventually(func(g Gomega) { + deletedNodes = env.Monitor.DeletedNodes() + g.Expect(len(deletedNodes)).To(BeNumerically(comparator, count), + fmt.Sprintf("expected %d deleted nodes, had %d (%v)", count, len(deletedNodes), NodeNames(deletedNodes))) + }).Should(Succeed()) + return deletedNodes +} + +func (env *Environment) EventuallyExpectDeletedNodeCountWithSelector(comparator string, count int, selector labels.Selector) []*v1.Node { + GinkgoHelper() + By(fmt.Sprintf("waiting for deleted nodes with selector %v to be %s to %d", selector, comparator, count)) + var deletedNodes []*v1.Node + Eventually(func(g Gomega) { + deletedNodes = env.Monitor.DeletedNodes() + deletedNodes = lo.Filter(deletedNodes, func(n *v1.Node, _ int) bool { + return selector.Matches(labels.Set(n.Labels)) + }) + g.Expect(len(deletedNodes)).To(BeNumerically(comparator, count), + fmt.Sprintf("expected %d deleted nodes, had %d (%v)", count, len(deletedNodes), NodeNames(deletedNodes))) + }).Should(Succeed()) + return deletedNodes +} + +func (env *Environment) EventuallyExpectInitializedNodeCount(comparator string, count int) []*v1.Node { + GinkgoHelper() + By(fmt.Sprintf("waiting for initialized nodes to be %s to %d", comparator, count)) + var nodes []*v1.Node + Eventually(func(g Gomega) { + nodes = env.Monitor.CreatedNodes() + nodes = lo.Filter(nodes, func(n *v1.Node, _ int) bool { + return n.Labels[v1alpha5.LabelNodeInitialized] == "true" + }) + g.Expect(len(nodes)).To(BeNumerically(comparator, count)) + }).Should(Succeed()) + return nodes +} + +func (env *Environment) EventuallyExpectCreatedMachineCount(comparator string, count int) []*v1alpha5.Machine { + GinkgoHelper() + By(fmt.Sprintf("waiting for created machines to be %s to %d", comparator, count)) + machineList := &v1alpha5.MachineList{} + Eventually(func(g Gomega) { + g.Expect(env.Client.List(env.Context, machineList)).To(Succeed()) + g.Expect(len(machineList.Items)).To(BeNumerically(comparator, count)) + }).Should(Succeed()) + return lo.Map(machineList.Items, func(m v1alpha5.Machine, _ int) *v1alpha5.Machine { + return &m + }) +} + +func (env *Environment) EventuallyExpectMachinesReady(machines ...*v1alpha5.Machine) { + Eventually(func(g Gomega) { + for _, machine := range machines { + temp := &v1alpha5.Machine{} + g.Expect(env.Client.Get(env.Context, client.ObjectKeyFromObject(machine), temp)).Should(Succeed()) + g.Expect(temp.StatusConditions().IsHappy()).To(BeTrue()) + } + }).Should(Succeed()) +} + +func (env *Environment) EventuallyExpectCreatedNodeClaimCount(comparator string, count int) []*corev1beta1.NodeClaim { + GinkgoHelper() + By(fmt.Sprintf("waiting for created nodeclaims to be %s to %d", comparator, count)) + nodeClaimList := &corev1beta1.NodeClaimList{} + Eventually(func(g Gomega) { + g.Expect(env.Client.List(env.Context, nodeClaimList)).To(Succeed()) + g.Expect(len(nodeClaimList.Items)).To(BeNumerically(comparator, count)) + }).Should(Succeed()) + return lo.Map(nodeClaimList.Items, func(nc corev1beta1.NodeClaim, _ int) *corev1beta1.NodeClaim { + return &nc + }) +} + +func (env *Environment) EventuallyExpectNodeClaimsReady(nodeClaims ...*corev1beta1.NodeClaim) { + Eventually(func(g Gomega) { + for _, nc := range nodeClaims { + temp := &corev1beta1.NodeClaim{} + g.Expect(env.Client.Get(env.Context, client.ObjectKeyFromObject(nc), temp)).Should(Succeed()) + g.Expect(temp.StatusConditions().IsHappy()).To(BeTrue()) + } + }).Should(Succeed()) +} + +func (env *Environment) GetNode(nodeName string) v1.Node { + GinkgoHelper() + var node v1.Node + Expect(env.Client.Get(env.Context, types.NamespacedName{Name: nodeName}, &node)).To(Succeed()) + return node +} + +func (env *Environment) ExpectNoCrashes() { + GinkgoHelper() + _, crashed := lo.Find(lo.Values(env.Monitor.RestartCount()), func(restartCount int) bool { + return restartCount > 0 + }) + Expect(crashed).To(BeFalse(), "expected karpenter containers to not crash") +} + +var ( + lastLogged = metav1.Now() +) + +func (env *Environment) printControllerLogs(options *v1.PodLogOptions) { + fmt.Println("------- START CONTROLLER LOGS -------") + defer fmt.Println("------- END CONTROLLER LOGS -------") + + if options.SinceTime == nil { + options.SinceTime = lastLogged.DeepCopy() + lastLogged = metav1.Now() + } + pods := env.ExpectKarpenterPods() + for _, pod := range pods { + temp := options.DeepCopy() // local version of the log options + + fmt.Printf("------- pod/%s -------\n", pod.Name) + if pod.Status.ContainerStatuses[0].RestartCount > 0 { + fmt.Printf("[PREVIOUS CONTAINER LOGS]\n") + temp.Previous = true + } + stream, err := env.KubeClient.CoreV1().Pods("karpenter").GetLogs(pod.Name, temp).Stream(env.Context) + if err != nil { + logging.FromContext(env.Context).Errorf("fetching controller logs: %s", err) + return + } + log := &bytes.Buffer{} + _, err = io.Copy(log, stream) + Expect(err).ToNot(HaveOccurred()) + logging.FromContext(env.Context).Info(log) + } +} + +func (env *Environment) EventuallyExpectMinUtilization(resource v1.ResourceName, comparator string, value float64) { + GinkgoHelper() + Eventually(func(g Gomega) { + g.Expect(env.Monitor.MinUtilization(resource)).To(BeNumerically(comparator, value)) + }).Should(Succeed()) +} + +func (env *Environment) EventuallyExpectAvgUtilization(resource v1.ResourceName, comparator string, value float64) { + GinkgoHelper() + Eventually(func(g Gomega) { + g.Expect(env.Monitor.AvgUtilization(resource)).To(BeNumerically(comparator, value)) + }, 10*time.Minute).Should(Succeed()) +} + +func (env *Environment) ExpectDaemonSetEnvironmentVariableUpdated(obj client.ObjectKey, name, value string, containers ...string) { + GinkgoHelper() + ds := &appsv1.DaemonSet{} + Expect(env.Client.Get(env.Context, obj, ds)).To(Succeed()) + if len(containers) == 0 { + Expect(len(ds.Spec.Template.Spec.Containers)).To(BeNumerically("==", 1)) + containers = append(containers, ds.Spec.Template.Spec.Containers[0].Name) + } + patch := client.MergeFrom(ds.DeepCopy()) + containerNames := sets.New(containers...) + for ci := range ds.Spec.Template.Spec.Containers { + c := &ds.Spec.Template.Spec.Containers[ci] + if !containerNames.Has(c.Name) { + continue + } + // If the env var already exists, update its value. Otherwise, create a new var. + if _, i, ok := lo.FindIndexOf(c.Env, func(e v1.EnvVar) bool { + return e.Name == name + }); ok { + c.Env[i].Value = value + } else { + c.Env = append(c.Env, v1.EnvVar{Name: name, Value: value}) + } + } + Expect(env.Client.Patch(env.Context, ds, patch)).To(Succeed()) +} + +func (env *Environment) ExpectCABundle() string { + // Discover CA Bundle from the REST client. We could alternatively + // have used the simpler client-go InClusterConfig() method. + // However, that only works when Karpenter is running as a Pod + // within the same cluster it's managing. + GinkgoHelper() + transportConfig, err := env.Config.TransportConfig() + Expect(err).ToNot(HaveOccurred()) + _, err = transport.TLSConfigFor(transportConfig) // fills in CAData! + Expect(err).ToNot(HaveOccurred()) + logging.FromContext(env.Context).Debugf("Discovered caBundle, length %d", len(transportConfig.TLS.CAData)) + return base64.StdEncoding.EncodeToString(transportConfig.TLS.CAData) +} + +func (env *Environment) GetDaemonSetCountLegacy(prov *v1alpha5.Provisioner) int { + GinkgoHelper() + + // Performs the same logic as the scheduler to get the number of daemonset + // pods that we estimate we will need to schedule as overhead to each node + daemonSetList := &appsv1.DaemonSetList{} + Expect(env.Client.List(env.Context, daemonSetList)).To(Succeed()) + + return lo.CountBy(daemonSetList.Items, func(d appsv1.DaemonSet) bool { + p := &v1.Pod{Spec: d.Spec.Template.Spec} + nodeTemplate := pscheduling.NewNodeClaimTemplate(nodepoolutil.New(prov)) + if err := scheduling.Taints(nodeTemplate.Spec.Taints).Tolerates(p); err != nil { + return false + } + if err := nodeTemplate.Requirements.Compatible(scheduling.NewPodRequirements(p), scheduling.AllowUndefinedWellKnownLabelsV1Alpha5); err != nil { + return false + } + return true + }) +} + +func (env *Environment) GetDaemonSetCount(np *corev1beta1.NodePool) int { + GinkgoHelper() + + // Performs the same logic as the scheduler to get the number of daemonset + // pods that we estimate we will need to schedule as overhead to each node + daemonSetList := &appsv1.DaemonSetList{} + Expect(env.Client.List(env.Context, daemonSetList)).To(Succeed()) + + return lo.CountBy(daemonSetList.Items, func(d appsv1.DaemonSet) bool { + p := &v1.Pod{Spec: d.Spec.Template.Spec} + nodeTemplate := pscheduling.NewNodeClaimTemplate(np) + if err := scheduling.Taints(nodeTemplate.Spec.Taints).Tolerates(p); err != nil { + return false + } + if err := nodeTemplate.Requirements.Compatible(scheduling.NewPodRequirements(p), scheduling.AllowUndefinedWellKnownLabelsV1Beta1); err != nil { + return false + } + return true + }) +} diff --git a/test/pkg/environment/common/monitor.go b/test/pkg/environment/common/monitor.go new file mode 100644 index 000000000..c5cd7e995 --- /dev/null +++ b/test/pkg/environment/common/monitor.go @@ -0,0 +1,260 @@ +/* +Portions Copyright (c) Microsoft Corporation. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package common + +import ( + "context" + "fmt" + "math" + "sync" + + v1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/labels" + "k8s.io/apimachinery/pkg/util/sets" + "knative.dev/pkg/logging" + "sigs.k8s.io/controller-runtime/pkg/client" + + "github.com/samber/lo" + + "github.com/aws/karpenter-core/pkg/apis/v1alpha5" + "github.com/aws/karpenter-core/pkg/apis/v1beta1" + "github.com/aws/karpenter-core/pkg/utils/resources" +) + +// Monitor is used to monitor the cluster state during a running test +type Monitor struct { + ctx context.Context + kubeClient client.Client + + mu sync.RWMutex + + nodesAtReset map[string]*v1.Node +} + +type state struct { + pods v1.PodList + nodes map[string]*v1.Node // node name -> node + nodePods map[string][]*v1.Pod // node name -> pods bound to the node + nodeRequests map[string]v1.ResourceList // node name -> sum of pod resource requests +} + +func NewMonitor(ctx context.Context, kubeClient client.Client) *Monitor { + m := &Monitor{ + ctx: ctx, + kubeClient: kubeClient, + nodesAtReset: map[string]*v1.Node{}, + } + m.Reset() + return m +} + +// Reset resets the cluster monitor prior to running a test. +func (m *Monitor) Reset() { + m.mu.Lock() + defer m.mu.Unlock() + + st := m.poll() + m.nodesAtReset = deepCopyMap(st.nodes) +} + +// RestartCount returns the containers and number of restarts for that container for all containers in the pods in the +// given namespace +func (m *Monitor) RestartCount() map[string]int { + st := m.poll() + + m.mu.RLock() + defer m.mu.RUnlock() + restarts := map[string]int{} + for _, pod := range st.pods.Items { + if pod.Namespace != "karpenter" { + continue + } + for _, cs := range pod.Status.ContainerStatuses { + name := fmt.Sprintf("%s/%s", pod.Name, cs.Name) + restarts[name] = int(cs.RestartCount) + } + } + return restarts +} + +// NodeCount returns the current number of nodes +func (m *Monitor) NodeCount() int { + return len(m.poll().nodes) +} + +// NodeCountAtReset returns the number of nodes that were running when the monitor was last reset, typically at the +// beginning of a test +func (m *Monitor) NodeCountAtReset() int { + return len(m.NodesAtReset()) +} + +// CreatedNodeCount returns the number of nodes created since the last reset +func (m *Monitor) CreatedNodeCount() int { + return m.NodeCount() - m.NodeCountAtReset() +} + +// NodesAtReset returns a slice of nodes that the monitor saw at the last reset +func (m *Monitor) NodesAtReset() []*v1.Node { + m.mu.RLock() + defer m.mu.RUnlock() + return deepCopySlice(lo.Values(m.nodesAtReset)) +} + +// Nodes returns all the nodes on the cluster +func (m *Monitor) Nodes() []*v1.Node { + st := m.poll() + return lo.Values(st.nodes) +} + +// CreatedNodes returns the nodes that have been created since the last reset (essentially Nodes - NodesAtReset) +func (m *Monitor) CreatedNodes() []*v1.Node { + resetNodeNames := sets.NewString(lo.Map(m.NodesAtReset(), func(n *v1.Node, _ int) string { return n.Name })...) + return lo.Filter(m.Nodes(), func(n *v1.Node, _ int) bool { return !resetNodeNames.Has(n.Name) }) +} + +// DeletedNodes returns the nodes that have been deleted since the last reset (essentially NodesAtReset - Nodes) +func (m *Monitor) DeletedNodes() []*v1.Node { + currentNodeNames := sets.NewString(lo.Map(m.Nodes(), func(n *v1.Node, _ int) string { return n.Name })...) + return lo.Filter(m.NodesAtReset(), func(n *v1.Node, _ int) bool { return !currentNodeNames.Has(n.Name) }) +} + +// PendingPods returns the number of pending pods matching the given selector +func (m *Monitor) PendingPods(selector labels.Selector) []*v1.Pod { + var pods []*v1.Pod + for _, pod := range m.poll().pods.Items { + pod := pod + if pod.Status.Phase != v1.PodPending { + continue + } + if selector.Matches(labels.Set(pod.Labels)) { + pods = append(pods, &pod) + } + } + return pods +} + +func (m *Monitor) PendingPodsCount(selector labels.Selector) int { + return len(m.PendingPods(selector)) +} + +// RunningPods returns the number of running pods matching the given selector +func (m *Monitor) RunningPods(selector labels.Selector) []*v1.Pod { + var pods []*v1.Pod + for _, pod := range m.poll().pods.Items { + pod := pod + if pod.Status.Phase != v1.PodRunning { + continue + } + if selector.Matches(labels.Set(pod.Labels)) { + pods = append(pods, &pod) + } + } + return pods +} + +func (m *Monitor) RunningPodsCount(selector labels.Selector) int { + return len(m.RunningPods(selector)) +} + +func (m *Monitor) poll() state { + var nodes v1.NodeList + if err := m.kubeClient.List(m.ctx, &nodes); err != nil { + logging.FromContext(m.ctx).Errorf("listing nodes, %s", err) + } + var pods v1.PodList + if err := m.kubeClient.List(m.ctx, &pods); err != nil { + logging.FromContext(m.ctx).Errorf("listing pods, %s", err) + } + st := state{ + nodes: map[string]*v1.Node{}, + pods: pods, + nodePods: map[string][]*v1.Pod{}, + nodeRequests: map[string]v1.ResourceList{}, + } + for i := range nodes.Items { + st.nodes[nodes.Items[i].Name] = &nodes.Items[i] + } + // collect pods per node + for i := range pods.Items { + pod := &pods.Items[i] + if pod.Spec.NodeName == "" { + continue + } + st.nodePods[pod.Spec.NodeName] = append(st.nodePods[pod.Spec.NodeName], pod) + } + + for _, n := range nodes.Items { + st.nodeRequests[n.Name] = resources.RequestsForPods(st.nodePods[n.Name]...) + } + return st +} + +func (m *Monitor) AvgUtilization(resource v1.ResourceName) float64 { + utilization := m.nodeUtilization(resource) + sum := 0.0 + for _, v := range utilization { + sum += v + } + return sum / float64(len(utilization)) +} + +func (m *Monitor) MinUtilization(resource v1.ResourceName) float64 { + min := math.MaxFloat64 + for _, v := range m.nodeUtilization(resource) { + min = math.Min(v, min) + } + return min +} + +func (m *Monitor) nodeUtilization(resource v1.ResourceName) []float64 { + st := m.poll() + var utilization []float64 + for nodeName, requests := range st.nodeRequests { + allocatable := st.nodes[nodeName].Status.Allocatable[resource] + // skip any nodes we didn't launch + if st.nodes[nodeName].Labels[v1alpha5.ProvisionerNameLabelKey] == "" && + st.nodes[nodeName].Labels[v1beta1.NodePoolLabelKey] == "" { + continue + } + if allocatable.IsZero() { + continue + } + requested := requests[resource] + utilization = append(utilization, requested.AsApproximateFloat64()/allocatable.AsApproximateFloat64()) + } + return utilization +} + +type copyable[T any] interface { + DeepCopy() T +} + +func deepCopyMap[K comparable, V copyable[V]](m map[K]V) map[K]V { + ret := map[K]V{} + for k, v := range m { + ret[k] = v.DeepCopy() + } + return ret +} + +func deepCopySlice[T copyable[T]](s []T) []T { + var ret []T + for _, elem := range s { + ret = append(ret, elem.DeepCopy()) + } + return ret +} diff --git a/test/pkg/environment/common/setup.go b/test/pkg/environment/common/setup.go new file mode 100644 index 000000000..37f0a3662 --- /dev/null +++ b/test/pkg/environment/common/setup.go @@ -0,0 +1,147 @@ +/* +Portions Copyright (c) Microsoft Corporation. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package common + +import ( + "fmt" + "sync" + "time" + + "github.com/Azure/karpenter/pkg/apis/v1alpha2" + . "github.com/onsi/ginkgo/v2" //nolint:revive,stylecheck + . "github.com/onsi/gomega" //nolint:revive,stylecheck + "github.com/samber/lo" + appsv1 "k8s.io/api/apps/v1" + v1 "k8s.io/api/core/v1" + policyv1 "k8s.io/api/policy/v1" + schedulingv1 "k8s.io/api/scheduling/v1" + storagev1 "k8s.io/api/storage/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/util/workqueue" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/apiutil" + + "github.com/aws/karpenter-core/pkg/apis" + "github.com/aws/karpenter-core/pkg/apis/v1alpha5" + corev1beta1 "github.com/aws/karpenter-core/pkg/apis/v1beta1" + "github.com/aws/karpenter-core/pkg/operator/injection" + "github.com/aws/karpenter-core/pkg/test" + "github.com/aws/karpenter-core/pkg/utils/pod" +) + +var ( + CleanableObjects = []client.Object{ + &v1.Pod{}, + &appsv1.Deployment{}, + &appsv1.DaemonSet{}, + &policyv1.PodDisruptionBudget{}, + &v1.PersistentVolumeClaim{}, + &v1.PersistentVolume{}, + &storagev1.StorageClass{}, + &v1alpha5.Provisioner{}, + &corev1beta1.NodePool{}, + &v1.LimitRange{}, + &schedulingv1.PriorityClass{}, + &v1.Node{}, + &v1alpha5.Machine{}, + &corev1beta1.NodeClaim{}, + } +) + +// nolint:gocyclo +func (env *Environment) BeforeEach() { + // TODO (charliedmcb): uncomment debugging lib, but removing references for now to avoid the need to review the entire package + // debug.BeforeEach(env.Context, env.Config, env.Client) + env.Context = injection.WithSettingsOrDie(env.Context, env.KubeClient, apis.Settings...) + + // Expect this cluster to be clean for test runs to execute successfully + env.ExpectCleanCluster() + + env.Monitor.Reset() + env.StartingNodeCount = env.Monitor.NodeCountAtReset() +} + +func (env *Environment) ExpectCleanCluster() { + var nodes v1.NodeList + Expect(env.Client.List(env.Context, &nodes)).To(Succeed()) + for _, node := range nodes.Items { + if len(node.Spec.Taints) == 0 && !node.Spec.Unschedulable { + Fail(fmt.Sprintf("expected system pool node %s to be tainted", node.Name)) + } + } + var pods v1.PodList + Expect(env.Client.List(env.Context, &pods)).To(Succeed()) + for i := range pods.Items { + Expect(pods.Items[i].Namespace).ToNot(Equal("default"), + fmt.Sprintf("expected no pods in the `default` namespace, found %s/%s", pods.Items[i].Namespace, pods.Items[i].Name)) + } + Eventually(func(g Gomega) { + var pods v1.PodList + g.Expect(env.Client.List(env.Context, &pods)).To(Succeed()) + for i := range pods.Items { + g.Expect(pod.IsProvisionable(&pods.Items[i])).To(BeFalse(), + fmt.Sprintf("expected to have no provisionable pods, found %s/%s", pods.Items[i].Namespace, pods.Items[i].Name)) + } + }).WithPolling(10 * time.Second).WithTimeout(5 * time.Minute).Should(Succeed()) + for _, obj := range []client.Object{&v1alpha5.Provisioner{}, &corev1beta1.NodePool{}, &v1alpha2.AKSNodeClass{}} { + metaList := &metav1.PartialObjectMetadataList{} + gvk := lo.Must(apiutil.GVKForObject(obj, env.Client.Scheme())) + metaList.SetGroupVersionKind(gvk) + Expect(env.Client.List(env.Context, metaList, client.Limit(1))).To(Succeed()) + Expect(metaList.Items).To(HaveLen(0), fmt.Sprintf("expected no %s to exist", gvk.Kind)) + } +} + +func (env *Environment) Cleanup() { + env.CleanupObjects(CleanableObjects...) + env.eventuallyExpectScaleDown() + env.ExpectNoCrashes() +} + +func (env *Environment) AfterEach() { + // TODO (charliedmcb): uncomment debugging lib, but removing references for now to avoid the need to review the entire package + // debug.AfterEach(env.Context) + env.printControllerLogs(&v1.PodLogOptions{Container: "controller"}) +} + +func (env *Environment) CleanupObjects(cleanableObjects ...client.Object) { + time.Sleep(time.Second) // wait one second to let the caches get up-to-date for deletion + wg := sync.WaitGroup{} + for _, obj := range cleanableObjects { + wg.Add(1) + go func(obj client.Object) { + defer wg.Done() + defer GinkgoRecover() + Eventually(func(g Gomega) { + // This only gets the metadata for the objects since we don't need all the details of the objects + metaList := &metav1.PartialObjectMetadataList{} + metaList.SetGroupVersionKind(lo.Must(apiutil.GVKForObject(obj, env.Client.Scheme()))) + g.Expect(env.Client.List(env, metaList, client.HasLabels([]string{test.DiscoveryLabel}))).To(Succeed()) + // Limit the concurrency of these calls to 50 workers per object so that we try to limit how aggressively we + // are deleting so that we avoid getting client-side throttled + workqueue.ParallelizeUntil(env, 50, len(metaList.Items), func(i int) { + defer GinkgoRecover() + g.Expect(client.IgnoreNotFound(env.Client.Delete(env, &metaList.Items[i], client.PropagationPolicy(metav1.DeletePropagationForeground)))).To(Succeed()) + }) + // If the deletes eventually succeed, we should have no elements here at the end of the test + g.Expect(env.Client.List(env, metaList, client.HasLabels([]string{test.DiscoveryLabel}), client.Limit(1))).To(Succeed()) + g.Expect(metaList.Items).To(HaveLen(0)) + }).Should(Succeed()) + }(obj) + } + wg.Wait() +} diff --git a/test/suites/drift/suite_test.go b/test/suites/drift/suite_test.go new file mode 100644 index 000000000..de8f6b127 --- /dev/null +++ b/test/suites/drift/suite_test.go @@ -0,0 +1,155 @@ +/* +Portions Copyright (c) Microsoft Corporation. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package drift + +import ( + "fmt" + "testing" + "time" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + v1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/controller-runtime/pkg/client" + + "github.com/Azure/karpenter/pkg/apis/v1alpha2" + "github.com/Azure/karpenter/test/pkg/environment/azure" + + corev1beta1 "github.com/aws/karpenter-core/pkg/apis/v1beta1" + + "github.com/aws/karpenter-core/pkg/test" +) + +var env *azure.Environment +var nodeClass *v1alpha2.AKSNodeClass +var nodePool *corev1beta1.NodePool + +func TestDrift(t *testing.T) { + RegisterFailHandler(Fail) + BeforeSuite(func() { + env = azure.NewEnvironment(t) + }) + RunSpecs(t, "Drift") +} + +var _ = BeforeEach(func() { + env.BeforeEach() + nodeClass = env.DefaultAKSNodeClass() + nodePool = env.DefaultNodePool(nodeClass) +}) +var _ = AfterEach(func() { env.Cleanup() }) +var _ = AfterEach(func() { env.AfterEach() }) + +var _ = Describe("Drift", func() { + + var pod *v1.Pod + + BeforeEach(func() { + env.ExpectSettingsOverridden(v1.EnvVar{Name: "FEATURE_GATES", Value: "Drift=true"}) + + nodePool.Spec.Template.Spec.Requirements = []v1.NodeSelectorRequirement{{ + Key: v1.LabelInstanceTypeStable, + Operator: v1.NodeSelectorOpIn, + Values: []string{"Standard_DS2_v2"}, + }} + + // Add a do-not-disrupt pod so that we can check node metadata before we disrupt + pod = test.Pod(test.PodOptions{ + ObjectMeta: metav1.ObjectMeta{ + Annotations: map[string]string{ + corev1beta1.DoNotDisruptAnnotationKey: "true", + }, + }, + ResourceRequirements: v1.ResourceRequirements{Requests: v1.ResourceList{v1.ResourceCPU: resource.MustParse("0.5")}}, + Image: "mcr.microsoft.com/oss/kubernetes/pause:3.6", + }) + }) + It("should deprovision nodes that have drifted due to labels", func() { + + By(fmt.Sprintf("creating pod %s, nodepool %s, and nodeclass %s", pod.Name, nodePool.Name, nodeClass.Name)) + env.ExpectCreated(pod, nodeClass, nodePool) + + By(fmt.Sprintf("expect pod %s to be healthy", pod.Name)) + env.EventuallyExpectHealthy(pod) + + By("expect created node count to be 1") + env.ExpectCreatedNodeCount("==", 1) + + nodeClaim := env.EventuallyExpectCreatedNodeClaimCount("==", 1)[0] + node := env.EventuallyExpectNodeCount("==", 1)[0] + + By(fmt.Sprintf("waiting for nodepool %s update", nodePool.Name)) + nodePool.Spec.Template.Labels["triggerdrift"] = "value" + env.ExpectCreatedOrUpdated(nodePool) + + By(fmt.Sprintf("waiting for nodeclaim %s to be marked as drifted", nodeClaim.Name)) + Eventually(func(g Gomega) { + g.Expect(env.Client.Get(env, client.ObjectKeyFromObject(nodeClaim), nodeClaim)).To(Succeed()) + g.Expect(nodeClaim.StatusConditions().GetCondition(corev1beta1.Drifted)).ToNot(BeNil()) + g.Expect(nodeClaim.StatusConditions().GetCondition(corev1beta1.Drifted).IsTrue()).To(BeTrue()) + }).Should(Succeed()) + + By(fmt.Sprintf("waiting for pod %s to to update", pod.Name)) + delete(pod.Annotations, corev1beta1.DoNotDisruptAnnotationKey) + env.ExpectUpdated(pod) + + By(fmt.Sprintf("expect pod %s, nodeclaim %s, and node %s to eventually not exist", pod.Name, nodeClaim.Name, node.Name)) + SetDefaultEventuallyTimeout(10 * time.Minute) + env.EventuallyExpectNotFound(pod, nodeClaim, node) + SetDefaultEventuallyTimeout(5 * time.Minute) + }) + It("should upgrade nodes using drift based on node image version change", func() { + startingImageVersion := "202309.29.0" + upgradedImageVersion := "202310.01.0" + + nodeClass.Spec.ImageVersion = &startingImageVersion + + By(fmt.Sprintf("creating pod %s, nodepool %s, and nodeclass %s", pod.Name, nodePool.Name, nodeClass.Name)) + env.ExpectCreated(pod, nodeClass, nodePool) + + By(fmt.Sprintf("expect pod %s to be healthy", pod.Name)) + env.EventuallyExpectHealthy(pod) + + By("expect created node count to be 1") + env.ExpectCreatedNodeCount("==", 1) + + nodeClaim := env.EventuallyExpectCreatedNodeClaimCount("==", 1)[0] + node := env.EventuallyExpectNodeCount("==", 1)[0] + + By(fmt.Sprintf("waiting for nodeClass %s update", nodeClass.Name)) + nodeClass.Spec.ImageVersion = &upgradedImageVersion + env.ExpectCreatedOrUpdated(nodeClass) + + By(fmt.Sprintf("waiting for nodeclaim %s to be marked as drifted", nodeClaim.Name)) + Eventually(func(g Gomega) { + g.Expect(env.Client.Get(env, client.ObjectKeyFromObject(nodeClaim), nodeClaim)).To(Succeed()) + g.Expect(nodeClaim.StatusConditions().GetCondition(corev1beta1.Drifted)).ToNot(BeNil()) + g.Expect(nodeClaim.StatusConditions().GetCondition(corev1beta1.Drifted).IsTrue()).To(BeTrue()) + }).Should(Succeed()) + + By(fmt.Sprintf("waiting for pod %s to to update", pod.Name)) + delete(pod.Annotations, corev1beta1.DoNotDisruptAnnotationKey) + env.ExpectUpdated(pod) + + By(fmt.Sprintf("expect pod %s, nodeclaim %s, and node %s to eventually not exist", pod.Name, nodeClaim.Name, node.Name)) + SetDefaultEventuallyTimeout(10 * time.Minute) + env.EventuallyExpectNotFound(pod, nodeClaim, node) + SetDefaultEventuallyTimeout(5 * time.Minute) + }) +}) diff --git a/test/suites/gpu/suite_test.go b/test/suites/gpu/suite_test.go new file mode 100644 index 000000000..eed89fbff --- /dev/null +++ b/test/suites/gpu/suite_test.go @@ -0,0 +1,155 @@ +/* +Portions Copyright (c) Microsoft Corporation. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package gpu_test + +import ( + "testing" + "time" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "github.com/samber/lo" + appsv1 "k8s.io/api/apps/v1" + v1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/labels" + + corev1beta1 "github.com/aws/karpenter-core/pkg/apis/v1beta1" + "github.com/aws/karpenter-core/pkg/test" + + "github.com/Azure/karpenter/pkg/apis/v1alpha2" + "github.com/Azure/karpenter/test/pkg/environment/azure" +) + +var env *azure.Environment + +func TestGPU(t *testing.T) { + RegisterFailHandler(Fail) + BeforeSuite(func() { + env = azure.NewEnvironment(t) + }) + RunSpecs(t, "GPU") +} + +var _ = BeforeEach(func() { env.BeforeEach() }) +var _ = AfterEach(func() { env.Cleanup() }) +var _ = AfterEach(func() { env.AfterEach() }) + +var _ = Describe("GPU", func() { + It("should provision one GPU node and one GPU Pod", func() { + + nodeClass := env.DefaultAKSNodeClass() + nodePool := env.DefaultNodePool(nodeClass) + + // relax default SKU family selector to allow for GPU nodes + test.ReplaceRequirements(nodePool, v1.NodeSelectorRequirement{ + Key: v1alpha2.LabelSKUFamily, + Operator: v1.NodeSelectorOpExists, + }) + // exclude some of the more expensive GPU SKUs + nodePool.Spec.Limits = corev1beta1.Limits{ + v1.ResourceCPU: resource.MustParse("25"), + v1.ResourceName("nvidia.com/gpu"): resource.MustParse("1"), + } + + minstPodOptions := test.PodOptions{ + ObjectMeta: metav1.ObjectMeta{ + Name: "samples-fake-minst", + Labels: map[string]string{ + "app": "samples-tf-mnist-demo", + }, + }, + Image: "mcr.microsoft.com/oss/kubernetes/pause:3.6", + ResourceRequirements: v1.ResourceRequirements{ + Limits: v1.ResourceList{ + "nvidia.com/gpu": resource.MustParse("1"), + }, + }, + } + deployment := test.Deployment(test.DeploymentOptions{ + Replicas: 1, + PodOptions: minstPodOptions, + }) + + devicePlugin := &appsv1.DaemonSet{ + ObjectMeta: metav1.ObjectMeta{ + Name: "nvidia-device-plugin-daemonset", + Namespace: "kube-system", + }, + Spec: appsv1.DaemonSetSpec{ + Selector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + "name": "nvidia-device-plugin-ds", + }, + }, + Template: v1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{ + "name": "nvidia-device-plugin-ds", + }, + }, + Spec: v1.PodSpec{ + Tolerations: []v1.Toleration{ + { + Key: "nvidia.com/gpu", + Operator: v1.TolerationOpExists, + Effect: v1.TaintEffectNoSchedule, + }, + }, + PriorityClassName: "system-node-critical", + Volumes: []v1.Volume{ + { + Name: "device-plugin", + VolumeSource: v1.VolumeSource{ + HostPath: &v1.HostPathVolumeSource{ + Path: "/var/lib/kubelet/device-plugins", + }, + }, + }, + }, + Containers: []v1.Container{ + { + Name: "nvidia-device-plugin-ctr", + Image: "nvcr.io/nvidia/k8s-device-plugin:v0.14.1", + SecurityContext: &v1.SecurityContext{ + AllowPrivilegeEscalation: lo.ToPtr(false), + Capabilities: &v1.Capabilities{ + Drop: []v1.Capability{ + "ALL", + }, + }, + }, + VolumeMounts: []v1.VolumeMount{ + { + Name: "device-plugin", + MountPath: "/var/lib/kubelet/device-plugins", + }, + }, + }, + }, + }, + }, + }, + } + env.ExpectCreated(nodeClass, nodePool, deployment, devicePlugin) + // This test exercises the full lifecycle of the GPU Node, and validates it can successfully schedule GPU Resources. + env.EventuallyExpectHealthyPodCountWithTimeout(time.Minute*15, labels.SelectorFromSet(deployment.Spec.Selector.MatchLabels), int(*deployment.Spec.Replicas)) + env.ExpectCreatedNodeCount("==", int(*deployment.Spec.Replicas)) + + }) +}) diff --git a/test/suites/integration/daemonset_test.go b/test/suites/integration/daemonset_test.go new file mode 100644 index 000000000..284e9ad4f --- /dev/null +++ b/test/suites/integration/daemonset_test.go @@ -0,0 +1,141 @@ +/* +Portions Copyright (c) Microsoft Corporation. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package integration_test + +import ( + appsv1 "k8s.io/api/apps/v1" + v1 "k8s.io/api/core/v1" + schedulingv1 "k8s.io/api/scheduling/v1" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/labels" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "sigs.k8s.io/controller-runtime/pkg/client" + + corev1beta1 "github.com/aws/karpenter-core/pkg/apis/v1beta1" + "github.com/aws/karpenter-core/pkg/test" +) + +var _ = Describe("DaemonSet", func() { + var limitrange *v1.LimitRange + var priorityclass *schedulingv1.PriorityClass + var daemonset *appsv1.DaemonSet + var dep *appsv1.Deployment + + BeforeEach(func() { + nodePool.Spec.Disruption.ConsolidationPolicy = corev1beta1.ConsolidationPolicyWhenUnderutilized + nodePool.Spec.Disruption.ConsolidateAfter = nil + nodePool.Spec.Template.Labels = map[string]string{"testing/cluster": "test"} + + priorityclass = &schedulingv1.PriorityClass{ + ObjectMeta: metav1.ObjectMeta{ + Name: "high-priority-daemonsets", + }, + Value: int32(10000000), + GlobalDefault: false, + Description: "This priority class should be used for daemonsets.", + } + limitrange = &v1.LimitRange{ + ObjectMeta: metav1.ObjectMeta{ + Name: "limitrange", + Namespace: "default", + }, + } + daemonset = test.DaemonSet(test.DaemonSetOptions{ + PodOptions: test.PodOptions{ + ResourceRequirements: v1.ResourceRequirements{Limits: v1.ResourceList{v1.ResourceMemory: resource.MustParse("1Gi")}}, + PriorityClassName: "high-priority-daemonsets", + }, + }) + numPods := 1 + dep = test.Deployment(test.DeploymentOptions{ + Replicas: int32(numPods), + PodOptions: test.PodOptions{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{"app": "large-app"}, + }, + ResourceRequirements: v1.ResourceRequirements{ + Requests: v1.ResourceList{v1.ResourceMemory: resource.MustParse("4")}, + }, + }, + }) + }) + It("should account for LimitRange Default on daemonSet pods for resources", func() { + limitrange.Spec.Limits = []v1.LimitRangeItem{ + { + Type: v1.LimitTypeContainer, + Default: v1.ResourceList{ + v1.ResourceCPU: resource.MustParse("2"), + v1.ResourceMemory: resource.MustParse("1Gi"), + }, + }, + } + + podSelector := labels.SelectorFromSet(dep.Spec.Selector.MatchLabels) + daemonSetSelector := labels.SelectorFromSet(daemonset.Spec.Selector.MatchLabels) + env.ExpectCreated(nodeClass, nodePool, limitrange, priorityclass, daemonset, dep) + + // Eventually expect a single node to exist and both the deployment pod and the daemonset pod to schedule to it + Eventually(func(g Gomega) { + nodeList := &v1.NodeList{} + g.Expect(env.Client.List(env, nodeList, client.HasLabels{"testing/cluster"})).To(Succeed()) + g.Expect(nodeList.Items).To(HaveLen(1)) + + deploymentPods := env.Monitor.RunningPods(podSelector) + g.Expect(deploymentPods).To(HaveLen(1)) + + daemonSetPods := env.Monitor.RunningPods(daemonSetSelector) + g.Expect(daemonSetPods).To(HaveLen(1)) + + g.Expect(deploymentPods[0].Spec.NodeName).To(Equal(nodeList.Items[0].Name)) + g.Expect(daemonSetPods[0].Spec.NodeName).To(Equal(nodeList.Items[0].Name)) + }).Should(Succeed()) + }) + It("should account for LimitRange DefaultRequest on daemonSet pods for resources", func() { + limitrange.Spec.Limits = []v1.LimitRangeItem{ + { + Type: v1.LimitTypeContainer, + DefaultRequest: v1.ResourceList{ + v1.ResourceCPU: resource.MustParse("2"), + v1.ResourceMemory: resource.MustParse("1Gi"), + }, + }, + } + + podSelector := labels.SelectorFromSet(dep.Spec.Selector.MatchLabels) + daemonSetSelector := labels.SelectorFromSet(daemonset.Spec.Selector.MatchLabels) + env.ExpectCreated(nodeClass, nodePool, limitrange, priorityclass, daemonset, dep) + + // Eventually expect a single node to exist and both the deployment pod and the daemonset pod to schedule to it + Eventually(func(g Gomega) { + nodeList := &v1.NodeList{} + g.Expect(env.Client.List(env, nodeList, client.HasLabels{"testing/cluster"})).To(Succeed()) + g.Expect(nodeList.Items).To(HaveLen(1)) + + deploymentPods := env.Monitor.RunningPods(podSelector) + g.Expect(deploymentPods).To(HaveLen(1)) + + daemonSetPods := env.Monitor.RunningPods(daemonSetSelector) + g.Expect(daemonSetPods).To(HaveLen(1)) + + g.Expect(deploymentPods[0].Spec.NodeName).To(Equal(nodeList.Items[0].Name)) + g.Expect(daemonSetPods[0].Spec.NodeName).To(Equal(nodeList.Items[0].Name)) + }).Should(Succeed()) + }) +}) diff --git a/test/suites/integration/emptiness_test.go b/test/suites/integration/emptiness_test.go new file mode 100644 index 000000000..b9117e826 --- /dev/null +++ b/test/suites/integration/emptiness_test.go @@ -0,0 +1,66 @@ +/* +Portions Copyright (c) Microsoft Corporation. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package integration_test + +import ( + "time" + + "github.com/samber/lo" + "k8s.io/apimachinery/pkg/labels" + "knative.dev/pkg/ptr" + + "sigs.k8s.io/controller-runtime/pkg/client" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + corev1beta1 "github.com/aws/karpenter-core/pkg/apis/v1beta1" + "github.com/aws/karpenter-core/pkg/test" +) + +var _ = Describe("Emptiness", func() { + It("should terminate an empty node", func() { + nodePool.Spec.Disruption.ConsolidationPolicy = corev1beta1.ConsolidationPolicyWhenEmpty + nodePool.Spec.Disruption.ConsolidateAfter = &corev1beta1.NillableDuration{Duration: lo.ToPtr(time.Hour * 300)} + + const numPods = 1 + deployment := test.Deployment(test.DeploymentOptions{Replicas: numPods}) + + By("kicking off provisioning for a deployment") + env.ExpectCreated(nodeClass, nodePool, deployment) + nodeClaim := env.EventuallyExpectCreatedNodeClaimCount("==", 1)[0] + node := env.EventuallyExpectCreatedNodeCount("==", 1)[0] + env.EventuallyExpectHealthyPodCount(labels.SelectorFromSet(deployment.Spec.Selector.MatchLabels), numPods) + + By("making the nodeclaim empty") + persisted := deployment.DeepCopy() + deployment.Spec.Replicas = ptr.Int32(0) + Expect(env.Client.Patch(env, deployment, client.MergeFrom(persisted))).To(Succeed()) + + By("waiting for the nodeclaim emptiness status condition to propagate") + Eventually(func(g Gomega) { + g.Expect(env.Client.Get(env, client.ObjectKeyFromObject(nodeClaim), nodeClaim)).To(Succeed()) + g.Expect(nodeClaim.StatusConditions().GetCondition(corev1beta1.Empty).IsTrue()).To(BeTrue()) + }).Should(Succeed()) + + By("waiting for the nodeclaim to deprovision when past its ConsolidateAfter timeout of 0") + nodePool.Spec.Disruption.ConsolidateAfter = &corev1beta1.NillableDuration{Duration: lo.ToPtr(time.Duration(0))} + env.ExpectUpdated(nodePool) + + env.EventuallyExpectNotFound(nodeClaim, node) + }) +}) diff --git a/test/suites/integration/expiration_test.go b/test/suites/integration/expiration_test.go new file mode 100644 index 000000000..bd3debc1b --- /dev/null +++ b/test/suites/integration/expiration_test.go @@ -0,0 +1,166 @@ +/* +Portions Copyright (c) Microsoft Corporation. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package integration_test + +import ( + "time" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "github.com/samber/lo" + v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/labels" + "k8s.io/apimachinery/pkg/util/intstr" + "sigs.k8s.io/controller-runtime/pkg/client" + + corev1beta1 "github.com/aws/karpenter-core/pkg/apis/v1beta1" + + coretest "github.com/aws/karpenter-core/pkg/test" +) + +var _ = Describe("Expiration", func() { + + var _ = BeforeEach(func() { + nodePool.Spec.Disruption.ExpireAfter = corev1beta1.NillableDuration{Duration: lo.ToPtr(time.Second * 30)} + }) + + It("should expire the node after the expiration is reached", func() { + var numPods int32 = 1 + dep := coretest.Deployment(coretest.DeploymentOptions{ + Replicas: numPods, + PodOptions: coretest.PodOptions{ + ObjectMeta: metav1.ObjectMeta{ + Annotations: map[string]string{ + corev1beta1.DoNotDisruptAnnotationKey: "true", + }, + Labels: map[string]string{"app": "large-app"}, + }, + }, + }) + selector := labels.SelectorFromSet(dep.Spec.Selector.MatchLabels) + env.ExpectCreated(nodeClass, nodePool, dep) + + nodeClaim := env.EventuallyExpectCreatedNodeClaimCount("==", 1)[0] + node := env.EventuallyExpectCreatedNodeCount("==", 1)[0] + env.EventuallyExpectHealthyPodCount(selector, int(numPods)) + env.Monitor.Reset() // Reset the monitor so that we can expect a single node to be spun up after expiration + + // Expect that the NodeClaim will get an expired status condition + Eventually(func(g Gomega) { + g.Expect(env.Client.Get(env, client.ObjectKeyFromObject(nodeClaim), nodeClaim)).To(Succeed()) + g.Expect(nodeClaim.StatusConditions().GetCondition(corev1beta1.Expired).IsTrue()).To(BeTrue()) + }).Should(Succeed()) + + // Remove the do-not-disrupt annotation so that the Nodes are now deprovisionable + for _, pod := range env.ExpectPodsMatchingSelector(selector) { + delete(pod.Annotations, corev1beta1.DoNotDisruptAnnotationKey) + env.ExpectUpdated(pod) + } + + // Eventually the node will be set as unschedulable, which means its actively being deprovisioned + Eventually(func(g Gomega) { + g.Expect(env.Client.Get(env.Context, client.ObjectKeyFromObject(node), node)).Should(Succeed()) + _, ok := lo.Find(node.Spec.Taints, func(t v1.Taint) bool { + return corev1beta1.IsDisruptingTaint(t) + }) + g.Expect(ok).To(BeTrue()) + }).Should(Succeed()) + + // Set the expireAfter to "Never" to make sure new node isn't deleted + // This is CRITICAL since it prevents nodes that are immediately spun up from immediately being expired and + // racing at the end of the E2E test, leaking node resources into subsequent tests + nodePool.Spec.Disruption.ExpireAfter.Duration = nil + env.ExpectUpdated(nodePool) + + // After the deletion timestamp is set and all pods are drained + // the node should be gone + env.EventuallyExpectNotFound(nodeClaim, node) + + env.EventuallyExpectCreatedNodeClaimCount("==", 1) + env.EventuallyExpectCreatedNodeCount("==", 1) + env.EventuallyExpectHealthyPodCount(selector, int(numPods)) + }) + It("should replace expired node with a single node and schedule all pods", func() { + var numPods int32 = 5 + // We should setup a PDB that will only allow a minimum of 1 pod to be pending at a time + minAvailable := intstr.FromInt32(numPods - 1) + pdb := coretest.PodDisruptionBudget(coretest.PDBOptions{ + Labels: map[string]string{ + "app": "large-app", + }, + MinAvailable: &minAvailable, + }) + dep := coretest.Deployment(coretest.DeploymentOptions{ + Replicas: numPods, + PodOptions: coretest.PodOptions{ + ObjectMeta: metav1.ObjectMeta{ + Annotations: map[string]string{ + corev1beta1.DoNotDisruptAnnotationKey: "true", + }, + Labels: map[string]string{"app": "large-app"}, + }, + }, + }) + selector := labels.SelectorFromSet(dep.Spec.Selector.MatchLabels) + env.ExpectCreated(nodeClass, nodePool, pdb, dep) + + nodeClaim := env.EventuallyExpectCreatedNodeClaimCount("==", 1)[0] + node := env.EventuallyExpectCreatedNodeCount("==", 1)[0] + env.EventuallyExpectHealthyPodCount(selector, int(numPods)) + env.Monitor.Reset() // Reset the monitor so that we can expect a single node to be spun up after expiration + + // Set the expireAfter value to get the node deleted + nodePool.Spec.Disruption.ExpireAfter.Duration = lo.ToPtr(time.Minute) + env.ExpectUpdated(nodePool) + + // Expect that the NodeClaim will get an expired status condition + Eventually(func(g Gomega) { + g.Expect(env.Client.Get(env, client.ObjectKeyFromObject(nodeClaim), nodeClaim)).To(Succeed()) + g.Expect(nodeClaim.StatusConditions().GetCondition(corev1beta1.Expired).IsTrue()).To(BeTrue()) + }).Should(Succeed()) + + // Remove the do-not-disruption annotation so that the Nodes are now deprovisionable + for _, pod := range env.ExpectPodsMatchingSelector(selector) { + delete(pod.Annotations, corev1beta1.DoNotDisruptAnnotationKey) + env.ExpectUpdated(pod) + } + + // Eventually the node will be set as unschedulable, which means its actively being deprovisioned + Eventually(func(g Gomega) { + g.Expect(env.Client.Get(env.Context, client.ObjectKeyFromObject(node), node)).Should(Succeed()) + _, ok := lo.Find(node.Spec.Taints, func(t v1.Taint) bool { + return corev1beta1.IsDisruptingTaint(t) + }) + g.Expect(ok).To(BeTrue()) + }).Should(Succeed()) + + // Set the expireAfter to "Never" to make sure new node isn't deleted + // This is CRITICAL since it prevents nodes that are immediately spun up from immediately being expired and + // racing at the end of the E2E test, leaking node resources into subsequent tests + nodePool.Spec.Disruption.ExpireAfter.Duration = nil + env.ExpectUpdated(nodePool) + + // After the deletion timestamp is set and all pods are drained + // the node should be gone + env.EventuallyExpectNotFound(nodeClaim, node) + + env.EventuallyExpectCreatedNodeClaimCount("==", 1) + env.EventuallyExpectCreatedNodeCount("==", 1) + env.EventuallyExpectHealthyPodCount(selector, int(numPods)) + }) +}) diff --git a/test/suites/integration/storage_test.go b/test/suites/integration/storage_test.go new file mode 100644 index 000000000..8a3598403 --- /dev/null +++ b/test/suites/integration/storage_test.go @@ -0,0 +1,124 @@ +/* +Portions Copyright (c) Microsoft Corporation. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package integration_test + +import ( + "fmt" + + appsv1 "k8s.io/api/apps/v1" + v1 "k8s.io/api/core/v1" + storagev1 "k8s.io/api/storage/v1" + "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/controller-runtime/pkg/client" + + "github.com/aws/karpenter-core/pkg/test" + "github.com/samber/lo" + + . "github.com/onsi/ginkgo/v2" +) + +// This test requires the Azure Disk CSI driver to be installed +var _ = Describe("Dynamic PVC", func() { + It("should run a pod with a dynamic persistent volume", func() { + // Ensure that the Azure Disk driver is installed, or we can't run the test. + var ds appsv1.DaemonSet + if err := env.Client.Get(env.Context, client.ObjectKey{ + Namespace: "kube-system", + Name: "csi-azuredisk-node", + }, &ds); err != nil { + if errors.IsNotFound(err) { + Skip(fmt.Sprintf("skipping dynamic PVC test due to missing Azure Disk driver %s", err)) + } else { + Fail(fmt.Sprintf("determining Azure Disk driver status, %s", err)) + } + } + + storageClassName := lo.ToPtr("azuredisk-sc-test") + bindMode := storagev1.VolumeBindingWaitForFirstConsumer + sc := test.StorageClass(test.StorageClassOptions{ + ObjectMeta: metav1.ObjectMeta{ + Name: *storageClassName, + }, + Provisioner: lo.ToPtr("disk.csi.azure.com"), + VolumeBindingMode: &bindMode, + }) + + pvc := test.PersistentVolumeClaim(test.PersistentVolumeClaimOptions{ + ObjectMeta: metav1.ObjectMeta{ + Name: "azuredisk-claim", + }, + StorageClassName: storageClassName, + Resources: v1.ResourceRequirements{Requests: v1.ResourceList{v1.ResourceStorage: resource.MustParse("5Gi")}}, + }) + + pod := test.Pod(test.PodOptions{ + PersistentVolumeClaims: []string{pvc.Name}, + }) + + env.ExpectCreated(nodeClass, nodePool, sc, pvc, pod) + env.EventuallyExpectHealthy(pod) + env.ExpectCreatedNodeCount("==", 1) + env.ExpectDeleted(pod) + }) +}) + +var _ = Describe("Static PVC", func() { + It("should run a pod with a static persistent volume using Azure File", func() { + storageClassName := lo.ToPtr("azurefile-test") + bindMode := storagev1.VolumeBindingImmediate + sc := test.StorageClass(test.StorageClassOptions{ + ObjectMeta: metav1.ObjectMeta{ + Name: *storageClassName, + }, + Provisioner: lo.ToPtr("file.csi.azure.com"), + VolumeBindingMode: &bindMode, + }) + + pv := test.PersistentVolume(test.PersistentVolumeOptions{ + ObjectMeta: metav1.ObjectMeta{Name: "azurefile-test-volume"}, + StorageClassName: *storageClassName, + }) + + // Set up Azure File source + pv.Spec.AzureFile = &v1.AzureFilePersistentVolumeSource{ + SecretName: "azure-secret", // Should have Azure Storage Account Name and Key + ShareName: "myshare", // Name of the Azure File Share + ReadOnly: false, + } + pv.Spec.CSI = nil + + pvc := test.PersistentVolumeClaim(test.PersistentVolumeClaimOptions{ + ObjectMeta: metav1.ObjectMeta{ + Name: "azurefile-claim", + }, + StorageClassName: storageClassName, + VolumeName: pv.Name, + Resources: v1.ResourceRequirements{Requests: v1.ResourceList{v1.ResourceStorage: resource.MustParse("5Gi")}}, + }) + + pod := test.Pod(test.PodOptions{ + PersistentVolumeClaims: []string{pvc.Name}, + }) + + env.ExpectCreated(nodeClass, nodePool, sc, pv, pvc, pod) + env.EventuallyExpectHealthy(pod) + env.ExpectCreatedNodeCount("==", 1) + env.ExpectDeleted(pod) + }) +}) diff --git a/test/suites/integration/suite_test.go b/test/suites/integration/suite_test.go new file mode 100644 index 000000000..88b1d582b --- /dev/null +++ b/test/suites/integration/suite_test.go @@ -0,0 +1,51 @@ +/* +Portions Copyright (c) Microsoft Corporation. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package integration_test + +import ( + "testing" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "github.com/Azure/karpenter/pkg/apis/v1alpha2" + "github.com/Azure/karpenter/test/pkg/environment/azure" + corev1beta1 "github.com/aws/karpenter-core/pkg/apis/v1beta1" +) + +var env *azure.Environment +var nodeClass *v1alpha2.AKSNodeClass +var nodePool *corev1beta1.NodePool + +func TestIntegration(t *testing.T) { + RegisterFailHandler(Fail) + BeforeSuite(func() { + env = azure.NewEnvironment(t) + }) + AfterSuite(func() { + env.Stop() + }) + RunSpecs(t, "Integration") +} + +var _ = BeforeEach(func() { + env.BeforeEach() + nodeClass = env.DefaultAKSNodeClass() + nodePool = env.DefaultNodePool(nodeClass) +}) +var _ = AfterEach(func() { env.Cleanup() }) +var _ = AfterEach(func() { env.AfterEach() }) diff --git a/test/suites/integration/termination_test.go b/test/suites/integration/termination_test.go new file mode 100644 index 000000000..554528257 --- /dev/null +++ b/test/suites/integration/termination_test.go @@ -0,0 +1,43 @@ +/* +Portions Copyright (c) Microsoft Corporation. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package integration_test + +import ( + . "github.com/onsi/ginkgo/v2" + + "github.com/aws/karpenter-core/pkg/test" +) + +var _ = Describe("Termination", func() { + It("should terminate the node and the instance on deletion", func() { + pod := test.Pod() + env.ExpectCreated(nodeClass, nodePool, pod) + env.EventuallyExpectHealthy(pod) + env.ExpectCreatedNodeCount("==", 1) + + nodes := env.Monitor.CreatedNodes() + + // Pod is deleted so that we don't re-provision after node deletion + // NOTE: We have to do this right now to deal with a race condition in provisioner ownership + // This can be removed once this race is resolved with the Machine + env.ExpectDeleted(pod) + + // Node is deleted and now should be not found + env.ExpectDeleted(nodes[0]) + env.EventuallyExpectNotFound(nodes[0]) + }) +}) diff --git a/test/suites/utilization/suite_test.go b/test/suites/utilization/suite_test.go new file mode 100644 index 000000000..3d19f37d9 --- /dev/null +++ b/test/suites/utilization/suite_test.go @@ -0,0 +1,67 @@ +/* +Portions Copyright (c) Microsoft Corporation. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package utilization_test + +import ( + "testing" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + v1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" + "k8s.io/apimachinery/pkg/labels" + + "github.com/aws/karpenter-core/pkg/test" + + "github.com/Azure/karpenter/pkg/apis/v1alpha2" + "github.com/Azure/karpenter/test/pkg/environment/azure" +) + +var env *azure.Environment + +func TestUtilization(t *testing.T) { + RegisterFailHandler(Fail) + BeforeSuite(func() { + env = azure.NewEnvironment(t) + }) + RunSpecs(t, "Utilization") +} + +var _ = BeforeEach(func() { env.BeforeEach() }) +var _ = AfterEach(func() { env.Cleanup() }) +var _ = AfterEach(func() { env.AfterEach() }) + +// TODO (charliedmcb): add back in referenced to debug lib. For now I'm commenting out all references for this test, to avoid vetting the lib +// var _ = Describe("Utilization", Label(debug.NoWatch), Label(debug.NoEvents), func() { +var _ = Describe("Utilization", func() { + It("should provision one pod per node", func() { + nodeClass := env.DefaultAKSNodeClass() + nodePool := env.DefaultNodePool(nodeClass) + test.ReplaceRequirements(nodePool, v1.NodeSelectorRequirement{ + Key: v1alpha2.LabelSKUCPU, + Operator: v1.NodeSelectorOpLt, + Values: []string{"3"}, + }) + deployment := test.Deployment(test.DeploymentOptions{ + Replicas: 10, + PodOptions: test.PodOptions{ResourceRequirements: v1.ResourceRequirements{Requests: v1.ResourceList{v1.ResourceCPU: resource.MustParse("1.1")}}, Image: "mcr.microsoft.com/oss/kubernetes/pause:3.6"}}) // See above comment on the required adding of the azure PodOptions.Image reference + + env.ExpectCreated(nodePool, nodeClass, deployment) + env.EventuallyExpectHealthyPodCount(labels.SelectorFromSet(deployment.Spec.Selector.MatchLabels), int(*deployment.Spec.Replicas)) + env.ExpectCreatedNodeCount("==", int(*deployment.Spec.Replicas)) // One pod per node enforced by instance size + }) +})