diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md
index 9586e7b999822..3361d1335ba7b 100644
--- a/.github/pull_request_template.md
+++ b/.github/pull_request_template.md
@@ -12,5 +12,4 @@ Fixes #<issue number>
 - [ ] Title matches the required conventional commits format, see [here](https://www.conventionalcommits.org/en/v1.0.0/)
   - **Note** that Promtail is considered to be feature complete, and future development for logs collection will be in [Grafana Alloy](https://github.com/grafana/alloy). As such, `feat` PRs are unlikely to be accepted unless a case can be made for the feature actually being a bug fix to existing behavior.
 - [ ] Changes that require user attention or interaction to upgrade are documented in `docs/sources/setup/upgrade/_index.md`
-- [ ] For Helm chart changes bump the Helm chart version in `production/helm/loki/Chart.yaml` and update `production/helm/loki/CHANGELOG.md` and `production/helm/loki/README.md`. [Example PR](https://github.com/grafana/loki/commit/d10549e3ece02120974929894ee333d07755d213)
 - [ ] If the change is deprecating or removing a configuration option, update the `deprecated-config.yaml` and `deleted-config.yaml` files respectively in the `tools/deprecated-config-checker` directory. [Example PR](https://github.com/grafana/loki/pull/10840/commits/0d4416a4b03739583349934b96f272fb4f685d15)
diff --git a/.github/workflows/backport.yml b/.github/workflows/backport.yml
index 241d662251451..fbdc3e8a65e32 100644
--- a/.github/workflows/backport.yml
+++ b/.github/workflows/backport.yml
@@ -31,4 +31,4 @@ jobs:
           metricsWriteAPIKey: ${{secrets.GRAFANA_MISC_STATS_API_KEY}}
           token: ${{ steps.get_github_app_token.outputs.token }}
           labelsToAdd: "backport"
-          title: "chore: [{{base}}] {{originalTitle}}"
+          title: "{{originalTitle}} (backport {{base}})"
diff --git a/.github/workflows/helm-ci.yml b/.github/workflows/helm-ci.yml
index cbd51c60b8fac..80ea1db668653 100644
--- a/.github/workflows/helm-ci.yml
+++ b/.github/workflows/helm-ci.yml
@@ -4,6 +4,7 @@ on:
   pull_request:
     paths:
       - "production/helm/**"
+      - ".github/workflows/helm-ci.yml"
 
 env:
   CT_CONFIGFILE: production/helm/ct.yaml
@@ -27,14 +28,6 @@ jobs:
       - name: Lint Yaml
         run: make helm-lint
 
-      - id: "get_github_app_token"
-        name: "Get Github app token"
-        uses: "actions/create-github-app-token@v1"
-        with:
-          app-id: "${{ secrets.APP_ID }}"
-          owner: "${{ github.repository_owner }}"
-          private-key: "${{ secrets.APP_PRIVATE_KEY }}"
-
       - name: Lint Code Base
         uses: docker://github/super-linter:v3.12.0
         env:
@@ -45,7 +38,7 @@ jobs:
           VALIDATE_YAML: false
           VALIDATE_GO: false
           DEFAULT_BRANCH: main
-          GITHUB_TOKEN: ${{ steps.get_github_app_token.outputs.token }}
+          GITHUB_TOKEN: ${{secrets.GITHUB_TOKEN}}
   call-test:
     name: Test Helm Chart
     runs-on: ubuntu-latest
diff --git a/.github/workflows/helm-release.yaml b/.github/workflows/helm-release.yaml
index d2f9fe97ba4f6..b0ee40c36568d 100644
--- a/.github/workflows/helm-release.yaml
+++ b/.github/workflows/helm-release.yaml
@@ -1,12 +1,7 @@
 name: helm-release
 
 on:
-  push:
-    branches:
-      - main
-      - helm-5.48
-    paths:
-      - 'production/helm/loki/Chart.yaml'
+  workflow_dispatch: # must be invoked manually
 
 jobs:
   call-update-helm-repo:
diff --git a/.github/workflows/helm-tagged-release-pr.yaml b/.github/workflows/helm-tagged-release-pr.yaml
new file mode 100644
index 0000000000000..1a5e6bdeccff8
--- /dev/null
+++ b/.github/workflows/helm-tagged-release-pr.yaml
@@ -0,0 +1,35 @@
+name: helm-weekly-release-pr
+
+on:
+  release:
+    types:
+      - released
+
+jobs:
+  weekly-release-pr:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - id: "get_github_app_token"
+        name: "get github app token"
+        uses: "actions/create-github-app-token@v1"
+        with:
+          app-id: "${{ secrets.APP_ID }}"
+          owner: "${{ github.repository_owner }}"
+          private-key: "${{ secrets.APP_PRIVATE_KEY }}"
+
+      - name: Update/regenerate files
+        id: update
+        run: bash .github/workflows/scripts/helm-tagged-release.sh ${{ github.event.release.tag_name }}
+
+      - name: Create Pull Request
+        uses: peter-evans/create-pull-request@v5
+        with:
+          token: ${{ steps.get_github_app_token.outputs.token }}
+          title: Release loki Helm chart ${{ steps.update.outputs.new_chart_version }}
+          body: Automated PR created by [helm-tagged-release-pr.yaml](https://github.com/grafana/loki/blob/main/.github/workflows/helm-tagged-release-pr.yaml)
+          commit-message: Update loki chart to ${{ steps.update.outputs.new_chart_version }}
+          branch: helm-chart-tagged-${{ steps.update.outputs.new_chart_version }}
+          base: main
+          labels: helm
diff --git a/.github/workflows/helm-weekly-release-pr.yaml b/.github/workflows/helm-weekly-release-pr.yaml
new file mode 100644
index 0000000000000..7ac88b7b95841
--- /dev/null
+++ b/.github/workflows/helm-weekly-release-pr.yaml
@@ -0,0 +1,37 @@
+name: helm-weekly-release-pr
+
+on:
+  schedule:
+    - cron: '0 10 * * 1-5' # 10 UTC on weekdays; if we miss published images one day, they should align the day after
+
+  workflow_dispatch: # for manual testing
+
+jobs:
+  weekly-release-pr:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: imjasonh/setup-crane@v0.4
+
+      - id: "get_github_app_token"
+        name: "get github app token"
+        uses: "actions/create-github-app-token@v1"
+        with:
+          app-id: "${{ secrets.APP_ID }}"
+          owner: "${{ github.repository_owner }}"
+          private-key: "${{ secrets.APP_PRIVATE_KEY }}"
+
+      - name: Update/regenerate files
+        id: update
+        run: bash .github/workflows/scripts/helm-weekly-release.sh
+
+      - name: Create Pull Request
+        uses: peter-evans/create-pull-request@v5
+        with:
+          token: ${{ steps.get_github_app_token.outputs.token }}
+          title: Release loki Helm chart ${{ steps.update.outputs.new_chart_version }}
+          body: Automated PR created by [helm-weekly-release-pr.yaml](https://github.com/grafana/loki/blob/main/.github/workflows/helm-weekly-release-pr.yaml)
+          commit-message: Update loki chart to ${{ steps.update.outputs.new_chart_version }}
+          branch: helm-chart-weekly-${{ steps.update.outputs.new_chart_version }}
+          base: main
+          labels: helm
diff --git a/.github/workflows/scripts/common.sh b/.github/workflows/scripts/common.sh
new file mode 100644
index 0000000000000..b5cba118af718
--- /dev/null
+++ b/.github/workflows/scripts/common.sh
@@ -0,0 +1,45 @@
+#!/usr/bin/env bash
+
+set -exo pipefail
+
+# This generates a new file where the yaml node is updated.
+# The problem is that yq strips new lines when you update the file.
+# So we use a workaround from https://github.com/mikefarah/yq/issues/515 which:
+# generates the new file, diffs it with the original, removes all non-whitespace changes, and applies that to the original file.
+update_yaml_node() {
+  local filename=$1
+  local yaml_node=$2
+  local new_value=$3
+  patch "${filename}" <<<"$(diff -U0 -w -b --ignore-blank-lines "${filename}" <(yq eval "${yaml_node} = \"${new_value}\"" "${filename}"))"
+}
+
+get_yaml_node() {
+  local filename=$1
+  local yaml_node=$2
+  yq "${yaml_node}" "${filename}"
+}
+
+# Increments the part of the semver string
+# $1: version itself
+# $2: number of part: 0 – major, 1 – minor, 2 – patch
+increment_semver() {
+  local delimiter=.
+  local array=("$(echo "$1" | tr "${delimiter}" '\n')")
+  array[$2]=$((array[$2] + 1))
+  echo "$(
+    local IFS=${delimiter}
+    echo "${array[*]}"
+  )"
+}
+
+# Sets the patch segment of a semver to 0
+# $1: version itself
+set_semver_patch_to_zero() {
+  local delimiter=.
+  local array=("$(echo "$1" | tr "${delimiter}" '\n')")
+  array[2]="0"
+  echo "$(
+    local IFS=${delimiter}
+    echo "${array[*]}"
+  )"
+}
diff --git a/.github/workflows/scripts/helm-tagged-release.sh b/.github/workflows/scripts/helm-tagged-release.sh
new file mode 100755
index 0000000000000..fd6c06f520d04
--- /dev/null
+++ b/.github/workflows/scripts/helm-tagged-release.sh
@@ -0,0 +1,54 @@
+#!/usr/bin/env bash
+# SPDX-License-Identifier: AGPL-3.0-only
+
+set -exo pipefail
+
+script_dir=$(cd "$(dirname "$0")" && pwd)
+# shellcheck disable=SC2250,SC1091
+source "${script_dir}/common.sh"
+
+calculate_next_chart_version() {
+  local current_chart_version=$1
+
+  local current_chart_semver
+  current_chart_semver="$(echo "${current_chart_version}" | grep -P -o '^(\d+.){2}\d+')"
+  local new_chart_semver="${current_chart_semver}"
+  new_chart_semver=$(increment_semver "${current_chart_semver}" 1)
+  new_chart_semver=$(set_semver_patch_to_zero "${new_chart_semver}")
+  echo "${new_chart_semver}"
+}
+
+validate_version_update() {
+  local new_chart_version=$1
+  local current_chart_version=$2
+  local latest_loki_tag=$3
+
+  if [[ "${new_chart_version}" == "${current_chart_version}" ]]; then
+    echo "New chart version (${new_chart_version}) is the same as current version (${current_chart_version}); not submitting PR"
+    exit 1
+  fi
+}
+
+latest_loki_tag=$(sed -E "s/v(.*)/\1/g" <<<"$1")
+
+values_file=production/helm/loki/values.yaml
+chart_file=production/helm/loki/Chart.yaml
+
+current_chart_version=$(get_yaml_node "${chart_file}" .version)
+new_chart_version=$(calculate_next_chart_version "${current_chart_version}")
+
+validate_version_update "${new_chart_version}" "${current_chart_version}" "${latest_loki_tag}"
+
+update_yaml_node "${values_file}" .loki.image.tag "${latest_loki_tag}"
+
+update_yaml_node "${values_file}" .enterprise.image.tag "${latest_loki_tag}"
+update_yaml_node "${chart_file}" .appVersion "${latest_loki_tag}"
+update_yaml_node "${chart_file}" .version "${new_chart_version}"
+
+sed --in-place \
+  --regexp-extended \
+  "s/(.*\<AUTOMATED_UPDATES_LOCATOR\>.*)/\1\n\n## ${new_chart_version}\n\n- \[CHANGE\] Changed version of Grafana Loki to ${latest_loki_tag}/g" production/helm/loki/CHANGELOG.md
+
+make TTY='' helm-docs
+
+echo "::set-output name=new_chart_version::${new_chart_version}"
diff --git a/.github/workflows/scripts/helm-weekly-release.sh b/.github/workflows/scripts/helm-weekly-release.sh
new file mode 100755
index 0000000000000..64d5f29f4557f
--- /dev/null
+++ b/.github/workflows/scripts/helm-weekly-release.sh
@@ -0,0 +1,84 @@
+#!/usr/bin/env bash
+# SPDX-License-Identifier: AGPL-3.0-only
+
+set -exo pipefail
+
+script_dir=$(cd "$(dirname "$0")" && pwd)
+# shellcheck disable=SC2250,SC1091
+source "${script_dir}/common.sh"
+
+# Uses docker hub image tags to figure out what is the latest image tag
+find_latest_image_tag() {
+  local docker_hub_repo=$1
+  local regExp="^(k|weekly-k)\d+-[a-z0-9]+"
+  crane ls "${docker_hub_repo}" | grep -P "${regExp}" | sed -E "s/([weekly-]*k[[:digit:]]*)-([^-]*).*/\1-\2/g" | uniq | sort -Vur | head -1
+}
+
+# takes k197-abcdef and returns r197, k197-abcdef-arm64 and returns k197, weekly-k197-abcdef and returns k197
+extract_k_version() {
+  sed -E "s/[weekly-]*(k[[:digit:]]*).*/\1/g" <<<"$1"
+}
+
+calculate_next_chart_version() {
+  local current_chart_version=$1
+  local latest_image_tag=$2
+
+  local current_chart_semver
+  current_chart_semver=$(echo "${current_chart_version}" | grep -P -o '^(\d+.){2}\d+')
+  local new_chart_weekly
+  new_chart_weekly=$(extract_k_version "${latest_image_tag}" | grep -P -o '\d+')
+  local new_chart_semver="${current_chart_semver}"
+  if [[ "${current_chart_version}" != *weekly* ]]; then
+    # If previous version was not a weekly, then it was a stable release.
+    # _This_ weekly release should have a semver that's one above the stable release.
+    new_chart_semver=$(increment_semver "${current_chart_semver}" 1)
+    # Also reset the patch release number to 0.
+    new_chart_semver=$(set_semver_patch_to_zero "${new_chart_semver}")
+  fi
+  echo "${new_chart_semver}-weekly.${new_chart_weekly}"
+}
+
+validate_version_update() {
+  local new_chart_version=$1
+  local current_chart_version=$2
+  local latest_gel_tag=$3
+  local latest_loki_tag=$4
+
+  if [[ "${new_chart_version}" == "${current_chart_version}" ]]; then
+    echo "New chart version (${new_chart_version}) is the same as current version (${current_chart_version}); not submitting weekly PR"
+    exit 1
+  fi
+
+  local gel_weekly_version
+  gel_weekly_version=$(extract_k_version "${latest_gel_tag}")
+  local loki_weekly_version
+  loki_weekly_version=$(extract_k_version "${latest_loki_tag}")
+  echo "Comparing GEL weekly version (${gel_weekly_version}) with Loki weekly version (${loki_weekly_version})"
+  if [[ "${gel_weekly_version}" != "${loki_weekly_version}" ]]; then
+    echo "GEL weekly version (${gel_weekly_version}) does not match Loki weekly version (${loki_weekly_version}); not submitting PR"
+    exit 1
+  fi
+}
+
+values_file=production/helm/loki/values.yaml
+chart_file=production/helm/loki/Chart.yaml
+
+latest_loki_tag=$(find_latest_image_tag grafana/loki)
+latest_gel_tag=$(find_latest_image_tag grafana/enterprise-logs)
+current_chart_version=$(get_yaml_node "${chart_file}" .version)
+new_chart_version=$(calculate_next_chart_version "${current_chart_version}" "${latest_loki_tag}")
+
+validate_version_update "${new_chart_version}" "${current_chart_version}" "${latest_gel_tag}" "${latest_loki_tag}"
+
+update_yaml_node "${values_file}" .loki.image.tag "${latest_loki_tag}"
+update_yaml_node "${values_file}" .enterprise.image.tag "${latest_gel_tag}"
+update_yaml_node "${chart_file}" .appVersion "$(extract_k_version "${latest_loki_tag}")"
+update_yaml_node "${chart_file}" .version "${new_chart_version}"
+
+sed --in-place \
+  --regexp-extended \
+  "s/(.*\<AUTOMATED_UPDATES_LOCATOR\>.*)/\1\n\n## ${new_chart_version}\n\n- \[CHANGE\] Changed version of Grafana Loki to ${latest_loki_tag}\n- \[CHANGE\] Changed version of Grafana Enterprise Logs to ${latest_gel_tag}/g" production/helm/loki/CHANGELOG.md
+
+make TTY='' helm-docs
+
+echo "::set-output name=new_chart_version::${new_chart_version}"
diff --git a/.github/workflows/verify-drone.yml b/.github/workflows/verify-drone.yml
deleted file mode 100644
index 04f3c05ff43f7..0000000000000
--- a/.github/workflows/verify-drone.yml
+++ /dev/null
@@ -1,52 +0,0 @@
-name: Verify drone updates
-on: [pull_request]
-jobs:
-  check-drone-changes:
-    runs-on: ubuntu-latest
-    steps:
-    - uses: actions/checkout@v4
-    - name: Get changed files
-      # we need continue on error because the git diff | grep pipe can return a non-zero error code if no result is found
-      continue-on-error: true
-      id: changed-files
-      run: |
-            echo "changed_files=$(git diff --name-only main --  .drone/ | xargs)" >> $GITHUB_OUTPUT
-            git diff main .drone/ | grep "+hmac "
-            echo "sha_updated=$?" >> $GITHUB_OUTPUT
-    - name: Check that drone was updated properly
-      if: always()
-      run: |
-          jsonnetChanged=false
-          yamlChanged=false
-            echo "sha updated? ${{ steps.changed-files.outputs.sha_updated }}"
-
-          # check whether the drone jsonnet and yaml files were updated
-          for file in ${{ steps.changed-files.outputs.changed_files }}; do
-            if [ "$file" == ".drone/drone.jsonnet" ]; then
-              echo "$file was changed"
-              jsonnetChanged=true
-            fi
-            if [ "$file" == ".drone/drone.yml" ]; then
-              echo "$file was changed"
-              yamlChanged=true
-            fi
-          done
-
-          # if niether file was changed we're okay
-          if { [ "$yamlChanged" = false ] && [ "$jsonnetChanged" = false ]; } then
-            echo "neither file was changed"
-            exit 0
-          fi
-          # if both files were changed then we should ensure that the sha in the yaml was also updated
-          if { [ "$yamlChanged" = true ] && [ "$jsonnetChanged" = true ]; } then
-            # annoyingly, the return value is a string
-            if [ "${{ steps.changed-files.outputs.sha_updated }}" = "0" ]; then
-              echo "both files were changed and sha was updated"
-              exit 0
-            fi
-            echo "both drone yaml and jsonnet were updated but the sha in the yaml file was not updated"
-            exit 1
-          fi
-          # only one of the two files was updated
-          echo "if one of the drone files (yaml or jsonnet) was changed then bothy files must be updated"
-          exit 1
\ No newline at end of file
diff --git a/CODEOWNERS b/CODEOWNERS
index 32ae698aa9e0a..a118c5a40c0b1 100644
--- a/CODEOWNERS
+++ b/CODEOWNERS
@@ -5,8 +5,8 @@
 /docs/ @grafana/docs-logs @grafana/loki-team
 
 # Loki operator
-/operator/ @grafana/loki-team @periklis @xperimental
-/.github/workflows/operator* @grafana/loki-team @periklis @xperimental
+/operator/ @grafana/loki-team @periklis @xperimental @JoaoBraveCoding
+/.github/workflows/operator* @grafana/loki-team @periklis @xperimental @JoaoBraveCoding
 
 # Logql grammar
 # The observability logs team is listed as co-codeowner for grammar file. This is to receive notifications about updates, so these can be implemented in https://github.com/grafana/lezer-logql
diff --git a/clients/cmd/fluentd/Dockerfile b/clients/cmd/fluentd/Dockerfile
index ba20c9d095188..d68baa68db31f 100644
--- a/clients/cmd/fluentd/Dockerfile
+++ b/clients/cmd/fluentd/Dockerfile
@@ -1,4 +1,4 @@
-FROM ruby:3.3.4 AS build
+FROM ruby:3.3.5 AS build
 
 ENV DEBIAN_FRONTEND=noninteractive
 
diff --git a/clients/cmd/fluentd/lib/fluent/plugin/out_loki.rb b/clients/cmd/fluentd/lib/fluent/plugin/out_loki.rb
index f2bb30538a400..b3d789ea684b9 100644
--- a/clients/cmd/fluentd/lib/fluent/plugin/out_loki.rb
+++ b/clients/cmd/fluentd/lib/fluent/plugin/out_loki.rb
@@ -61,6 +61,9 @@ class LogPostError < StandardError; end
       desc 'TLS: disable server certificate verification'
       config_param :insecure_tls, :bool, default: false
 
+      desc 'Custom HTTP headers'
+      config_param :custom_headers, :hash, default: {}
+
       desc 'Loki tenant id'
       config_param :tenant, :string, default: nil
 
@@ -232,6 +235,9 @@ def loki_http_request(body, tenant)
         req = Net::HTTP::Post.new(
           @uri.request_uri
         )
+        @custom_headers.each do |key, value|
+          req.add_field(key, value)
+        end
         req.add_field('Content-Type', 'application/json')
         req.add_field('Authorization', "Bearer #{@auth_token_bearer}") unless @auth_token_bearer.nil?
         req.add_field('X-Scope-OrgID', tenant) if tenant
diff --git a/docs/sources/query/template_functions.md b/docs/sources/query/template_functions.md
index 68f42965cfa7e..499220c5e377c 100644
--- a/docs/sources/query/template_functions.md
+++ b/docs/sources/query/template_functions.md
@@ -1,31 +1,31 @@
 ---
-title: Template functions
-menuTItle:  
-description: Describes functions that are supported by the Go text template.
-aliases: 
+title: LogQL template functions
+menuTItle:  Template functions
+description: Describes query functions that are supported by the Go text template.
+aliases:
 - ../logql/template_functions/
 weight: 30
 ---
 
-# Template functions
+# LogQL template functions
 
+The Go templating language is embedded in the Loki query language, LogQL.
 The [text template](https://golang.org/pkg/text/template) format used in `| line_format` and `| label_format` support the usage of functions.
 
-{{% admonition type="note" %}}
+{{< admonition type="note" >}}
 In the examples below we use backticks to quote the template strings. This is because some template strings contain double quotes, and using backticks lets us avoid escaping the double quotes.
 If you are using a different quoting style, you may need to escape the double quotes.
-{{% /admonition %}}
+{{< /admonition >}}
 
-All labels are added as variables in the template engine. They can be referenced using they label name prefixed by a `.`(e.g `.label_name`). For example the following template will output the value of the path label:
+For more information, refer to the [Go template documentation](https://pkg.go.dev/text/template).
 
-```template
-`{{ .path }}`
-```
+Additionally you can also access the log line using the `__line__` function and the timestamp using the `__timestamp__` function.
+
+## Template pipeline syntax
 
-Additionally you can also access the log line using the [`__line__`](#__line__) function and the timestamp using the [`__timestamp__`](#__timestamp__) function.
+A pipeline is a possibly chained sequence of "commands". A command is a simple value (argument) or a function or method call, possibly with multiple arguments. A pipeline may be "chained" by separating a sequence of commands with pipeline characters '|'. In a chained pipeline, the result of each command is passed as the last argument of the following command. The output of the final command in the pipeline is the value of the pipeline.
 
 You can take advantage of [pipeline](https://golang.org/pkg/text/template/#hdr-Pipelines) to join together multiple functions.
-In a chained pipeline, the result of each command is passed as the last argument of the following command.
 
 Example:
 
@@ -40,12 +40,24 @@ Example:
 ```template
 `{{ if and (contains "he" "hello") (contains "llo" "hello") }} yes {{end}}`
 `{{ if or (contains "he" "hello") (contains("llo" "hello") }} yes {{end}}`
-`{{ if contains "ErrTimeout" .err }} timeout {{else if contains "he" "hello"}} yes {{else}} no {{end}}`
+`{{ if contains .err "ErrTimeout" }} timeout {{else if contains "he" "hello"}} yes {{else}} no {{end}}`
+```
+
+## Built-in variables for log line properties
+
+These variables provide a way of referencing something from the log line when writing a template expression.
+
+### .label_name
+
+All labels from the Log line are added as variables in the template engine. They can be referenced using the label name prefixed by a `.`(for example,`.label_name`). For example the following template will output the value of the `path` label:
+
+```template
+`{{ .path }}`
 ```
 
-## __line__
+### __line__
 
-This function returns the current log line.
+The `__line__` function returns the original log line without any modifications.
 
 Signature: `line() string`
 
@@ -56,9 +68,9 @@ Examples:
 `{{ __line__ }}`
 ```
 
-## __timestamp__
+### __timestamp__
 
-This function returns the current log lines timestamp.
+The `__timestamp__` function returns the current log line's timestamp.
 
 Signature: `timestamp() time.Time`
 
@@ -68,177 +80,158 @@ Signature: `timestamp() time.Time`
 `{{ __timestamp__ | unixEpoch }}`
 ```
 
-See the blog: [Parsing and formatting date/time in Go](https://www.pauladamsmith.com/blog/2011/05/go_time.html) for more information.
+For more information, refer to the blog [Parsing and formatting date/time in Go](https://www.pauladamsmith.com/blog/2011/05/go_time.html).
 
-## regexReplaceAll and regexReplaceAllLiteral
+## Date and time
 
-`regexReplaceAll` returns a copy of the input string, replacing matches of the Regexp with the replacement string replacement. Inside string replacement, $ signs are interpreted as in Expand, so for instance $1 represents the text of the first sub-match. See the golang [Regexp.replaceAll documentation](https://golang.org/pkg/regexp/#Regexp.ReplaceAll) for more examples.
+You can use the following functions to manipulate dates and times when building LogQL queries.
 
-Example:
+### date
 
-```template
-`{{ regexReplaceAll "(a*)bc" .some_label "${1}a" }}`
-```
+Returns a textual representation of the time value formatted according to the provided [golang datetime layout](https://pkg.go.dev/time#pkg-constants).
 
-`regexReplaceAllLiteral` function returns a copy of the input string and replaces matches of the Regexp with the replacement string replacement. The replacement string is substituted directly, without using Expand.
+Signature: `date(fmt string, date interface{}) string`
 
 Example:
 
 ```template
-`{{ regexReplaceAllLiteral "(ts=)" .timestamp "timestamp=" }}`
+`{{ date "2006-01-02" now }}`
 ```
 
-## lower
-
-Use this function to convert to lower case.
+### duration
 
-Signature: `lower(string) string`
+An alias for `duration_seconds`
 
 Examples:
 
 ```template
-`{{ .request_method | lower }}`
-`{{ lower  "HELLO"}}`
+`{{ .foo | duration }}`
+`{{ duration .foo }}`
 ```
 
-The last example will return `hello`.
-
-## upper
+### duration_seconds
 
-Use this function to convert to upper case.
+Convert a humanized time duration to seconds using [time.ParseDuration](https://pkg.go.dev/time#ParseDuration)
 
-Signature: `upper(string) string`
+Signature: `duration_seconds(string) float64`
 
 Examples:
 
 ```template
-`{ .request_method | upper }}`
-`{{ upper  "hello"}}`
+`{{ .foo | duration_seconds }}`
+`{{ duration_seconds .foo }}`
 ```
 
-This results in `HELLO`.
-
-## title
+### now
 
-Convert to title case.
+Returns the current time in the local timezone of the Loki server.
 
-Signature: `title(string) string`
+Signature: `Now() time.Time`
 
-Examples:
+Example:
 
 ```template
-`{{.request_method | title}}`
-`{{ title "hello world"}}`
+`{{ now }}`
 ```
 
-The last example will return `Hello World`.
+### toDate
 
-## trunc
+Parses a formatted string and returns the time value it represents using the local timezone of the server running Loki.
 
-Truncate a string and add no suffix.
+For more consistency between Loki installations, it's recommended to use `toDateInZone`
 
-Signature: `trunc(count int,value string) string`
+The format string must use the exact date as defined in the [golang datetime layout](https://pkg.go.dev/time#pkg-constants)
+
+Signature: `toDate(fmt, str string) time.Time`
 
 Examples:
 
 ```template
-`{{ .path | trunc 2 }}`
-`{{ trunc 5 "hello world"}}`   // output: hello
-`{{ trunc -5 "hello world"}}`  // output: world
+`{{ toDate "2006-01-02" "2021-11-02" }}`
+`{{ .foo | toDate "2006-01-02T15:04:05.999999999Z" }}`
 ```
 
-## substr
+### toDateInZone
 
-Get a substring from a string.
+Parses a formatted string and returns the time value it represents in the provided timezone.
 
-Signature: `substr(start int,end int,value string) string`
+The format string must use the exact date as defined in the [golang datetime layout](https://pkg.go.dev/time#pkg-constants)
 
-If start is < 0, this calls value[:end].
-If start is >= 0 and end < 0 or end bigger than s length, this calls value[start:]
-Otherwise, this calls value[start, end].
+The timezone value can be `Local`, `UTC`, or any of the IANA Time Zone database values
+
+Signature: `toDateInZone(fmt, zone, str string) time.Time`
 
 Examples:
 
 ```template
-`{{ .path | substr 2 5 }}`
-`{{ substr 0 5 "hello world"}}`  // output: hello
-`{{ substr 6 11 "hello world"}}` // output: world
+`{{ toDateInZone "2006-01-02" "UTC" "2021-11-02" }}`
+`{{ .foo | toDateInZone "2006-01-02T15:04:05.999999999Z" "UTC" }}`
 ```
 
-## replace
-
-This function performs simple string replacement.
-
-Signature: `replace(old string, new string, src string) string`
+### unixEpoch
 
-It takes three arguments:
+Returns the number of seconds elapsed since January 1, 1970 UTC.
 
-- `old` string to replace
-- `new` string to replace with
-- `src` source string
+Signature: `unixEpoch(date time.Time) string`
 
 Examples:
 
 ```template
-`{{ .cluster | replace "-cluster" "" }}`
-`{{ replace "hello" "world" "hello world" }}`
+`{{ unixEpoch now }}`
+`{{ .foo | toDateInZone "2006-01-02T15:04:05.999999999Z" "UTC" | unixEpoch }}`
 ```
 
-The last example will return `world world`.
-
-## trim
-
-The trim function removes space from either side of a string.
-
-Signature: `trim(string) string`
-
-Examples:
+Example of a query to filter Loki querier jobs which create time is 1 day before:
 
-```template
-`{{ .ip | trim }}`
-`{{ trim "   hello    " }}` // output: hello
+```logql
+{job="loki/querier"} | label_format nowEpoch=`{{(unixEpoch now)}}`,createDateEpoch=`{{unixEpoch (toDate "2006-01-02" .createDate)}}` | label_format dateTimeDiff=`{{sub .nowEpoch .createDateEpoch}}` | dateTimeDiff > 86400
 ```
 
-## trimAll
+### unixEpochMillis
 
-Use this function to remove given characters from the front or back of a string.
+Returns the number of milliseconds elapsed since January 1, 1970 UTC.
 
-Signature: `trimAll(chars string,src string) string`
+Signature: `unixEpochMillis(date time.Time) string`
 
 Examples:
 
 ```template
-`{{ .path | trimAll "/" }}`
-`{{ trimAll "$" "$5.00" }}` // output: 5.00
+`{{ unixEpochMillis now }}`
+`{{ .foo | toDateInZone "2006-01-02T15:04:05.999999999Z" "UTC" | unixEpochMillis }}`
 ```
 
-## trimSuffix
+### unixEpochNanos
 
-Use this function to trim just the suffix from a string.
+Returns the number of nanoseconds elapsed since January 1, 1970 UTC.
 
-Signature: `trimSuffix(suffix string, src string) string`
+Signature: `unixEpochNanos(date time.Time) string`
 
 Examples:
 
 ```template
-`{{  .path | trimSuffix "/" }}`
-`{{ trimSuffix "-" "hello-" }}` // output: hello
+`{{ unixEpochNanos now }}`
+`{{ .foo | toDateInZone "2006-01-02T15:04:05.999999999Z" "UTC" | unixEpochNanos }}`
 ```
 
-## trimPrefix
+### unixToTime
 
-Use this function to trim just the prefix from a string.
+Converts the string epoch to the time value it represents. Epoch times in days, seconds, milliseconds, microseconds and nanoseconds are supported.
 
-Signature: `trimPrefix(prefix string, src string) string`
+Signature: `unixToTime(epoch string) time.Time`
 
 Examples:
 
-```template
-`{{  .path | trimPrefix "/" }}`
-`{{ trimPrefix "-" "-hello" }}` // output: hello
+Consider the following log line `{"from": "1679577215","to":"1679587215","message":"some message"}`. To print the `from` field as human readable add the following at the end of the LogQL query:
+
+```logql
+... | json | line_format `from="{{date "2006-01-02" (unixToTime .from)}}"`
 ```
 
-## alignLeft
+## String manipulation
+
+You can use the following templates to manipulate strings when building LogQL Queries.
+
+### alignLeft
 
 Use this function to format a string to a fixed with, aligning the content to the left.
 
@@ -251,7 +244,7 @@ Examples:
 `{{ alignLeft 5 "hi"}}`          // output: "hi   "
 ```
 
-## alignRight
+### alignRight
 
 Use this function to format a string to a fixed with, aligning the content to the right.
 
@@ -264,568 +257,643 @@ Examples:
 `{{ alignRight 5 "hi"}}`          // output: "   hi"
 ```
 
-## indent
+### b64enc
 
-The indent function indents every line in a given string to the specified indent width. This is useful when aligning multi-line strings.
+Base64 encode a string.
 
-Signature: `indent(spaces int,src string) string`
+Signature: `b64enc(string) string`
 
-Example:
+Examples:
 
 ```template
-`{{ indent 4 .query }}`
+`{{ .foo | b64enc }}`
+`{{ b64enc  .foo }}`
 ```
 
-This indents each line contained in the `.query` by four (4) spaces.
-
-## nindent
+### b64dec
 
-The nindent function is the same as the indent function, but prepends a new line to the beginning of the string.
+Base64 decode a string.
 
-Signature: `nindent(spaces int,src string) string`
+Signature: `b64dec(string) string`
 
-Example:
+Examples:
 
 ```template
-`{{ nindent 4 .query }}`
+`{{ .foo | b64dec }}`
+`{{ b64dec  .foo }}`
 ```
 
-This will indent every line of text by 4 space characters and add a new line to the beginning.
-
-## repeat
+### bytes
 
-Use this function to repeat a string multiple times.
+Convert a humanized byte string to bytes using go-humanize. Durations can be turned into strings such as "3 days ago", numbers
+representing sizes like 82854982 into useful strings like, "83 MB" or
+"79 MiB"
 
-Signature: `repeat(c int,value string) string`
+Signature: `bytes(string) string`
 
-Example:
+Examples:
 
 ```template
-`{{ repeat 3 "hello" }}` // output: hellohellohello
+`{{ .foo | bytes }}`
+`{{ bytes .foo }}`
 ```
 
-## contains
+### default
 
-Use this function to test to see if one string is contained inside of another.
+Enables outputting a default value if the source string is otherwise empty. If the 'src' parameter is not empty, this function returns the value of 'src'. Useful for JSON fields that can be missing, like HTTP headers in a log line that aren't required, as in the following example:
 
-Signature: `contains(s string, src string,) bool`
+```logql
+{job="access_log"} | json | line_format `{{.http_request_headers_x_forwarded_for | default "-"}}`
+```
+
+Signature: `default(d string, src string) string`
 
 Examples:
 
 ```template
-`{{ if contains "ErrTimeout" .err }} timeout {{end}}`
-`{{ if contains "he" "hello" }} yes {{end}}`
+`{{ default "-" "" }}` // output: -
+`{{ default "-" "foo" }}` // output: foo
 ```
 
-## eq
+Example of a query to print a `-` if the `http_request_headers_x_forwarded_for` label is empty:
 
-Use this function to test to see if one string has exact matching inside of another.
+```logql
+{job="access_log"} | json | line_format `{{.http_request_headers_x_forwarded_for | default "-"}}`
+```
 
-Signature: `eq(s string, src string) bool`
+### fromJson
 
-Examples:
+Decodes a JSON document into a structure. If the input cannot be decoded as JSON the function will return an empty string.
+
+Signature: `fromJson(v string) interface{}`
+
+Example:
 
 ```template
-`{{ if eq "ErrTimeout" .err }} timeout {{end}}`
-`{{ if eq "hello" "hello" }} yes {{end}}`
+`{{fromJson "{\"foo\": 55}"}}`
 ```
 
-## hasPrefix and hasSuffix
+Example of a query to print a newline per queries stored as a json array in the log line:
 
-The `hasPrefix` and `hasSuffix` functions test whether a string has a given prefix or suffix.
+```logql
+{job="loki/querier"} |= "finish in prometheus" | logfmt | line_format `{{ range $q := fromJson .queries }} {{ $q.query }} {{ end }}`
+```
 
-Signatures:
+### lower
 
-- `hasPrefix(prefix string, src string) bool`
-- `hasSuffix(suffix string, src string) bool`
+Use this function to convert to lower case.
+
+Signature: `lower(string) string`
 
 Examples:
 
 ```template
-`{{ if hasSuffix .err "Timeout" }} timeout {{end}}`
-`{{ if hasPrefix "he" "hello" }} yes {{end}}`
+`{{ .request_method | lower }}`
+`{{ lower  "HELLO"}}`
 ```
 
-## add
+The last example will return `hello`.
+
+### indent
 
-Sum numbers. Supports multiple numbers
+The indent function indents every line in a given string to the specified indent width. This is useful when aligning multi-line strings.
 
-Signature: `func(i ...interface{}) int64`
+Signature: `indent(spaces int,src string) string`
 
 Example:
 
 ```template
-`{{ add 3 2 5 }}` // output: 10
+`{{ indent 4 .query }}`
 ```
 
-## sub
+This indents each line contained in the `.query` by four (4) spaces.
+
+### nindent
 
-Subtract numbers.
+The nindent function is the same as the indent function, but prepends a new line to the beginning of the string.
 
-Signature: `func(a, b interface{}) int64`
+Signature: `nindent(spaces int,src string) string`
 
 Example:
 
 ```template
-`{{ sub 5 2 }}` // output: 3
+`{{ nindent 4 .query }}`
 ```
 
-## mul
+This will indent every line of text by 4 space characters and add a new line to the beginning.
 
-Multiply numbers. Supports multiple numbers.
+### repeat
 
-Signature: `func(a interface{}, v ...interface{}) int64`
+Use this function to repeat a string multiple times.
+
+Signature: `repeat(c int,value string) string`
 
 Example:
 
 ```template
-`{{ mul 5 2 3}}` // output: 30
+`{{ repeat 3 "hello" }}` // output: hellohellohello
 ```
 
-## div
+### printf
 
-Integer divide numbers.
+Use this function to format a string in a custom way. For more information about the syntax, refer to the [Go documentation](https://pkg.go.dev/fmt).
 
-Signature: `func(a, b interface{}) int64`
+Signature: `printf(format string, a ...interface{})`
 
-Example:
+Examples:
 
 ```template
-`{{ div 10 2}}` // output: 5
-```
-
-## addf
-
-Sum numbers. Supports multiple numbers.
+`{{ printf "The IP address was %s" .remote_addr }}` // output: The IP address was 129.168.1.1
 
-Signature: `func(i ...interface{}) float64`
-
-Example:
+`{{ printf "%-40.40s" .request_uri}} {{printf "%-5.5s" .request_method}}`
+// output: 
+// /a/509965767/alternative-to-my-mtg.html  GET
+// /id/609259548/hpr.html                   GET
+```
 
 ```template
-`{{ addf 3.5 2 5 }}` // output: 10.5
+line_format "\"|\" {{printf \"%15.15s\" .ClientHost}} \"|\""
 ```
 
-## subf
+### replace
 
-Subtract numbers. Supports multiple numbers.
+This function performs simple string replacement.
 
-Signature: `func(a interface{}, v ...interface{}) float64`
+Signature: `replace(old string, new string, src string) string`
 
-Example:
+It takes three arguments:
+
+- `old` string to replace
+- `new` string to replace with
+- `src` source string
+
+Examples:
 
 ```template
-`{{ subf  5.5 2 1.5 }}` // output: 2
+`{{ .cluster | replace "-cluster" "" }}`
+`{{ replace "hello" "world" "hello world" }}`
 ```
 
-## mulf
+The last example will return `world world`.
 
-Multiply numbers. Supports multiple numbers
+### substr
 
-Signature: `func(a interface{}, v ...interface{}) float64`
+Get a substring from a string.
 
-Example:
+Signature: `substr(start int,end int,value string) string`
+
+If start is < 0, this calls value[:end].
+If start is >= 0 and end < 0 or end bigger than s length, this calls value[start:]
+Otherwise, this calls value[start, end].
+
+Examples:
 
 ```template
-`{{ mulf 5.5 2 2.5 }}` // output: 27.5
+`{{ .path | substr 2 5 }}`
+`{{ substr 0 5 "hello world"}}`  // output: hello
+`{{ substr 6 11 "hello world"}}` // output: world
 ```
 
-## divf
+### title
 
-Divide numbers. Supports multiple numbers.
+Convert to title case.
 
-Signature: `func(a interface{}, v ...interface{}) float64`
+Signature: `title(string) string`
 
-Example:
+Examples:
 
 ```template
-`{{ divf 10 2 4}}` // output: 1.25
+`{{.request_method | title}}`
+`{{ title "hello world"}}`
 ```
 
-## mod
+The last example will return `Hello World`.
 
-Modulo wit mod.
+### trim
 
-Signature: `func(a, b interface{}) int64`
+The trim function removes space from either side of a string.
 
-Example:
+Signature: `trim(string) string`
+
+Examples:
 
 ```template
-`{{ mod 10 3}}` // output: 1
+`{{ .ip | trim }}`
+`{{ trim "   hello    " }}` // output: hello
 ```
 
-## max
+### trimAll
 
-Return the largest of a series of integers:
+Use this function to remove given characters from the front or back of a string.
 
-Signature: `max(a interface{}, i ...interface{}) int64`
+Signature: `trimAll(chars string,src string) string`
 
-Example:
+Examples:
 
 ```template
-`{{ max 1 2 3 }}` //output 3
+`{{ .path | trimAll "/" }}`
+`{{ trimAll "$" "$5.00" }}` // output: 5.00
 ```
 
-## min
+### trimPrefix
 
-Return the smallest of a series of integers.
+Use this function to trim just the prefix from a string.
 
-Signature: `min(a interface{}, i ...interface{}) int64`
+Signature: `trimPrefix(prefix string, src string) string`
 
-Example:
+Examples:
 
 ```template
-`{{ min 1 2 3 }}`//output 1
+`{{  .path | trimPrefix "/" }}`
+`{{ trimPrefix "-" "-hello" }}` // output: hello
 ```
 
-## maxf
+### trimSuffix
 
-Return the largest of a series of floats:
+Use this function to trim just the suffix from a string.
 
-Signature: `maxf(a interface{}, i ...interface{}) float64`
+Signature: `trimSuffix(suffix string, src string) string`
 
-Example:
+Examples:
 
 ```template
-`{{ maxf 1 2.5 3 }}` //output 3
+`{{  .path | trimSuffix "/" }}`
+`{{ trimSuffix "-" "hello-" }}` // output: hello
 ```
 
-## minf
+### trunc
 
-Return the smallest of a series of floats.
+Truncate a string and add no suffix.
 
-Signature: `minf(a interface{}, i ...interface{}) float64`
+Signature: `trunc(count int,value string) string`
 
-Example:
+Examples:
 
 ```template
-`{{ minf 1 2.5 3 }}` //output 1.5
+`{{ .path | trunc 2 }}`
+`{{ trunc 5 "hello world"}}`   // output: hello
+`{{ trunc -5 "hello world"}}`  // output: world
 ```
 
-## ceil
+### upper
 
-Returns the greatest float value greater than or equal to input value
+Use this function to convert to upper case.
 
-Signature: `ceil(a interface{}) float64`
+Signature: `upper(string) string`
 
-Example:
+Examples:
 
 ```template
-`{{ ceil 123.001 }}` //output 124.0
+`{ .request_method | upper }}`
+`{{ upper  "hello"}}`
 ```
 
-## floor
+This results in `HELLO`.
+
+### urlencode
 
-Returns the greatest float value less than or equal to input value
+Use this function to [urlencode](https://en.wikipedia.org/wiki/URL_encoding) a string.
 
-Signature: `floor(a interface{}) float64`
+Signature: `urlencode(string) string`
 
-Example:
+Examples:
 
 ```template
-`{{ floor 123.9999 }}` //output 123.0
+`{{ .request_url | urlencode }}`
+`{{ urlencode  .request_url}}`
 ```
 
-## round
+### urldecode
 
-Returns a float value with the remainder rounded to the given number of digits after the decimal point.
+Use this function to [urldecode](https://en.wikipedia.org/wiki/URL_encoding) a string.
 
-Signature: `round(a interface{}, p int, rOpt ...float64) float64`
+Signature: `urldecode(string) string`
 
-Example:
+Examples:
 
 ```template
-`{{ round 123.555555 3 }}` //output 123.556
+`{{ .request_url | urldecode }}`
+`{{ urldecode  .request_url}}`
 ```
 
-We can also provide a `roundOn` number as third parameter
+## Logical functions
 
-Example:
+You can use the following logical functions to compare strings when building a template expression.
+
+### contains
+
+Use this function to test to see if one string is contained inside of another.
+
+Signature: `contains(s string, src string,) bool`
+
+Examples:
 
 ```template
-`{{ round 123.88571428571 5 .2 }}` //output 123.88572
+`{{ if contains "ErrTimeout" .err }} timeout {{end}}`
+`{{ if contains "he" "hello" }} yes {{end}}`
 ```
 
-With default `roundOn` of `.5` the above value would be `123.88571`
-
-## int
+### eq
 
-Convert value to an int.
+Use this function to test to see if one string has exact matching inside of another.
 
-Signature: `toInt(v interface{}) int`
+Signature: `eq(s string, src string) bool`
 
-Example:
+Examples:
 
 ```template
-`{{ "3" | int }}` //output 3
+`{{ if eq "ErrTimeout" .err }} timeout {{end}}`
+`{{ if eq "hello" "hello" }} yes {{end}}`
 ```
 
-## float64
+### hasPrefix and hasSuffix
 
-Convert to a float64.
+The `hasPrefix` and `hasSuffix` functions test whether a string has a given prefix or suffix.
 
-Signature: `toFloat64(v interface{}) float64`
+Signatures:
 
-Example:
+- `hasPrefix(prefix string, src string) bool`
+- `hasSuffix(suffix string, src string) bool`
+
+Examples:
 
 ```template
-`{{ "3.5" | float64 }}` //output 3.5
+`{{ if hasSuffix .err "Timeout" }} timeout {{end}}`
+`{{ if hasPrefix "he" "hello" }} yes {{end}}`
 ```
 
-## fromJson
+## Mathematical functions
 
-Decodes a JSON document into a structure. If the input cannot be decoded as JSON the function will return an empty string.
+You can use the following mathematical functions when writing template expressions.
 
-Signature: `fromJson(v string) interface{}`
+### add
+
+Sum numbers. Supports multiple numbers
+
+Signature: `func(i ...interface{}) int64`
 
 Example:
 
 ```template
-`{{fromJson "{\"foo\": 55}"}}`
+`{{ add 3 2 5 }}` // output: 10
 ```
 
-Example of a query to print a newline per queries stored as a json array in the log line:
+### addf
 
-```logql
-{job="loki/querier"} |= "finish in prometheus" | logfmt | line_format `{{ range $q := fromJson .queries }} {{ $q.query }} {{ end }}`
+Sum floating point numbers. Supports multiple numbers.
+
+Signature: `func(i ...interface{}) float64`
+
+Example:
+
+```template
+`{{ addf 3.5 2 5 }}` // output: 10.5
 ```
 
-## now
+### ceil
 
-Returns the current time in the local timezone of the Loki server.
+Returns the greatest float value greater than or equal to input value
 
-Signature: `Now() time.Time`
+Signature: `ceil(a interface{}) float64`
 
 Example:
 
 ```template
-`{{ now }}`
+`{{ ceil 123.001 }}` //output 124.0
 ```
 
-## toDate
+### div
 
-Parses a formatted string and returns the time value it represents using the local timezone of the server running Loki.
+Divide two integers.
 
-For more consistency between Loki installations, it's recommended to use `toDateInZone`
-
-The format string must use the exact date as defined in the [golang datetime layout](https://pkg.go.dev/time#pkg-constants)
-
-Signature: `toDate(fmt, str string) time.Time`
+Signature: `func(a, b interface{}) int64`
 
-Examples: 
+Example:
 
 ```template
-`{{ toDate "2006-01-02" "2021-11-02" }}`
-`{{ .foo | toDate "2006-01-02T15:04:05.999999999Z" }}`
+`{{ div 10 2}}` // output: 5
 ```
 
-## toDateInZone
+### divf
 
-Parses a formatted string and returns the time value it represents in the provided timezone.
-
-The format string must use the exact date as defined in the [golang datetime layout](https://pkg.go.dev/time#pkg-constants)
+Divide floating point numbers. Supports multiple numbers.
 
-The timezone value can be `Local`, `UTC`, or any of the IANA Time Zone database values
-
-Signature: `toDateInZone(fmt, zone, str string) time.Time`
+Signature: `func(a interface{}, v ...interface{}) float64`
 
-Examples:
+Example:
 
 ```template
-`{{ toDateInZone "2006-01-02" "UTC" "2021-11-02" }}`
-`{{ .foo | toDateInZone "2006-01-02T15:04:05.999999999Z" "UTC" }}`
+`{{ divf 10 2 4}}` // output: 1.25
 ```
 
-## date
+### float64
 
-Returns a textual representation of the time value formatted according to the provided [golang datetime layout](https://pkg.go.dev/time#pkg-constants).
+Convert a string to a float64.
 
-Signature: `date(fmt string, date interface{}) string`
+Signature: `toFloat64(v interface{}) float64`
 
 Example:
 
 ```template
-`{{ date "2006-01-02" now }}`
+`{{ "3.5" | float64 }}` //output 3.5
 ```
 
-## unixEpoch
+### floor
 
-Returns the number of seconds elapsed since January 1, 1970 UTC.
+Returns the greatest float value less than or equal to input value.
 
-Signature: `unixEpoch(date time.Time) string`
+Signature: `floor(a interface{}) float64`
 
-Examples:
+Example:
 
 ```template
-`{{ unixEpoch now }}`
-`{{ .foo | toDateInZone "2006-01-02T15:04:05.999999999Z" "UTC" | unixEpoch }}`
+`{{ floor 123.9999 }}` //output 123.0
 ```
 
-Example of a query to filter Loki querier jobs which create time is 1 day before:
-```logql
-{job="loki/querier"} | label_format nowEpoch=`{{(unixEpoch now)}}`,createDateEpoch=`{{unixEpoch (toDate "2006-01-02" .createDate)}}` | label_format dateTimeDiff=`{{sub .nowEpoch .createDateEpoch}}` | dateTimeDiff > 86400
-```
+### int
 
-## unixEpochMillis
+Convert value to an integer.
 
-Returns the number of milliseconds elapsed since January 1, 1970 UTC.
-
-Signature: `unixEpochMillis(date time.Time) string`
+Signature: `toInt(v interface{}) int`
 
-Examples:
+Example:
 
 ```template
-`{{ unixEpochMillis now }}`
-`{{ .foo | toDateInZone "2006-01-02T15:04:05.999999999Z" "UTC" | unixEpochMillis }}`
+`{{ "3" | int }}` //output 3
 ```
 
-## unixEpochNanos
+### max
 
-Returns the number of nanoseconds elapsed since January 1, 1970 UTC.
+Return the largest of a series of integers:
 
-Signature: `unixEpochNanos(date time.Time) string`
+Signature: `max(a interface{}, i ...interface{}) int64`
 
-Examples:
+Example:
 
 ```template
-`{{ unixEpochNanos now }}`
-`{{ .foo | toDateInZone "2006-01-02T15:04:05.999999999Z" "UTC" | unixEpochNanos }}`
+`{{ max 1 2 3 }}` //output 3
 ```
 
-## unixToTime
+### maxf
 
-Converts the string epoch to the time value it represents. Epoch times in days, seconds, milliseconds, microseconds and nanoseconds are supported.
+Return the largest of a series of floats:
 
-Signature: `unixToTime(epoch string) time.Time`
+Signature: `maxf(a interface{}, i ...interface{}) float64`
 
-Examples:
+Example:
 
-Consider the following log line `{"from": "1679577215","to":"1679587215","message":"some message"}`. To print the `from` field as human readable add the following at the end of the LogQL query:
-```logql
-... | json | line_format `from="{{date "2006-01-02" (unixToTime .from)}}"`
+```template
+`{{ maxf 1 2.5 3 }}` //output 3
 ```
 
-## default
+### min
 
-Checks whether the string(`src`) is set, and returns default(`d`) if not set.
+Return the smallest of a series of integers.
 
-Signature: `default(d string, src string) string`
+Signature: `min(a interface{}, i ...interface{}) int64`
 
-Examples:
+Example:
 
 ```template
-`{{ default "-" "" }}` // output: -
-`{{ default "-" "foo" }}` // output: foo
+`{{ min 1 2 3 }}`//output 1
 ```
 
-Example of a query to print a `-` if the `http_request_headers_x_forwarded_for` label is empty:
-```logql
-{job="access_log"} | json | line_format `{{.http_request_headers_x_forwarded_for | default "-"}}`
+### minf
+
+Return the smallest of a series of floats.
+
+Signature: `minf(a interface{}, i ...interface{}) float64`
+
+Example:
+
+```template
+`{{ minf 1 2.5 3 }}` //output 1.5
 ```
 
-## count
+### mul
 
-Counts occurrences of the regex (`regex`) in (`src`).
+Multiply numbers. Supports multiple numbers.
 
-Signature: `count(regex string, src string) int`
+Signature: `mul(a interface{}, v ...interface{}) int64`
 
-Examples:
+Example:
 
 ```template
-`{{ count "a|b" "abab" }}` // output: 4
-`{{ count "o" "foo" }}`    // output: 2
+`{{ mul 5 2 3}}` // output: 30
 ```
 
-Example of a query to print how many times XYZ occurs in a line:
-```logql
-{job="xyzlog"} | line_format `{{ __line__ | count "XYZ"}}`
+### mulf
+
+Multiply floating numbers. Supports multiple numbers
+
+Signature: `mulf(a interface{}, v ...interface{}) float64`
+
+Example:
+
+```template
+`{{ mulf 5.5 2 2.5 }}` // output: 27.5
 ```
 
-## urlencode
+### mod
 
-Use this function to [urlencode](https://en.wikipedia.org/wiki/URL_encoding) a string.
+Returns the remainder when number 'a' is divided by number 'b'.
 
-Signature: `urlencode(string) string`
+Signature: `mod(a, b interface{}) int64`
 
-Examples:
+Example:
 
 ```template
-`{{ .request_url | urlencode }}`
-`{{ urlencode  .request_url}}`
+`{{ mod 10 3}}` // output: 1
 ```
 
-## urldecode
+### round
 
-Use this function to [urldecode](https://en.wikipedia.org/wiki/URL_encoding) a string.
+Returns a float value with the remainder rounded to the given number of digits after the decimal point.
 
-Signature: `urldecode(string) string`
+Signature: `round(a interface{}, p int, rOpt ...float64) float64`
 
-Examples:
+Example:
 
 ```template
-`{{ .request_url | urldecode }}`
-`{{ urldecode  .request_url}}`
+`{{ round 123.555555 3 }}` //output 123.556
 ```
 
-## b64enc
+We can also provide a `roundOn` number as third parameter
 
-Base64 encode a string.
+Example:
 
-Signature: `b64enc(string) string`
+```template
+`{{ round 123.88571428571 5 .2 }}` //output 123.88572
+```
 
-Examples:
+With default `roundOn` of `.5` the above value would be `123.88571`
+
+### sub
+
+Subtract one number from another.
+
+Signature: `func(a, b interface{}) int64`
+
+Example:
 
 ```template
-`{{ .foo | b64enc }}`
-`{{ b64enc  .foo }}`
+`{{ sub 5 2 }}` // output: 3
 ```
 
-## b64dec
+### subf
 
-Base64 decode a string.
+Subtract floating numbers. Supports multiple numbers.
 
-Signature: `b64dec(string) string`
+Signature: `func(a interface{}, v ...interface{}) float64`
 
-Examples:
+Example:
 
 ```template
-`{{ .foo | b64dec }}`
-`{{ b64dec  .foo }}`
+`{{ subf  5.5 2 1.5 }}` // output: 2
 ```
 
-## bytes
+## Regular expression functions
 
-Convert a humanized byte string to bytes using [go-humanize](https://pkg.go.dev/github.com/dustin/go-humanize#ParseBytes)
+You can use the following functions to perform regular expressions in a template expression.
 
-Signature: `bytes(string) string`
+### count
+
+Counts occurrences of the regex (`regex`) in (`src`).
+
+Signature: `count(regex string, src string) int`
 
 Examples:
 
 ```template
-`{{ .foo | bytes }}`
-`{{ bytes .foo }}`
+`{{ count "a|b" "abab" }}` // output: 4
+`{{ count "o" "foo" }}`    // output: 2
 ```
 
-## duration
+Example of a query to print how many times XYZ occurs in a line:
 
-An alias for `duration_seconds`
+```logql
+{job="xyzlog"} | line_format `{{ __line__ | count "XYZ"}}`
+```
 
-Examples:
+### regexReplaceAll and regexReplaceAllLiteral
+
+`regexReplaceAll` returns a copy of the input string, replacing matches of the Regexp with the replacement string replacement. Inside string replacement, $ signs are interpreted as in Expand, so for instance $1 represents the text of the first sub-match. See the golang [Regexp.replaceAll documentation](https://golang.org/pkg/regexp/#Regexp.ReplaceAll) for more examples.
+
+Signature: regexReplaceAll(regex string, src string, replacement string)
+(source)
+
+Example:
 
 ```template
-`{{ .foo | duration }}`
-`{{ duration .foo }}`
+`{{ regexReplaceAll "(a*)bc" .some_label "${1}a" }}`
 ```
 
-## duration_seconds
-
-Convert a humanized time duration to seconds using [time.ParseDuration](https://pkg.go.dev/time#ParseDuration)
+`regexReplaceAllLiteral` function returns a copy of the input string and replaces matches of the Regexp with the replacement string replacement. The replacement string is substituted directly, without using Expand.
 
-Signature: `duration_seconds(string) float64`
+Signature: regexReplaceAllLiteral(regex string, src string, replacement string)
 
-Examples:
+Example:
 
 ```template
-`{{ .foo | duration_seconds }}`
-`{{ duration_seconds .foo }}`
+`{{ regexReplaceAllLiteral "(ts=)" .timestamp "timestamp=" }}`
 ```
diff --git a/docs/sources/setup/install/helm/reference.md b/docs/sources/setup/install/helm/reference.md
index 9add6158f3a6a..b74a559b12e7f 100644
--- a/docs/sources/setup/install/helm/reference.md
+++ b/docs/sources/setup/install/helm/reference.md
@@ -3039,6 +3039,7 @@ null
   },
   "provisioner": {
     "additionalTenants": [],
+    "affinity": {},
     "annotations": {},
     "enabled": true,
     "env": [],
@@ -3051,6 +3052,7 @@ null
       "tag": null
     },
     "labels": {},
+    "nodeSelector": {},
     "priorityClassName": null,
     "provisionedSecretPrefix": null,
     "securityContext": {
@@ -3058,9 +3060,11 @@ null
       "runAsGroup": 10001,
       "runAsNonRoot": true,
       "runAsUser": 10001
-    }
+    },
+    "tolerations": []
   },
   "tokengen": {
+    "affinity": {},
     "annotations": {},
     "enabled": true,
     "env": [],
@@ -3069,6 +3073,7 @@ null
     "extraVolumeMounts": [],
     "extraVolumes": [],
     "labels": {},
+    "nodeSelector": {},
     "priorityClassName": "",
     "securityContext": {
       "fsGroup": 10001,
@@ -3222,6 +3227,7 @@ null
 			<td><pre lang="json">
 {
   "additionalTenants": [],
+  "affinity": {},
   "annotations": {},
   "enabled": true,
   "env": [],
@@ -3234,6 +3240,7 @@ null
     "tag": null
   },
   "labels": {},
+  "nodeSelector": {},
   "priorityClassName": null,
   "provisionedSecretPrefix": null,
   "securityContext": {
@@ -3241,7 +3248,8 @@ null
     "runAsGroup": 10001,
     "runAsNonRoot": true,
     "runAsUser": 10001
-  }
+  },
+  "tolerations": []
 }
 </pre>
 </td>
@@ -3253,6 +3261,15 @@ null
 			<td><pre lang="json">
 []
 </pre>
+</td>
+		</tr>
+		<tr>
+			<td>enterprise.provisioner.affinity</td>
+			<td>object</td>
+			<td>Affinity for tokengen Pods</td>
+			<td><pre lang="json">
+{}
+</pre>
 </td>
 		</tr>
 		<tr>
@@ -3358,6 +3375,15 @@ null
 			<td><pre lang="json">
 {}
 </pre>
+</td>
+		</tr>
+		<tr>
+			<td>enterprise.provisioner.nodeSelector</td>
+			<td>object</td>
+			<td>Node selector for tokengen Pods</td>
+			<td><pre lang="json">
+{}
+</pre>
 </td>
 		</tr>
 		<tr>
@@ -3390,6 +3416,15 @@ null
   "runAsUser": 10001
 }
 </pre>
+</td>
+		</tr>
+		<tr>
+			<td>enterprise.provisioner.tolerations</td>
+			<td>list</td>
+			<td>Tolerations for tokengen Pods</td>
+			<td><pre lang="json">
+[]
+</pre>
 </td>
 		</tr>
 		<tr>
@@ -3398,6 +3433,7 @@ null
 			<td>Configuration for `tokengen` target</td>
 			<td><pre lang="json">
 {
+  "affinity": {},
   "annotations": {},
   "enabled": true,
   "env": [],
@@ -3406,6 +3442,7 @@ null
   "extraVolumeMounts": [],
   "extraVolumes": [],
   "labels": {},
+  "nodeSelector": {},
   "priorityClassName": "",
   "securityContext": {
     "fsGroup": 10001,
@@ -3417,6 +3454,15 @@ null
   "tolerations": []
 }
 </pre>
+</td>
+		</tr>
+		<tr>
+			<td>enterprise.tokengen.affinity</td>
+			<td>object</td>
+			<td>Affinity for tokengen Pods</td>
+			<td><pre lang="json">
+{}
+</pre>
 </td>
 		</tr>
 		<tr>
@@ -3489,6 +3535,15 @@ true
 			<td><pre lang="json">
 {}
 </pre>
+</td>
+		</tr>
+		<tr>
+			<td>enterprise.tokengen.nodeSelector</td>
+			<td>object</td>
+			<td>Node selector for tokengen Pods</td>
+			<td><pre lang="json">
+{}
+</pre>
 </td>
 		</tr>
 		<tr>
@@ -5907,7 +5962,7 @@ null
 		<tr>
 			<td>loki.image.tag</td>
 			<td>string</td>
-			<td>Overrides the image tag whose default is the chart's appVersion TODO: needed for 3rd target backend functionality revert to null or latest once this behavior is relased</td>
+			<td>Overrides the image tag whose default is the chart's appVersion</td>
 			<td><pre lang="json">
 null
 </pre>
diff --git a/go.mod b/go.mod
index 3981ee97bf44a..62887962ff38b 100644
--- a/go.mod
+++ b/go.mod
@@ -17,7 +17,7 @@ require (
 	github.com/alicebob/miniredis/v2 v2.30.4
 	github.com/aliyun/aliyun-oss-go-sdk v2.2.10+incompatible
 	github.com/aws/aws-sdk-go v1.54.19
-	github.com/baidubce/bce-sdk-go v0.9.189
+	github.com/baidubce/bce-sdk-go v0.9.192
 	github.com/bmatcuk/doublestar v1.3.4
 	github.com/c2h5oh/datasize v0.0.0-20231215233829-aa82cc1e6500
 	github.com/cespare/xxhash v1.1.0
@@ -32,7 +32,7 @@ require (
 	github.com/dustin/go-humanize v1.0.1
 	github.com/facette/natsort v0.0.0-20181210072756-2cd4dd1e2dcb
 	github.com/fatih/color v1.16.0
-	github.com/felixge/fgprof v0.9.4
+	github.com/felixge/fgprof v0.9.5
 	github.com/fluent/fluent-bit-go v0.0.0-20230731091245-a7a013e2473c
 	github.com/fsouza/fake-gcs-server v1.7.0
 	github.com/go-kit/log v0.2.1
@@ -49,7 +49,7 @@ require (
 	github.com/gorilla/mux v1.8.1
 	github.com/gorilla/websocket v1.5.3
 	github.com/grafana/cloudflare-go v0.0.0-20230110200409-c627cf6792f2
-	github.com/grafana/dskit v0.0.0-20240925193654-7c41a4057319
+	github.com/grafana/dskit v0.0.0-20240930165212-f52de24af9bc
 	github.com/grafana/go-gelf/v2 v2.0.1
 	github.com/grafana/gomemcache v0.0.0-20240229205252-cd6a66d6fb56
 	github.com/grafana/regexp v0.0.0-20240518133315-a468a5bfb3bc
@@ -96,10 +96,10 @@ require (
 	go.etcd.io/bbolt v1.3.10
 	go.uber.org/atomic v1.11.0
 	go.uber.org/goleak v1.3.0
-	golang.org/x/crypto v0.26.0
-	golang.org/x/net v0.28.0
+	golang.org/x/crypto v0.27.0
+	golang.org/x/net v0.29.0
 	golang.org/x/sync v0.8.0
-	golang.org/x/sys v0.24.0
+	golang.org/x/sys v0.25.0
 	golang.org/x/time v0.6.0
 	google.golang.org/api v0.193.0
 	google.golang.org/grpc v1.65.0
@@ -114,7 +114,7 @@ require (
 	github.com/DataDog/sketches-go v1.4.6
 	github.com/DmitriyVTitov/size v1.5.0
 	github.com/IBM/go-sdk-core/v5 v5.17.5
-	github.com/IBM/ibm-cos-sdk-go v1.11.0
+	github.com/IBM/ibm-cos-sdk-go v1.11.1
 	github.com/axiomhq/hyperloglog v0.0.0-20240507144631-af9851f82b27
 	github.com/buger/jsonparser v1.1.1
 	github.com/coder/quartz v0.1.0
@@ -149,7 +149,7 @@ require (
 	go4.org/netipx v0.0.0-20230125063823-8449b0a6169f
 	golang.org/x/exp v0.0.0-20240325151524-a685a6edb6d8
 	golang.org/x/oauth2 v0.22.0
-	golang.org/x/text v0.17.0
+	golang.org/x/text v0.18.0
 	google.golang.org/protobuf v1.34.2
 	gotest.tools v2.2.0+incompatible
 	k8s.io/apimachinery v0.29.3
@@ -356,7 +356,7 @@ require (
 	go.uber.org/multierr v1.11.0 // indirect
 	go.uber.org/zap v1.21.0 // indirect
 	golang.org/x/mod v0.19.0 // indirect
-	golang.org/x/term v0.23.0 // indirect
+	golang.org/x/term v0.24.0 // indirect
 	golang.org/x/tools v0.23.0 // indirect
 	google.golang.org/genproto v0.0.0-20240820151423-278611b39280 // indirect
 	google.golang.org/genproto/googleapis/api v0.0.0-20240820151423-278611b39280 // indirect
diff --git a/go.sum b/go.sum
index fabf4aa4fdca5..7fbf5705ba189 100644
--- a/go.sum
+++ b/go.sum
@@ -252,8 +252,8 @@ github.com/HdrHistogram/hdrhistogram-go v1.1.2 h1:5IcZpTvzydCQeHzK4Ef/D5rrSqwxob
 github.com/HdrHistogram/hdrhistogram-go v1.1.2/go.mod h1:yDgFjdqOqDEKOvasDdhWNXYg9BVp4O+o5f6V/ehm6Oo=
 github.com/IBM/go-sdk-core/v5 v5.17.5 h1:AjGC7xNee5tgDIjndekBDW5AbypdERHSgib3EZ1KNsA=
 github.com/IBM/go-sdk-core/v5 v5.17.5/go.mod h1:KsAAI7eStAWwQa4F96MLy+whYSh39JzNjklZRbN/8ns=
-github.com/IBM/ibm-cos-sdk-go v1.11.0 h1:Jp55NLN3OvBwucMGpP5wNybyjncsmTZ9+GPHai/1cE8=
-github.com/IBM/ibm-cos-sdk-go v1.11.0/go.mod h1:FnWOym0CvrPM0nHoXvceClOEvGVXecPpmVIO5RFjlFk=
+github.com/IBM/ibm-cos-sdk-go v1.11.1 h1:Pye61hmWA4ZVCfOfFLTJBjPka4HIGrLqmpZ2d2KlrCE=
+github.com/IBM/ibm-cos-sdk-go v1.11.1/go.mod h1:d8vET3w8wgmGwCsCVs+0y4V8+1hRNT6+pbpGaEHvSCI=
 github.com/Knetic/govaluate v3.0.1-0.20171022003610-9aa49832a739+incompatible/go.mod h1:r7JcOSlj0wfOMncg0iLm8Leh48TZaKVeNIfJntJ2wa0=
 github.com/MasslessParticle/azure-storage-blob-go v0.14.1-0.20240322194317-344980fda573 h1:DCPjdUAi+jcGnL7iN+A7uNY8xG584oMRuisYh/VE21E=
 github.com/MasslessParticle/azure-storage-blob-go v0.14.1-0.20240322194317-344980fda573/go.mod h1:SMqIBi+SuiQH32bvyjngEewEeXoPfKMgWlBDaYf6fck=
@@ -383,8 +383,8 @@ github.com/aws/smithy-go v1.11.1 h1:IQ+lPZVkSM3FRtyaDox41R8YS6iwPMYIreejOgPW49g=
 github.com/aws/smithy-go v1.11.1/go.mod h1:3xHYmszWVx2c0kIwQeEVf9uSm4fYZt67FBJnwub1bgM=
 github.com/axiomhq/hyperloglog v0.0.0-20240507144631-af9851f82b27 h1:60m4tnanN1ctzIu4V3bfCNJ39BiOPSm1gHFlFjTkRE0=
 github.com/axiomhq/hyperloglog v0.0.0-20240507144631-af9851f82b27/go.mod h1:k08r+Yj1PRAmuayFiRK6MYuR5Ve4IuZtTfxErMIh0+c=
-github.com/baidubce/bce-sdk-go v0.9.189 h1:Dz0EAdM9wgbGHY4jeXFyQea8KeOdrRlbqwCbDLudcYw=
-github.com/baidubce/bce-sdk-go v0.9.189/go.mod h1:zbYJMQwE4IZuyrJiFO8tO8NbtYiKTFTbwh4eIsqjVdg=
+github.com/baidubce/bce-sdk-go v0.9.192 h1:cnSFxCiwAZ2ZMEpxeDi2DVvrzq65gRH4TakSoWQJeC4=
+github.com/baidubce/bce-sdk-go v0.9.192/go.mod h1:zbYJMQwE4IZuyrJiFO8tO8NbtYiKTFTbwh4eIsqjVdg=
 github.com/baiyubin/aliyun-sts-go-sdk v0.0.0-20180326062324-cfa1a18b161f/go.mod h1:AuiFmCCPBSrqvVMvuqFuk0qogytodnVFVSN5CeJB8Gc=
 github.com/bboreham/go-loser v0.0.0-20230920113527-fcc2c21820a3 h1:6df1vn4bBlDDo4tARvBm7l6KA9iVMnE3NWizDeWSrps=
 github.com/bboreham/go-loser v0.0.0-20230920113527-fcc2c21820a3/go.mod h1:CIWtjkly68+yqLPbvwwR/fjNJA/idrtULjZWh2v1ys0=
@@ -622,8 +622,8 @@ github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYF
 github.com/fatih/color v1.16.0 h1:zmkK9Ngbjj+K0yRhTVONQh1p/HknKYSlNT+vZCzyokM=
 github.com/fatih/color v1.16.0/go.mod h1:fL2Sau1YI5c0pdGEVCbKQbLXB6edEj1ZgiY4NijnWvE=
 github.com/fatih/structs v1.1.0/go.mod h1:9NiDSp5zOcgEDl+j00MP/WkGVPOlPRLejGD8Ga6PJ7M=
-github.com/felixge/fgprof v0.9.4 h1:ocDNwMFlnA0NU0zSB3I52xkO4sFXk80VK9lXjLClu88=
-github.com/felixge/fgprof v0.9.4/go.mod h1:yKl+ERSa++RYOs32d8K6WEXCB4uXdLls4ZaZPpayhMM=
+github.com/felixge/fgprof v0.9.5 h1:8+vR6yu2vvSKn08urWyEuxx75NWPEvybbkBirEpsbVY=
+github.com/felixge/fgprof v0.9.5/go.mod h1:yKl+ERSa++RYOs32d8K6WEXCB4uXdLls4ZaZPpayhMM=
 github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg=
 github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U=
 github.com/fluent/fluent-bit-go v0.0.0-20230731091245-a7a013e2473c h1:yKN46XJHYC/gvgH2UsisJ31+n4K3S7QYZSfU2uAWjuI=
@@ -1042,8 +1042,8 @@ github.com/gorilla/websocket v1.5.3 h1:saDtZ6Pbx/0u+bgYQ3q96pZgCzfhKXGPqt7kZ72aN
 github.com/gorilla/websocket v1.5.3/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
 github.com/grafana/cloudflare-go v0.0.0-20230110200409-c627cf6792f2 h1:qhugDMdQ4Vp68H0tp/0iN17DM2ehRo1rLEdOFe/gB8I=
 github.com/grafana/cloudflare-go v0.0.0-20230110200409-c627cf6792f2/go.mod h1:w/aiO1POVIeXUQyl0VQSZjl5OAGDTL5aX+4v0RA1tcw=
-github.com/grafana/dskit v0.0.0-20240925193654-7c41a4057319 h1:KACpOOTqA4WqyyKF2fFPQFiaSOpZdOT5f5gg0qkPLiU=
-github.com/grafana/dskit v0.0.0-20240925193654-7c41a4057319/go.mod h1:SPLNCARd4xdjCkue0O6hvuoveuS1dGJjDnfxYe405YQ=
+github.com/grafana/dskit v0.0.0-20240930165212-f52de24af9bc h1:OLRT3mpHjvTjq4Km7L36lipZ+hTOX1U3jct0DFMADdo=
+github.com/grafana/dskit v0.0.0-20240930165212-f52de24af9bc/go.mod h1:SPLNCARd4xdjCkue0O6hvuoveuS1dGJjDnfxYe405YQ=
 github.com/grafana/go-gelf/v2 v2.0.1 h1:BOChP0h/jLeD+7F9mL7tq10xVkDG15he3T1zHuQaWak=
 github.com/grafana/go-gelf/v2 v2.0.1/go.mod h1:lexHie0xzYGwCgiRGcvZ723bSNyNI8ZRD4s0CLobh90=
 github.com/grafana/gocql v0.0.0-20200605141915-ba5dc39ece85 h1:xLuzPoOzdfNb/RF/IENCw+oLVdZB4G21VPhkHBgwSHY=
@@ -2012,8 +2012,8 @@ golang.org/x/crypto v0.0.0-20220829220503-c86fa9a7ed90/go.mod h1:IxCIyHEi3zRg3s0
 golang.org/x/crypto v0.3.0/go.mod h1:hebNnKkNXi2UzZN1eVRvBB7co0a+JxK6XbPiWVs/3J4=
 golang.org/x/crypto v0.6.0/go.mod h1:OFC/31mSvZgRz0V1QTNCzfAI1aIRzbiufJtkMIlEp58=
 golang.org/x/crypto v0.17.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq4=
-golang.org/x/crypto v0.26.0 h1:RrRspgV4mU+YwB4FYnuBoKsUapNIL5cohGAmSH3azsw=
-golang.org/x/crypto v0.26.0/go.mod h1:GY7jblb9wI+FOo5y8/S2oY4zWP07AkOJ4+jxCqdqn54=
+golang.org/x/crypto v0.27.0 h1:GXm2NjJrPaiv/h1tb2UH8QfgC/hOf/+z0p6PT8o1w7A=
+golang.org/x/crypto v0.27.0/go.mod h1:1Xngt8kV6Dvbssa53Ziq6Eqn0HqbZi5Z6R0ZpwQzt70=
 golang.org/x/exp v0.0.0-20180321215751-8460e604b9de/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20180807140117-3d87b88a115f/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
@@ -2141,8 +2141,8 @@ golang.org/x/net v0.2.0/go.mod h1:KqCZLdyyvdV855qA2rE3GC2aiw5xGR5TEjj8smXukLY=
 golang.org/x/net v0.4.0/go.mod h1:MBQ8lrhLObU/6UmLb4fmbmk5OcyYmqtbGd/9yIeKjEE=
 golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
 golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg=
-golang.org/x/net v0.28.0 h1:a9JDOJc5GMUJ0+UDqmLT86WiEy7iWyIhz8gz8E4e5hE=
-golang.org/x/net v0.28.0/go.mod h1:yqtgsTWOOnlGLG9GFRrK3++bGOUEkNBoHZc8MEDWPNg=
+golang.org/x/net v0.29.0 h1:5ORfpBpCs4HzDYoodCDBbwHzdR5UrLBZ3sOnUJmFoHo=
+golang.org/x/net v0.29.0/go.mod h1:gLkgy8jTGERgjzMic6DS9+SP0ajcu6Xu3Orq/SpETg0=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
 golang.org/x/oauth2 v0.0.0-20190402181905-9f3314589c9a/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
@@ -2312,8 +2312,8 @@ golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
 golang.org/x/sys v0.22.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
-golang.org/x/sys v0.24.0 h1:Twjiwq9dn6R1fQcyiK+wQyHWfaz/BJB+YIpzU/Cv3Xg=
-golang.org/x/sys v0.24.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/sys v0.25.0 h1:r+8e+loiHxRqhXVl6ML1nO3l1+oFoWbnlu2Ehimmi34=
+golang.org/x/sys v0.25.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
 golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
@@ -2323,8 +2323,8 @@ golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=
 golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo=
 golang.org/x/term v0.15.0/go.mod h1:BDl952bC7+uMoWR75FIrCDx79TPU9oHkTZ9yRbYOrX0=
 golang.org/x/term v0.22.0/go.mod h1:F3qCibpT5AMpCRfhfT53vVJwhLtIVHhB9XDjfFvnMI4=
-golang.org/x/term v0.23.0 h1:F6D4vR+EHoL9/sWAWgAR1H2DcHr4PareCbAaCo1RpuU=
-golang.org/x/term v0.23.0/go.mod h1:DgV24QBUrK6jhZXl+20l6UWznPlwAHm1Q1mGHtydmSk=
+golang.org/x/term v0.24.0 h1:Mh5cbb+Zk2hqqXNO7S1iTjEphVL+jb8ZWaqh/g+JWkM=
+golang.org/x/term v0.24.0/go.mod h1:lOBK/LVxemqiMij05LGJ0tzNr8xlmwBRJ81PX6wVLH8=
 golang.org/x/text v0.0.0-20160726164857-2910a502d2bf/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
@@ -2342,8 +2342,8 @@ golang.org/x/text v0.5.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
 golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
 golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
 golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
-golang.org/x/text v0.17.0 h1:XtiM5bkSOt+ewxlOE/aE/AKEHibwj/6gvWMl9Rsh0Qc=
-golang.org/x/text v0.17.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY=
+golang.org/x/text v0.18.0 h1:XvMDiNzPAl0jr17s6W9lcaIhGUfUORdGCNsuLmPG224=
+golang.org/x/text v0.18.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY=
 golang.org/x/time v0.0.0-20161028155119-f51c12702a4d/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
diff --git a/operator/bundle/community-openshift/manifests/loki-operator.clusterserviceversion.yaml b/operator/bundle/community-openshift/manifests/loki-operator.clusterserviceversion.yaml
index 6b3cff962a8a7..e088428b959ce 100644
--- a/operator/bundle/community-openshift/manifests/loki-operator.clusterserviceversion.yaml
+++ b/operator/bundle/community-openshift/manifests/loki-operator.clusterserviceversion.yaml
@@ -150,7 +150,7 @@ metadata:
     categories: OpenShift Optional, Logging & Tracing
     certified: "false"
     containerImage: docker.io/grafana/loki-operator:0.6.2
-    createdAt: "2024-09-25T09:56:42Z"
+    createdAt: "2024-10-01T11:39:45Z"
     description: The Community Loki Operator provides Kubernetes native deployment
       and management of Loki and related logging components.
     features.operators.openshift.io/disconnected: "true"
@@ -1846,7 +1846,7 @@ spec:
                 - /manager
                 env:
                 - name: RELATED_IMAGE_LOKI
-                  value: docker.io/grafana/loki:3.1.0
+                  value: docker.io/grafana/loki:3.2.0
                 - name: RELATED_IMAGE_GATEWAY
                   value: quay.io/observatorium/api:latest
                 - name: RELATED_IMAGE_OPA
@@ -1970,7 +1970,7 @@ spec:
   provider:
     name: Grafana Loki SIG Operator
   relatedImages:
-  - image: docker.io/grafana/loki:3.1.0
+  - image: docker.io/grafana/loki:3.2.0
     name: loki
   - image: quay.io/observatorium/api:latest
     name: gateway
diff --git a/operator/bundle/community/manifests/loki-operator.clusterserviceversion.yaml b/operator/bundle/community/manifests/loki-operator.clusterserviceversion.yaml
index a52fb9b070cf8..f8137b4c9f3d4 100644
--- a/operator/bundle/community/manifests/loki-operator.clusterserviceversion.yaml
+++ b/operator/bundle/community/manifests/loki-operator.clusterserviceversion.yaml
@@ -150,7 +150,7 @@ metadata:
     categories: OpenShift Optional, Logging & Tracing
     certified: "false"
     containerImage: docker.io/grafana/loki-operator:0.6.2
-    createdAt: "2024-09-25T09:56:40Z"
+    createdAt: "2024-10-01T11:39:42Z"
     description: The Community Loki Operator provides Kubernetes native deployment
       and management of Loki and related logging components.
     operators.operatorframework.io/builder: operator-sdk-unknown
@@ -1826,7 +1826,7 @@ spec:
                 - /manager
                 env:
                 - name: RELATED_IMAGE_LOKI
-                  value: docker.io/grafana/loki:3.1.1
+                  value: docker.io/grafana/loki:3.2.0
                 - name: RELATED_IMAGE_GATEWAY
                   value: quay.io/observatorium/api:latest
                 - name: RELATED_IMAGE_OPA
@@ -1938,7 +1938,7 @@ spec:
   provider:
     name: Grafana Loki SIG Operator
   relatedImages:
-  - image: docker.io/grafana/loki:3.1.1
+  - image: docker.io/grafana/loki:3.2.0
     name: loki
   - image: quay.io/observatorium/api:latest
     name: gateway
diff --git a/operator/bundle/openshift/manifests/loki-operator.clusterserviceversion.yaml b/operator/bundle/openshift/manifests/loki-operator.clusterserviceversion.yaml
index d5bd203a90b88..d57ae2d5af902 100644
--- a/operator/bundle/openshift/manifests/loki-operator.clusterserviceversion.yaml
+++ b/operator/bundle/openshift/manifests/loki-operator.clusterserviceversion.yaml
@@ -150,7 +150,7 @@ metadata:
     categories: OpenShift Optional, Logging & Tracing
     certified: "false"
     containerImage: quay.io/openshift-logging/loki-operator:0.1.0
-    createdAt: "2024-09-25T09:56:44Z"
+    createdAt: "2024-10-01T11:39:47Z"
     description: |
       The Loki Operator for OCP provides a means for configuring and managing a Loki stack for cluster logging.
       ## Prerequisites and Requirements
@@ -161,7 +161,7 @@ metadata:
       set of OCP nodes may not be large enough to support the Loki stack.  Additional OCP nodes must be added
       to the OCP cluster if you desire to run with the recommended (or better) memory.
     features.operators.openshift.io/disconnected: "true"
-    features.operators.openshift.io/fips-compliant: "false"
+    features.operators.openshift.io/fips-compliant: "true"
     features.operators.openshift.io/proxy-aware: "true"
     features.operators.openshift.io/tls-profiles: "true"
     features.operators.openshift.io/token-auth-aws: "true"
@@ -1831,7 +1831,7 @@ spec:
                 - /manager
                 env:
                 - name: RELATED_IMAGE_LOKI
-                  value: quay.io/openshift-logging/loki:v3.1.1
+                  value: quay.io/openshift-logging/loki:v3.2.0
                 - name: RELATED_IMAGE_GATEWAY
                   value: quay.io/observatorium/api:latest
                 - name: RELATED_IMAGE_OPA
@@ -1955,7 +1955,7 @@ spec:
   provider:
     name: Red Hat
   relatedImages:
-  - image: quay.io/openshift-logging/loki:v3.1.1
+  - image: quay.io/openshift-logging/loki:v3.2.0
     name: loki
   - image: quay.io/observatorium/api:latest
     name: gateway
diff --git a/operator/config/manifests/openshift/bases/loki-operator.clusterserviceversion.yaml b/operator/config/manifests/openshift/bases/loki-operator.clusterserviceversion.yaml
index 9361198824dac..fd18cadf96e71 100644
--- a/operator/config/manifests/openshift/bases/loki-operator.clusterserviceversion.yaml
+++ b/operator/config/manifests/openshift/bases/loki-operator.clusterserviceversion.yaml
@@ -17,7 +17,7 @@ metadata:
       set of OCP nodes may not be large enough to support the Loki stack.  Additional OCP nodes must be added
       to the OCP cluster if you desire to run with the recommended (or better) memory.
     features.operators.openshift.io/disconnected: "true"
-    features.operators.openshift.io/fips-compliant: "false"
+    features.operators.openshift.io/fips-compliant: "true"
     features.operators.openshift.io/proxy-aware: "true"
     features.operators.openshift.io/tls-profiles: "true"
     features.operators.openshift.io/token-auth-aws: "true"
diff --git a/operator/config/overlays/community-openshift/manager_related_image_patch.yaml b/operator/config/overlays/community-openshift/manager_related_image_patch.yaml
index 5e454e400cc87..538e6658383b3 100644
--- a/operator/config/overlays/community-openshift/manager_related_image_patch.yaml
+++ b/operator/config/overlays/community-openshift/manager_related_image_patch.yaml
@@ -9,7 +9,7 @@ spec:
         - name: manager
           env:
           - name: RELATED_IMAGE_LOKI
-            value: docker.io/grafana/loki:3.1.0
+            value: docker.io/grafana/loki:3.2.0
           - name: RELATED_IMAGE_GATEWAY
             value: quay.io/observatorium/api:latest
           - name: RELATED_IMAGE_OPA
diff --git a/operator/config/overlays/community/manager_related_image_patch.yaml b/operator/config/overlays/community/manager_related_image_patch.yaml
index 52681b95a4b1c..538e6658383b3 100644
--- a/operator/config/overlays/community/manager_related_image_patch.yaml
+++ b/operator/config/overlays/community/manager_related_image_patch.yaml
@@ -9,7 +9,7 @@ spec:
         - name: manager
           env:
           - name: RELATED_IMAGE_LOKI
-            value: docker.io/grafana/loki:3.1.1
+            value: docker.io/grafana/loki:3.2.0
           - name: RELATED_IMAGE_GATEWAY
             value: quay.io/observatorium/api:latest
           - name: RELATED_IMAGE_OPA
diff --git a/operator/config/overlays/development/manager_related_image_patch.yaml b/operator/config/overlays/development/manager_related_image_patch.yaml
index 21aa79f99a7d7..e4af25f50e449 100644
--- a/operator/config/overlays/development/manager_related_image_patch.yaml
+++ b/operator/config/overlays/development/manager_related_image_patch.yaml
@@ -9,6 +9,6 @@ spec:
         - name: manager
           env:
           - name: RELATED_IMAGE_LOKI
-            value: docker.io/grafana/loki:3.1.1
+            value: docker.io/grafana/loki:3.2.0
           - name: RELATED_IMAGE_GATEWAY
             value: quay.io/observatorium/api:latest
diff --git a/operator/config/overlays/openshift/manager_related_image_patch.yaml b/operator/config/overlays/openshift/manager_related_image_patch.yaml
index 3bb2b65cf4617..f7ae10c2aaf3c 100644
--- a/operator/config/overlays/openshift/manager_related_image_patch.yaml
+++ b/operator/config/overlays/openshift/manager_related_image_patch.yaml
@@ -9,7 +9,7 @@ spec:
         - name: manager
           env:
           - name: RELATED_IMAGE_LOKI
-            value: quay.io/openshift-logging/loki:v3.1.1
+            value: quay.io/openshift-logging/loki:v3.2.0
           - name: RELATED_IMAGE_GATEWAY
             value: quay.io/observatorium/api:latest
           - name: RELATED_IMAGE_OPA
diff --git a/operator/docs/operator/compatibility.md b/operator/docs/operator/compatibility.md
index 382e7e6a7673f..5f573dc89906e 100644
--- a/operator/docs/operator/compatibility.md
+++ b/operator/docs/operator/compatibility.md
@@ -29,3 +29,4 @@ The versions of Loki compatible to be run with the Loki Operator are:
 
 * v3.1.0
 * v3.1.1
+* v3.2.0
diff --git a/operator/hack/addons_dev.yaml b/operator/hack/addons_dev.yaml
index 320743bc37325..9c67cfc65b6f1 100644
--- a/operator/hack/addons_dev.yaml
+++ b/operator/hack/addons_dev.yaml
@@ -29,7 +29,7 @@ spec:
     spec:
       containers:
         - name: logcli
-          image: docker.io/grafana/logcli:3.1.1-amd64
+          image: docker.io/grafana/logcli:3.2.0-amd64
           imagePullPolicy: IfNotPresent
           command:
             - /bin/sh
@@ -73,7 +73,7 @@ spec:
     spec:
       containers:
         - name: promtail
-          image: docker.io/grafana/promtail:3.1.1
+          image: docker.io/grafana/promtail:3.2.0
           args:
             - -config.file=/etc/promtail/promtail.yaml
             - -log.level=info
diff --git a/operator/hack/addons_ocp.yaml b/operator/hack/addons_ocp.yaml
index 6e49cbd847c8a..0a00d83044624 100644
--- a/operator/hack/addons_ocp.yaml
+++ b/operator/hack/addons_ocp.yaml
@@ -29,7 +29,7 @@ spec:
     spec:
       containers:
         - name: logcli
-          image: docker.io/grafana/logcli:3.1.1-amd64
+          image: docker.io/grafana/logcli:3.2.0-amd64
           imagePullPolicy: IfNotPresent
           command:
             - /bin/sh
@@ -70,7 +70,7 @@ spec:
     spec:
       containers:
         - name: promtail
-          image: docker.io/grafana/promtail:3.1.1
+          image: docker.io/grafana/promtail:3.2.0
           args:
             - -config.file=/etc/promtail/promtail.yaml
             - -log.level=info
diff --git a/operator/internal/manifests/openshift/alertingrule.go b/operator/internal/manifests/openshift/alertingrule.go
index 22923ed15482e..e4869e9f7ca5c 100644
--- a/operator/internal/manifests/openshift/alertingrule.go
+++ b/operator/internal/manifests/openshift/alertingrule.go
@@ -5,23 +5,32 @@ import lokiv1 "github.com/grafana/loki/operator/apis/loki/v1"
 func AlertingRuleTenantLabels(ar *lokiv1.AlertingRule) {
 	switch ar.Spec.TenantID {
 	case tenantApplication:
-		for groupIdx, group := range ar.Spec.Groups {
-			group := group
-			for ruleIdx, rule := range group.Rules {
-				rule := rule
-				if rule.Labels == nil {
-					rule.Labels = map[string]string{}
-				}
-				rule.Labels[opaDefaultLabelMatcher] = ar.Namespace
-				group.Rules[ruleIdx] = rule
-			}
-			ar.Spec.Groups[groupIdx] = group
-		}
-	case tenantInfrastructure, tenantAudit:
-		// Do nothing
-	case tenantNetwork:
-		// Do nothing
+		appendAlertingRuleLabels(ar, map[string]string{
+			opaDefaultLabelMatcher:    ar.Namespace,
+			ocpMonitoringGroupByLabel: ar.Namespace,
+		})
+	case tenantInfrastructure, tenantAudit, tenantNetwork:
+		appendAlertingRuleLabels(ar, map[string]string{
+			ocpMonitoringGroupByLabel: ar.Namespace,
+		})
 	default:
 		// Do nothing
 	}
 }
+
+func appendAlertingRuleLabels(ar *lokiv1.AlertingRule, labels map[string]string) {
+	for groupIdx, group := range ar.Spec.Groups {
+		for ruleIdx, rule := range group.Rules {
+			if rule.Labels == nil {
+				rule.Labels = map[string]string{}
+			}
+
+			for name, value := range labels {
+				rule.Labels[name] = value
+			}
+
+			group.Rules[ruleIdx] = rule
+		}
+		ar.Spec.Groups[groupIdx] = group
+	}
+}
diff --git a/operator/internal/manifests/openshift/alertingrule_test.go b/operator/internal/manifests/openshift/alertingrule_test.go
index 91da560e2a6df..2a1d032e8ed47 100644
--- a/operator/internal/manifests/openshift/alertingrule_test.go
+++ b/operator/internal/manifests/openshift/alertingrule_test.go
@@ -46,7 +46,8 @@ func TestAlertingRuleTenantLabels(t *testing.T) {
 								{
 									Alert: "alert",
 									Labels: map[string]string{
-										opaDefaultLabelMatcher: "test-ns",
+										opaDefaultLabelMatcher:    "test-ns",
+										ocpMonitoringGroupByLabel: "test-ns",
 									},
 								},
 							},
@@ -57,6 +58,9 @@ func TestAlertingRuleTenantLabels(t *testing.T) {
 		},
 		{
 			rule: &lokiv1.AlertingRule{
+				ObjectMeta: metav1.ObjectMeta{
+					Namespace: "test-ns",
+				},
 				Spec: lokiv1.AlertingRuleSpec{
 					TenantID: tenantInfrastructure,
 					Groups: []*lokiv1.AlertingRuleGroup{
@@ -72,6 +76,9 @@ func TestAlertingRuleTenantLabels(t *testing.T) {
 				},
 			},
 			want: &lokiv1.AlertingRule{
+				ObjectMeta: metav1.ObjectMeta{
+					Namespace: "test-ns",
+				},
 				Spec: lokiv1.AlertingRuleSpec{
 					TenantID: tenantInfrastructure,
 					Groups: []*lokiv1.AlertingRuleGroup{
@@ -80,6 +87,9 @@ func TestAlertingRuleTenantLabels(t *testing.T) {
 							Rules: []*lokiv1.AlertingRuleGroupSpec{
 								{
 									Alert: "alert",
+									Labels: map[string]string{
+										ocpMonitoringGroupByLabel: "test-ns",
+									},
 								},
 							},
 						},
@@ -89,6 +99,9 @@ func TestAlertingRuleTenantLabels(t *testing.T) {
 		},
 		{
 			rule: &lokiv1.AlertingRule{
+				ObjectMeta: metav1.ObjectMeta{
+					Namespace: "test-ns",
+				},
 				Spec: lokiv1.AlertingRuleSpec{
 					TenantID: tenantAudit,
 					Groups: []*lokiv1.AlertingRuleGroup{
@@ -104,6 +117,9 @@ func TestAlertingRuleTenantLabels(t *testing.T) {
 				},
 			},
 			want: &lokiv1.AlertingRule{
+				ObjectMeta: metav1.ObjectMeta{
+					Namespace: "test-ns",
+				},
 				Spec: lokiv1.AlertingRuleSpec{
 					TenantID: tenantAudit,
 					Groups: []*lokiv1.AlertingRuleGroup{
@@ -112,6 +128,9 @@ func TestAlertingRuleTenantLabels(t *testing.T) {
 							Rules: []*lokiv1.AlertingRuleGroupSpec{
 								{
 									Alert: "alert",
+									Labels: map[string]string{
+										ocpMonitoringGroupByLabel: "test-ns",
+									},
 								},
 							},
 						},
@@ -121,6 +140,9 @@ func TestAlertingRuleTenantLabels(t *testing.T) {
 		},
 		{
 			rule: &lokiv1.AlertingRule{
+				ObjectMeta: metav1.ObjectMeta{
+					Namespace: "test-ns",
+				},
 				Spec: lokiv1.AlertingRuleSpec{
 					TenantID: tenantNetwork,
 					Groups: []*lokiv1.AlertingRuleGroup{
@@ -136,6 +158,9 @@ func TestAlertingRuleTenantLabels(t *testing.T) {
 				},
 			},
 			want: &lokiv1.AlertingRule{
+				ObjectMeta: metav1.ObjectMeta{
+					Namespace: "test-ns",
+				},
 				Spec: lokiv1.AlertingRuleSpec{
 					TenantID: tenantNetwork,
 					Groups: []*lokiv1.AlertingRuleGroup{
@@ -144,6 +169,9 @@ func TestAlertingRuleTenantLabels(t *testing.T) {
 							Rules: []*lokiv1.AlertingRuleGroupSpec{
 								{
 									Alert: "alert",
+									Labels: map[string]string{
+										ocpMonitoringGroupByLabel: "test-ns",
+									},
 								},
 							},
 						},
@@ -153,6 +181,9 @@ func TestAlertingRuleTenantLabels(t *testing.T) {
 		},
 		{
 			rule: &lokiv1.AlertingRule{
+				ObjectMeta: metav1.ObjectMeta{
+					Namespace: "test-ns",
+				},
 				Spec: lokiv1.AlertingRuleSpec{
 					TenantID: "unknown",
 					Groups: []*lokiv1.AlertingRuleGroup{
@@ -168,6 +199,9 @@ func TestAlertingRuleTenantLabels(t *testing.T) {
 				},
 			},
 			want: &lokiv1.AlertingRule{
+				ObjectMeta: metav1.ObjectMeta{
+					Namespace: "test-ns",
+				},
 				Spec: lokiv1.AlertingRuleSpec{
 					TenantID: "unknown",
 					Groups: []*lokiv1.AlertingRuleGroup{
diff --git a/operator/internal/manifests/openshift/internal/dashboards/static/grafana-dashboard-lokistack-reads.json b/operator/internal/manifests/openshift/internal/dashboards/static/grafana-dashboard-lokistack-reads.json
index ca4cca0f11058..ec1656e27bd7d 100644
--- a/operator/internal/manifests/openshift/internal/dashboards/static/grafana-dashboard-lokistack-reads.json
+++ b/operator/internal/manifests/openshift/internal/dashboards/static/grafana-dashboard-lokistack-reads.json
@@ -213,7 +213,7 @@
                "stack": true,
                "targets": [
                   {
-                     "expr": "sum by (status) (\n  label_replace(label_replace(rate(loki_request_duration_seconds_count{namespace=\"$namespace\",job=~\".+-query-frontend-http\", route=~\"(api_prom_rules|api_prom_rules_namespace_groupname|api_v1_rules|loki_api_v1_delete|loki_api_v1_detected_labels|loki_api_v1_index_stats|loki_api_v1_index_volume|loki_api_v1_index_volume_range|loki_api_v1_label_name_values|loki_api_v1_label_values|loki_api_v1_labels|loki_api_v1_patterns|loki_api_v1_query|loki_api_v1_query_range|loki_api_v1_series|otlp_v1_logs|prometheus_api_v1_rules)\"}[$__rate_interval]),\n  \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n  \"status\", \"${1}\", \"status_code\", \"([a-z]+)\"))\n",
+                     "expr": "sum by (status) (\n  label_replace(label_replace(rate(loki_request_duration_seconds_count{namespace=\"$namespace\",job=~\".+-query-frontend-http\", route=~\"(api_prom_rules|api_prom_rules_namespace_groupname|api_v1_rules|loki_api_v1_delete|loki_api_v1_detected_labels|loki_api_v1_index_stats|loki_api_v1_index_volume|loki_api_v1_index_volume_range|loki_api_v1_label_name_values|loki_api_v1_label_values|loki_api_v1_labels|loki_api_v1_patterns|loki_api_v1_query|loki_api_v1_query_range|loki_api_v1_series|prometheus_api_v1_rules)\"}[$__rate_interval]),\n  \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n  \"status\", \"${1}\", \"status_code\", \"([a-z]+)\"))\n",
                      "format": "time_series",
                      "intervalFactor": 2,
                      "legendFormat": "{{status}}",
@@ -263,7 +263,7 @@
                "span": 4,
                "targets": [
                   {
-                     "expr": "histogram_quantile(0.99, sum by (le,route) (namespace_job_route:loki_request_duration_seconds_bucket:sum_rate{namespace=\"$namespace\", job=~\".+-query-frontend-http\", route=~\"(api_prom_rules|api_prom_rules_namespace_groupname|api_v1_rules|loki_api_v1_delete|loki_api_v1_detected_labels|loki_api_v1_index_stats|loki_api_v1_index_volume|loki_api_v1_index_volume_range|loki_api_v1_label_name_values|loki_api_v1_label_values|loki_api_v1_labels|loki_api_v1_patterns|loki_api_v1_query|loki_api_v1_query_range|loki_api_v1_series|otlp_v1_logs|prometheus_api_v1_rules)\"})) * 1e3",
+                     "expr": "histogram_quantile(0.99, sum by (le,route) (namespace_job_route:loki_request_duration_seconds_bucket:sum_rate{namespace=\"$namespace\", job=~\".+-query-frontend-http\", route=~\"(api_prom_rules|api_prom_rules_namespace_groupname|api_v1_rules|loki_api_v1_delete|loki_api_v1_detected_labels|loki_api_v1_index_stats|loki_api_v1_index_volume|loki_api_v1_index_volume_range|loki_api_v1_label_name_values|loki_api_v1_label_values|loki_api_v1_labels|loki_api_v1_patterns|loki_api_v1_query|loki_api_v1_query_range|loki_api_v1_series|prometheus_api_v1_rules)\"})) * 1e3",
                      "format": "time_series",
                      "intervalFactor": 2,
                      "legendFormat": "{{ route }} 99th Percentile",
@@ -271,7 +271,7 @@
                      "step": 10
                   },
                   {
-                     "expr": "histogram_quantile(0.50, sum by (le,route) (namespace_job_route:loki_request_duration_seconds_bucket:sum_rate{namespace=\"$namespace\", job=~\".+-query-frontend-http\", route=~\"(api_prom_rules|api_prom_rules_namespace_groupname|api_v1_rules|loki_api_v1_delete|loki_api_v1_detected_labels|loki_api_v1_index_stats|loki_api_v1_index_volume|loki_api_v1_index_volume_range|loki_api_v1_label_name_values|loki_api_v1_label_values|loki_api_v1_labels|loki_api_v1_patterns|loki_api_v1_query|loki_api_v1_query_range|loki_api_v1_series|otlp_v1_logs|prometheus_api_v1_rules)\"})) * 1e3",
+                     "expr": "histogram_quantile(0.50, sum by (le,route) (namespace_job_route:loki_request_duration_seconds_bucket:sum_rate{namespace=\"$namespace\", job=~\".+-query-frontend-http\", route=~\"(api_prom_rules|api_prom_rules_namespace_groupname|api_v1_rules|loki_api_v1_delete|loki_api_v1_detected_labels|loki_api_v1_index_stats|loki_api_v1_index_volume|loki_api_v1_index_volume_range|loki_api_v1_label_name_values|loki_api_v1_label_values|loki_api_v1_labels|loki_api_v1_patterns|loki_api_v1_query|loki_api_v1_query_range|loki_api_v1_series|prometheus_api_v1_rules)\"})) * 1e3",
                      "format": "time_series",
                      "intervalFactor": 2,
                      "legendFormat": "{{ route }} 50th Percentile",
@@ -279,7 +279,7 @@
                      "step": 10
                   },
                   {
-                     "expr": "1e3 * sum(namespace_job_route:loki_request_duration_seconds_sum:sum_rate{namespace=\"$namespace\", job=~\".+-query-frontend-http\", route=~\"(api_prom_rules|api_prom_rules_namespace_groupname|api_v1_rules|loki_api_v1_delete|loki_api_v1_detected_labels|loki_api_v1_index_stats|loki_api_v1_index_volume|loki_api_v1_index_volume_range|loki_api_v1_label_name_values|loki_api_v1_label_values|loki_api_v1_labels|loki_api_v1_patterns|loki_api_v1_query|loki_api_v1_query_range|loki_api_v1_series|otlp_v1_logs|prometheus_api_v1_rules)\"}) by (route)  / sum(namespace_job_route:loki_request_duration_seconds_count:sum_rate{namespace=\"$namespace\", job=~\".+-query-frontend-http\", route=~\"(api_prom_rules|api_prom_rules_namespace_groupname|api_v1_rules|loki_api_v1_delete|loki_api_v1_detected_labels|loki_api_v1_index_stats|loki_api_v1_index_volume|loki_api_v1_index_volume_range|loki_api_v1_label_name_values|loki_api_v1_label_values|loki_api_v1_labels|loki_api_v1_patterns|loki_api_v1_query|loki_api_v1_query_range|loki_api_v1_series|otlp_v1_logs|prometheus_api_v1_rules)\"}) by (route) ",
+                     "expr": "1e3 * sum(namespace_job_route:loki_request_duration_seconds_sum:sum_rate{namespace=\"$namespace\", job=~\".+-query-frontend-http\", route=~\"(api_prom_rules|api_prom_rules_namespace_groupname|api_v1_rules|loki_api_v1_delete|loki_api_v1_detected_labels|loki_api_v1_index_stats|loki_api_v1_index_volume|loki_api_v1_index_volume_range|loki_api_v1_label_name_values|loki_api_v1_label_values|loki_api_v1_labels|loki_api_v1_patterns|loki_api_v1_query|loki_api_v1_query_range|loki_api_v1_series|prometheus_api_v1_rules)\"}) by (route)  / sum(namespace_job_route:loki_request_duration_seconds_count:sum_rate{namespace=\"$namespace\", job=~\".+-query-frontend-http\", route=~\"(api_prom_rules|api_prom_rules_namespace_groupname|api_v1_rules|loki_api_v1_delete|loki_api_v1_detected_labels|loki_api_v1_index_stats|loki_api_v1_index_volume|loki_api_v1_index_volume_range|loki_api_v1_label_name_values|loki_api_v1_label_values|loki_api_v1_labels|loki_api_v1_patterns|loki_api_v1_query|loki_api_v1_query_range|loki_api_v1_series|prometheus_api_v1_rules)\"}) by (route) ",
                      "format": "time_series",
                      "intervalFactor": 2,
                      "legendFormat": "{{ route }} Average",
@@ -347,7 +347,7 @@
                "span": 4,
                "targets": [
                   {
-                     "expr": "histogram_quantile(0.99, sum(rate(loki_request_duration_seconds_bucket{namespace=\"$namespace\", job=~\".+-query-frontend-http\", route=~\"(api_prom_rules|api_prom_rules_namespace_groupname|api_v1_rules|loki_api_v1_delete|loki_api_v1_detected_labels|loki_api_v1_index_stats|loki_api_v1_index_volume|loki_api_v1_index_volume_range|loki_api_v1_label_name_values|loki_api_v1_label_values|loki_api_v1_labels|loki_api_v1_patterns|loki_api_v1_query|loki_api_v1_query_range|loki_api_v1_series|otlp_v1_logs|prometheus_api_v1_rules)\"}[$__rate_interval])) by (le,pod)) * 1e3",
+                     "expr": "histogram_quantile(0.99, sum(rate(loki_request_duration_seconds_bucket{namespace=\"$namespace\", job=~\".+-query-frontend-http\", route=~\"(api_prom_rules|api_prom_rules_namespace_groupname|api_v1_rules|loki_api_v1_delete|loki_api_v1_detected_labels|loki_api_v1_index_stats|loki_api_v1_index_volume|loki_api_v1_index_volume_range|loki_api_v1_label_name_values|loki_api_v1_label_values|loki_api_v1_labels|loki_api_v1_patterns|loki_api_v1_query|loki_api_v1_query_range|loki_api_v1_series|prometheus_api_v1_rules)\"}[$__rate_interval])) by (le,pod)) * 1e3",
                      "format": "time_series",
                      "interval": "1m",
                      "intervalFactor": 2,
@@ -558,7 +558,7 @@
                "stack": true,
                "targets": [
                   {
-                     "expr": "sum by (status) (\n  label_replace(label_replace(rate(loki_request_duration_seconds_count{namespace=\"$namespace\",job=~\".+-querier-http\", route=~\"(api_prom_rules|api_prom_rules_namespace_groupname|api_v1_rules|loki_api_v1_delete|loki_api_v1_detected_labels|loki_api_v1_index_stats|loki_api_v1_index_volume|loki_api_v1_index_volume_range|loki_api_v1_label_name_values|loki_api_v1_label_values|loki_api_v1_labels|loki_api_v1_patterns|loki_api_v1_query|loki_api_v1_query_range|loki_api_v1_series|otlp_v1_logs|prometheus_api_v1_rules)\"}[$__rate_interval]),\n  \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n  \"status\", \"${1}\", \"status_code\", \"([a-z]+)\"))\n",
+                     "expr": "sum by (status) (\n  label_replace(label_replace(rate(loki_request_duration_seconds_count{namespace=\"$namespace\",job=~\".+-querier-http\", route=~\"(api_prom_rules|api_prom_rules_namespace_groupname|api_v1_rules|loki_api_v1_delete|loki_api_v1_detected_labels|loki_api_v1_index_stats|loki_api_v1_index_volume|loki_api_v1_index_volume_range|loki_api_v1_label_name_values|loki_api_v1_label_values|loki_api_v1_labels|loki_api_v1_patterns|loki_api_v1_query|loki_api_v1_query_range|loki_api_v1_series|prometheus_api_v1_rules)\"}[$__rate_interval]),\n  \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n  \"status\", \"${1}\", \"status_code\", \"([a-z]+)\"))\n",
                      "format": "time_series",
                      "intervalFactor": 2,
                      "legendFormat": "{{status}}",
@@ -608,7 +608,7 @@
                "span": 4,
                "targets": [
                   {
-                     "expr": "histogram_quantile(0.99, sum by (le,route) (namespace_job_route:loki_request_duration_seconds_bucket:sum_rate{namespace=\"$namespace\", job=~\".+-querier-http\", route=~\"(api_prom_rules|api_prom_rules_namespace_groupname|api_v1_rules|loki_api_v1_delete|loki_api_v1_detected_labels|loki_api_v1_index_stats|loki_api_v1_index_volume|loki_api_v1_index_volume_range|loki_api_v1_label_name_values|loki_api_v1_label_values|loki_api_v1_labels|loki_api_v1_patterns|loki_api_v1_query|loki_api_v1_query_range|loki_api_v1_series|otlp_v1_logs|prometheus_api_v1_rules)\"})) * 1e3",
+                     "expr": "histogram_quantile(0.99, sum by (le,route) (namespace_job_route:loki_request_duration_seconds_bucket:sum_rate{namespace=\"$namespace\", job=~\".+-querier-http\", route=~\"(api_prom_rules|api_prom_rules_namespace_groupname|api_v1_rules|loki_api_v1_delete|loki_api_v1_detected_labels|loki_api_v1_index_stats|loki_api_v1_index_volume|loki_api_v1_index_volume_range|loki_api_v1_label_name_values|loki_api_v1_label_values|loki_api_v1_labels|loki_api_v1_patterns|loki_api_v1_query|loki_api_v1_query_range|loki_api_v1_series|prometheus_api_v1_rules)\"})) * 1e3",
                      "format": "time_series",
                      "intervalFactor": 2,
                      "legendFormat": "{{ route }} 99th Percentile",
@@ -616,7 +616,7 @@
                      "step": 10
                   },
                   {
-                     "expr": "histogram_quantile(0.50, sum by (le,route) (namespace_job_route:loki_request_duration_seconds_bucket:sum_rate{namespace=\"$namespace\", job=~\".+-querier-http\", route=~\"(api_prom_rules|api_prom_rules_namespace_groupname|api_v1_rules|loki_api_v1_delete|loki_api_v1_detected_labels|loki_api_v1_index_stats|loki_api_v1_index_volume|loki_api_v1_index_volume_range|loki_api_v1_label_name_values|loki_api_v1_label_values|loki_api_v1_labels|loki_api_v1_patterns|loki_api_v1_query|loki_api_v1_query_range|loki_api_v1_series|otlp_v1_logs|prometheus_api_v1_rules)\"})) * 1e3",
+                     "expr": "histogram_quantile(0.50, sum by (le,route) (namespace_job_route:loki_request_duration_seconds_bucket:sum_rate{namespace=\"$namespace\", job=~\".+-querier-http\", route=~\"(api_prom_rules|api_prom_rules_namespace_groupname|api_v1_rules|loki_api_v1_delete|loki_api_v1_detected_labels|loki_api_v1_index_stats|loki_api_v1_index_volume|loki_api_v1_index_volume_range|loki_api_v1_label_name_values|loki_api_v1_label_values|loki_api_v1_labels|loki_api_v1_patterns|loki_api_v1_query|loki_api_v1_query_range|loki_api_v1_series|prometheus_api_v1_rules)\"})) * 1e3",
                      "format": "time_series",
                      "intervalFactor": 2,
                      "legendFormat": "{{ route }} 50th Percentile",
@@ -624,7 +624,7 @@
                      "step": 10
                   },
                   {
-                     "expr": "1e3 * sum(namespace_job_route:loki_request_duration_seconds_sum:sum_rate{namespace=\"$namespace\", job=~\".+-querier-http\", route=~\"(api_prom_rules|api_prom_rules_namespace_groupname|api_v1_rules|loki_api_v1_delete|loki_api_v1_detected_labels|loki_api_v1_index_stats|loki_api_v1_index_volume|loki_api_v1_index_volume_range|loki_api_v1_label_name_values|loki_api_v1_label_values|loki_api_v1_labels|loki_api_v1_patterns|loki_api_v1_query|loki_api_v1_query_range|loki_api_v1_series|otlp_v1_logs|prometheus_api_v1_rules)\"}) by (route)  / sum(namespace_job_route:loki_request_duration_seconds_count:sum_rate{namespace=\"$namespace\", job=~\".+-querier-http\", route=~\"(api_prom_rules|api_prom_rules_namespace_groupname|api_v1_rules|loki_api_v1_delete|loki_api_v1_detected_labels|loki_api_v1_index_stats|loki_api_v1_index_volume|loki_api_v1_index_volume_range|loki_api_v1_label_name_values|loki_api_v1_label_values|loki_api_v1_labels|loki_api_v1_patterns|loki_api_v1_query|loki_api_v1_query_range|loki_api_v1_series|otlp_v1_logs|prometheus_api_v1_rules)\"}) by (route) ",
+                     "expr": "1e3 * sum(namespace_job_route:loki_request_duration_seconds_sum:sum_rate{namespace=\"$namespace\", job=~\".+-querier-http\", route=~\"(api_prom_rules|api_prom_rules_namespace_groupname|api_v1_rules|loki_api_v1_delete|loki_api_v1_detected_labels|loki_api_v1_index_stats|loki_api_v1_index_volume|loki_api_v1_index_volume_range|loki_api_v1_label_name_values|loki_api_v1_label_values|loki_api_v1_labels|loki_api_v1_patterns|loki_api_v1_query|loki_api_v1_query_range|loki_api_v1_series|prometheus_api_v1_rules)\"}) by (route)  / sum(namespace_job_route:loki_request_duration_seconds_count:sum_rate{namespace=\"$namespace\", job=~\".+-querier-http\", route=~\"(api_prom_rules|api_prom_rules_namespace_groupname|api_v1_rules|loki_api_v1_delete|loki_api_v1_detected_labels|loki_api_v1_index_stats|loki_api_v1_index_volume|loki_api_v1_index_volume_range|loki_api_v1_label_name_values|loki_api_v1_label_values|loki_api_v1_labels|loki_api_v1_patterns|loki_api_v1_query|loki_api_v1_query_range|loki_api_v1_series|prometheus_api_v1_rules)\"}) by (route) ",
                      "format": "time_series",
                      "intervalFactor": 2,
                      "legendFormat": "{{ route }} Average",
@@ -692,7 +692,7 @@
                "span": 4,
                "targets": [
                   {
-                     "expr": "histogram_quantile(0.99, sum(rate(loki_request_duration_seconds_bucket{namespace=\"$namespace\", job=~\".+-querier-http\", route=~\"(api_prom_rules|api_prom_rules_namespace_groupname|api_v1_rules|loki_api_v1_delete|loki_api_v1_detected_labels|loki_api_v1_index_stats|loki_api_v1_index_volume|loki_api_v1_index_volume_range|loki_api_v1_label_name_values|loki_api_v1_label_values|loki_api_v1_labels|loki_api_v1_patterns|loki_api_v1_query|loki_api_v1_query_range|loki_api_v1_series|otlp_v1_logs|prometheus_api_v1_rules)\"}[$__rate_interval])) by (le,pod)) * 1e3",
+                     "expr": "histogram_quantile(0.99, sum(rate(loki_request_duration_seconds_bucket{namespace=\"$namespace\", job=~\".+-querier-http\", route=~\"(api_prom_rules|api_prom_rules_namespace_groupname|api_v1_rules|loki_api_v1_delete|loki_api_v1_detected_labels|loki_api_v1_index_stats|loki_api_v1_index_volume|loki_api_v1_index_volume_range|loki_api_v1_label_name_values|loki_api_v1_label_values|loki_api_v1_labels|loki_api_v1_patterns|loki_api_v1_query|loki_api_v1_query_range|loki_api_v1_series|prometheus_api_v1_rules)\"}[$__rate_interval])) by (le,pod)) * 1e3",
                      "format": "time_series",
                      "interval": "1m",
                      "intervalFactor": 2,
@@ -1298,7 +1298,7 @@
                "span": 4,
                "targets": [
                   {
-                     "expr": "histogram_quantile(0.99, sum by (le,route) (route:loki_request_duration_seconds_bucket:sum_rate{route=~\"(/base.Ruler/Rules|/indexgatewaypb.IndexGateway/GetChunkRef|/indexgatewaypb.IndexGateway/GetSeries|/indexgatewaypb.IndexGateway/GetShards|/indexgatewaypb.IndexGateway/GetStats|/indexgatewaypb.IndexGateway/GetVolume|/indexgatewaypb.IndexGateway/LabelNamesForMetricName|/indexgatewaypb.IndexGateway/LabelValuesForMetricName|/indexgatewaypb.IndexGateway/QueryIndex|/logproto.BloomGateway/FilterChunkRefs|/logproto.Pattern/Query|/logproto.Querier/GetChunkIDs|/logproto.Querier/GetDetectedLabels|/logproto.Querier/GetStats|/logproto.Querier/GetVolume|/logproto.Querier/Label|/logproto.Querier/Query|/logproto.Querier/QuerySample|/logproto.Querier/Series|/logproto.StreamData/GetStreamRates)\"})) * 1e3",
+                     "expr": "histogram_quantile(0.99, sum by (le,route) (namespace_job_route:loki_request_duration_seconds_bucket:sum_rate{namespace=\"$namespace\", job=~\".+-index-gateway-http\", route=~\"(/base.Ruler/Rules|/indexgatewaypb.IndexGateway/GetChunkRef|/indexgatewaypb.IndexGateway/GetSeries|/indexgatewaypb.IndexGateway/GetShards|/indexgatewaypb.IndexGateway/GetStats|/indexgatewaypb.IndexGateway/GetVolume|/indexgatewaypb.IndexGateway/LabelNamesForMetricName|/indexgatewaypb.IndexGateway/LabelValuesForMetricName|/indexgatewaypb.IndexGateway/QueryIndex|/logproto.BloomGateway/FilterChunkRefs|/logproto.Pattern/Query|/logproto.Querier/GetChunkIDs|/logproto.Querier/GetDetectedLabels|/logproto.Querier/GetStats|/logproto.Querier/GetVolume|/logproto.Querier/Label|/logproto.Querier/Query|/logproto.Querier/QuerySample|/logproto.Querier/Series|/logproto.StreamData/GetStreamRates)\"})) * 1e3",
                      "format": "time_series",
                      "intervalFactor": 2,
                      "legendFormat": "{{ route }} 99th Percentile",
@@ -1306,7 +1306,7 @@
                      "step": 10
                   },
                   {
-                     "expr": "histogram_quantile(0.50, sum by (le,route) (route:loki_request_duration_seconds_bucket:sum_rate{route=~\"(/base.Ruler/Rules|/indexgatewaypb.IndexGateway/GetChunkRef|/indexgatewaypb.IndexGateway/GetSeries|/indexgatewaypb.IndexGateway/GetShards|/indexgatewaypb.IndexGateway/GetStats|/indexgatewaypb.IndexGateway/GetVolume|/indexgatewaypb.IndexGateway/LabelNamesForMetricName|/indexgatewaypb.IndexGateway/LabelValuesForMetricName|/indexgatewaypb.IndexGateway/QueryIndex|/logproto.BloomGateway/FilterChunkRefs|/logproto.Pattern/Query|/logproto.Querier/GetChunkIDs|/logproto.Querier/GetDetectedLabels|/logproto.Querier/GetStats|/logproto.Querier/GetVolume|/logproto.Querier/Label|/logproto.Querier/Query|/logproto.Querier/QuerySample|/logproto.Querier/Series|/logproto.StreamData/GetStreamRates)\"})) * 1e3",
+                     "expr": "histogram_quantile(0.50, sum by (le,route) (namespace_job_route:loki_request_duration_seconds_bucket:sum_rate{namespace=\"$namespace\", job=~\".+-index-gateway-http\", route=~\"(/base.Ruler/Rules|/indexgatewaypb.IndexGateway/GetChunkRef|/indexgatewaypb.IndexGateway/GetSeries|/indexgatewaypb.IndexGateway/GetShards|/indexgatewaypb.IndexGateway/GetStats|/indexgatewaypb.IndexGateway/GetVolume|/indexgatewaypb.IndexGateway/LabelNamesForMetricName|/indexgatewaypb.IndexGateway/LabelValuesForMetricName|/indexgatewaypb.IndexGateway/QueryIndex|/logproto.BloomGateway/FilterChunkRefs|/logproto.Pattern/Query|/logproto.Querier/GetChunkIDs|/logproto.Querier/GetDetectedLabels|/logproto.Querier/GetStats|/logproto.Querier/GetVolume|/logproto.Querier/Label|/logproto.Querier/Query|/logproto.Querier/QuerySample|/logproto.Querier/Series|/logproto.StreamData/GetStreamRates)\"})) * 1e3",
                      "format": "time_series",
                      "intervalFactor": 2,
                      "legendFormat": "{{ route }} 50th Percentile",
@@ -1314,7 +1314,7 @@
                      "step": 10
                   },
                   {
-                     "expr": "1e3 * sum(route:loki_request_duration_seconds_sum:sum_rate{route=~\"(/base.Ruler/Rules|/indexgatewaypb.IndexGateway/GetChunkRef|/indexgatewaypb.IndexGateway/GetSeries|/indexgatewaypb.IndexGateway/GetShards|/indexgatewaypb.IndexGateway/GetStats|/indexgatewaypb.IndexGateway/GetVolume|/indexgatewaypb.IndexGateway/LabelNamesForMetricName|/indexgatewaypb.IndexGateway/LabelValuesForMetricName|/indexgatewaypb.IndexGateway/QueryIndex|/logproto.BloomGateway/FilterChunkRefs|/logproto.Pattern/Query|/logproto.Querier/GetChunkIDs|/logproto.Querier/GetDetectedLabels|/logproto.Querier/GetStats|/logproto.Querier/GetVolume|/logproto.Querier/Label|/logproto.Querier/Query|/logproto.Querier/QuerySample|/logproto.Querier/Series|/logproto.StreamData/GetStreamRates)\"}) by (route)  / sum(route:loki_request_duration_seconds_count:sum_rate{route=~\"(/base.Ruler/Rules|/indexgatewaypb.IndexGateway/GetChunkRef|/indexgatewaypb.IndexGateway/GetSeries|/indexgatewaypb.IndexGateway/GetShards|/indexgatewaypb.IndexGateway/GetStats|/indexgatewaypb.IndexGateway/GetVolume|/indexgatewaypb.IndexGateway/LabelNamesForMetricName|/indexgatewaypb.IndexGateway/LabelValuesForMetricName|/indexgatewaypb.IndexGateway/QueryIndex|/logproto.BloomGateway/FilterChunkRefs|/logproto.Pattern/Query|/logproto.Querier/GetChunkIDs|/logproto.Querier/GetDetectedLabels|/logproto.Querier/GetStats|/logproto.Querier/GetVolume|/logproto.Querier/Label|/logproto.Querier/Query|/logproto.Querier/QuerySample|/logproto.Querier/Series|/logproto.StreamData/GetStreamRates)\"}) by (route) ",
+                     "expr": "1e3 * sum(namespace_job_route:loki_request_duration_seconds_sum:sum_rate{namespace=\"$namespace\", job=~\".+-index-gateway-http\", route=~\"(/base.Ruler/Rules|/indexgatewaypb.IndexGateway/GetChunkRef|/indexgatewaypb.IndexGateway/GetSeries|/indexgatewaypb.IndexGateway/GetShards|/indexgatewaypb.IndexGateway/GetStats|/indexgatewaypb.IndexGateway/GetVolume|/indexgatewaypb.IndexGateway/LabelNamesForMetricName|/indexgatewaypb.IndexGateway/LabelValuesForMetricName|/indexgatewaypb.IndexGateway/QueryIndex|/logproto.BloomGateway/FilterChunkRefs|/logproto.Pattern/Query|/logproto.Querier/GetChunkIDs|/logproto.Querier/GetDetectedLabels|/logproto.Querier/GetStats|/logproto.Querier/GetVolume|/logproto.Querier/Label|/logproto.Querier/Query|/logproto.Querier/QuerySample|/logproto.Querier/Series|/logproto.StreamData/GetStreamRates)\"}) by (route)  / sum(namespace_job_route:loki_request_duration_seconds_count:sum_rate{namespace=\"$namespace\", job=~\".+-index-gateway-http\", route=~\"(/base.Ruler/Rules|/indexgatewaypb.IndexGateway/GetChunkRef|/indexgatewaypb.IndexGateway/GetSeries|/indexgatewaypb.IndexGateway/GetShards|/indexgatewaypb.IndexGateway/GetStats|/indexgatewaypb.IndexGateway/GetVolume|/indexgatewaypb.IndexGateway/LabelNamesForMetricName|/indexgatewaypb.IndexGateway/LabelValuesForMetricName|/indexgatewaypb.IndexGateway/QueryIndex|/logproto.BloomGateway/FilterChunkRefs|/logproto.Pattern/Query|/logproto.Querier/GetChunkIDs|/logproto.Querier/GetDetectedLabels|/logproto.Querier/GetStats|/logproto.Querier/GetVolume|/logproto.Querier/Label|/logproto.Querier/Query|/logproto.Querier/QuerySample|/logproto.Querier/Series|/logproto.StreamData/GetStreamRates)\"}) by (route) ",
                      "format": "time_series",
                      "intervalFactor": 2,
                      "legendFormat": "{{ route }} Average",
diff --git a/operator/internal/manifests/openshift/internal/dashboards/static/grafana-dashboard-lokistack-writes.json b/operator/internal/manifests/openshift/internal/dashboards/static/grafana-dashboard-lokistack-writes.json
index 334045ddfc11c..50f5f492e8896 100644
--- a/operator/internal/manifests/openshift/internal/dashboards/static/grafana-dashboard-lokistack-writes.json
+++ b/operator/internal/manifests/openshift/internal/dashboards/static/grafana-dashboard-lokistack-writes.json
@@ -213,7 +213,7 @@
                "stack": true,
                "targets": [
                   {
-                     "expr": "sum by (status) (\n  label_replace(label_replace(rate(loki_request_duration_seconds_count{namespace=\"$namespace\",job=~\".+-distributor-http\", route=~\"api_prom_push|loki_api_v1_push|/httpgrpc.HTTP/Handle\"}[$__rate_interval]),\n  \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n  \"status\", \"${1}\", \"status_code\", \"([a-z]+)\"))\n",
+                     "expr": "sum by (status) (\n  label_replace(label_replace(rate(loki_request_duration_seconds_count{namespace=\"$namespace\",job=~\".+-distributor-http\", route=~\"api_prom_push|loki_api_v1_push|otlp_v1_logs|/httpgrpc.HTTP/Handle\"}[$__rate_interval]),\n  \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n  \"status\", \"${1}\", \"status_code\", \"([a-z]+)\"))\n",
                      "format": "time_series",
                      "intervalFactor": 2,
                      "legendFormat": "{{status}}",
@@ -263,7 +263,7 @@
                "span": 4,
                "targets": [
                   {
-                     "expr": "histogram_quantile(0.99, sum by (le) (namespace_job_route:loki_request_duration_seconds_bucket:sum_rate{namespace=\"$namespace\", job=~\".+-distributor-http\", route=~\"api_prom_push|loki_api_v1_push|/httpgrpc.HTTP/Handle\"})) * 1e3",
+                     "expr": "histogram_quantile(0.99, sum by (le) (namespace_job_route:loki_request_duration_seconds_bucket:sum_rate{namespace=\"$namespace\", job=~\".+-distributor-http\", route=~\"api_prom_push|loki_api_v1_push|otlp_v1_logs|/httpgrpc.HTTP/Handle\"})) * 1e3",
                      "format": "time_series",
                      "intervalFactor": 2,
                      "legendFormat": "99th Percentile",
@@ -271,7 +271,7 @@
                      "step": 10
                   },
                   {
-                     "expr": "histogram_quantile(0.50, sum by (le) (namespace_job_route:loki_request_duration_seconds_bucket:sum_rate{namespace=\"$namespace\", job=~\".+-distributor-http\", route=~\"api_prom_push|loki_api_v1_push|/httpgrpc.HTTP/Handle\"})) * 1e3",
+                     "expr": "histogram_quantile(0.50, sum by (le) (namespace_job_route:loki_request_duration_seconds_bucket:sum_rate{namespace=\"$namespace\", job=~\".+-distributor-http\", route=~\"api_prom_push|loki_api_v1_push|otlp_v1_logs|/httpgrpc.HTTP/Handle\"})) * 1e3",
                      "format": "time_series",
                      "intervalFactor": 2,
                      "legendFormat": "50th Percentile",
@@ -279,7 +279,7 @@
                      "step": 10
                   },
                   {
-                     "expr": "1e3 * sum(namespace_job_route:loki_request_duration_seconds_sum:sum_rate{namespace=\"$namespace\", job=~\".+-distributor-http\", route=~\"api_prom_push|loki_api_v1_push|/httpgrpc.HTTP/Handle\"}) / sum(namespace_job_route:loki_request_duration_seconds_count:sum_rate{namespace=\"$namespace\", job=~\".+-distributor-http\", route=~\"api_prom_push|loki_api_v1_push|/httpgrpc.HTTP/Handle\"})",
+                     "expr": "1e3 * sum(namespace_job_route:loki_request_duration_seconds_sum:sum_rate{namespace=\"$namespace\", job=~\".+-distributor-http\", route=~\"api_prom_push|loki_api_v1_push|otlp_v1_logs|/httpgrpc.HTTP/Handle\"}) / sum(namespace_job_route:loki_request_duration_seconds_count:sum_rate{namespace=\"$namespace\", job=~\".+-distributor-http\", route=~\"api_prom_push|loki_api_v1_push|otlp_v1_logs|/httpgrpc.HTTP/Handle\"})",
                      "format": "time_series",
                      "intervalFactor": 2,
                      "legendFormat": "Average",
@@ -347,7 +347,7 @@
                "span": 4,
                "targets": [
                   {
-                     "expr": "histogram_quantile(0.99, sum(rate(loki_request_duration_seconds_bucket{namespace=\"$namespace\", job=~\".+-distributor-http\", route=~\"api_prom_push|loki_api_v1_push|/httpgrpc.HTTP/Handle\"}[$__rate_interval])) by (le,pod)) * 1e3",
+                     "expr": "histogram_quantile(0.99, sum(rate(loki_request_duration_seconds_bucket{namespace=\"$namespace\", job=~\".+-distributor-http\", route=~\"api_prom_push|loki_api_v1_push|otlp_v1_logs|/httpgrpc.HTTP/Handle\"}[$__rate_interval])) by (le,pod)) * 1e3",
                      "format": "time_series",
                      "interval": "1m",
                      "intervalFactor": 2,
diff --git a/operator/internal/manifests/openshift/opa_openshift.go b/operator/internal/manifests/openshift/opa_openshift.go
index 9175983f89e14..ccf5eac09b7a7 100644
--- a/operator/internal/manifests/openshift/opa_openshift.go
+++ b/operator/internal/manifests/openshift/opa_openshift.go
@@ -13,14 +13,15 @@ import (
 )
 
 const (
-	envRelatedImageOPA      = "RELATED_IMAGE_OPA"
-	defaultOPAImage         = "quay.io/observatorium/opa-openshift:latest"
-	opaContainerName        = "opa"
-	opaDefaultPackage       = "lokistack"
-	opaDefaultAPIGroup      = "loki.grafana.com"
-	opaMetricsPortName      = "opa-metrics"
-	opaDefaultLabelMatcher  = "kubernetes_namespace_name"
-	opaNetworkLabelMatchers = "SrcK8S_Namespace,DstK8S_Namespace"
+	envRelatedImageOPA        = "RELATED_IMAGE_OPA"
+	defaultOPAImage           = "quay.io/observatorium/opa-openshift:latest"
+	opaContainerName          = "opa"
+	opaDefaultPackage         = "lokistack"
+	opaDefaultAPIGroup        = "loki.grafana.com"
+	opaMetricsPortName        = "opa-metrics"
+	opaDefaultLabelMatcher    = "kubernetes_namespace_name"
+	opaNetworkLabelMatchers   = "SrcK8S_Namespace,DstK8S_Namespace"
+	ocpMonitoringGroupByLabel = "namespace"
 )
 
 func newOPAOpenShiftContainer(mode lokiv1.ModeType, secretVolumeName, tlsDir, minTLSVersion, ciphers string, withTLS bool, adminGroups []string) corev1.Container {
diff --git a/operator/internal/manifests/openshift/recordingrule.go b/operator/internal/manifests/openshift/recordingrule.go
new file mode 100644
index 0000000000000..97be1bb4a17ec
--- /dev/null
+++ b/operator/internal/manifests/openshift/recordingrule.go
@@ -0,0 +1,36 @@
+package openshift
+
+import lokiv1 "github.com/grafana/loki/operator/apis/loki/v1"
+
+func RecordingRuleTenantLabels(r *lokiv1.RecordingRule) {
+	switch r.Spec.TenantID {
+	case tenantApplication:
+		appendRecordingRuleLabels(r, map[string]string{
+			opaDefaultLabelMatcher:    r.Namespace,
+			ocpMonitoringGroupByLabel: r.Namespace,
+		})
+	case tenantInfrastructure, tenantAudit, tenantNetwork:
+		appendRecordingRuleLabels(r, map[string]string{
+			ocpMonitoringGroupByLabel: r.Namespace,
+		})
+	default:
+		// Do nothing
+	}
+}
+
+func appendRecordingRuleLabels(r *lokiv1.RecordingRule, labels map[string]string) {
+	for groupIdx, group := range r.Spec.Groups {
+		for ruleIdx, rule := range group.Rules {
+			if rule.Labels == nil {
+				rule.Labels = map[string]string{}
+			}
+
+			for name, value := range labels {
+				rule.Labels[name] = value
+			}
+
+			group.Rules[ruleIdx] = rule
+		}
+		r.Spec.Groups[groupIdx] = group
+	}
+}
diff --git a/operator/internal/manifests/openshift/recordingrule_test.go b/operator/internal/manifests/openshift/recordingrule_test.go
index 49e30de999f35..6a620bc85d8de 100644
--- a/operator/internal/manifests/openshift/recordingrule_test.go
+++ b/operator/internal/manifests/openshift/recordingrule_test.go
@@ -46,7 +46,8 @@ func TestRecordingRuleTenantLabels(t *testing.T) {
 								{
 									Record: "record",
 									Labels: map[string]string{
-										opaDefaultLabelMatcher: "test-ns",
+										opaDefaultLabelMatcher:    "test-ns",
+										ocpMonitoringGroupByLabel: "test-ns",
 									},
 								},
 							},
@@ -57,6 +58,9 @@ func TestRecordingRuleTenantLabels(t *testing.T) {
 		},
 		{
 			rule: &lokiv1.RecordingRule{
+				ObjectMeta: metav1.ObjectMeta{
+					Namespace: "test-ns",
+				},
 				Spec: lokiv1.RecordingRuleSpec{
 					TenantID: tenantInfrastructure,
 					Groups: []*lokiv1.RecordingRuleGroup{
@@ -72,6 +76,9 @@ func TestRecordingRuleTenantLabels(t *testing.T) {
 				},
 			},
 			want: &lokiv1.RecordingRule{
+				ObjectMeta: metav1.ObjectMeta{
+					Namespace: "test-ns",
+				},
 				Spec: lokiv1.RecordingRuleSpec{
 					TenantID: tenantInfrastructure,
 					Groups: []*lokiv1.RecordingRuleGroup{
@@ -80,6 +87,9 @@ func TestRecordingRuleTenantLabels(t *testing.T) {
 							Rules: []*lokiv1.RecordingRuleGroupSpec{
 								{
 									Record: "record",
+									Labels: map[string]string{
+										ocpMonitoringGroupByLabel: "test-ns",
+									},
 								},
 							},
 						},
@@ -89,6 +99,9 @@ func TestRecordingRuleTenantLabels(t *testing.T) {
 		},
 		{
 			rule: &lokiv1.RecordingRule{
+				ObjectMeta: metav1.ObjectMeta{
+					Namespace: "test-ns",
+				},
 				Spec: lokiv1.RecordingRuleSpec{
 					TenantID: tenantAudit,
 					Groups: []*lokiv1.RecordingRuleGroup{
@@ -104,6 +117,9 @@ func TestRecordingRuleTenantLabels(t *testing.T) {
 				},
 			},
 			want: &lokiv1.RecordingRule{
+				ObjectMeta: metav1.ObjectMeta{
+					Namespace: "test-ns",
+				},
 				Spec: lokiv1.RecordingRuleSpec{
 					TenantID: tenantAudit,
 					Groups: []*lokiv1.RecordingRuleGroup{
@@ -112,6 +128,9 @@ func TestRecordingRuleTenantLabels(t *testing.T) {
 							Rules: []*lokiv1.RecordingRuleGroupSpec{
 								{
 									Record: "record",
+									Labels: map[string]string{
+										ocpMonitoringGroupByLabel: "test-ns",
+									},
 								},
 							},
 						},
@@ -121,6 +140,9 @@ func TestRecordingRuleTenantLabels(t *testing.T) {
 		},
 		{
 			rule: &lokiv1.RecordingRule{
+				ObjectMeta: metav1.ObjectMeta{
+					Namespace: "test-ns",
+				},
 				Spec: lokiv1.RecordingRuleSpec{
 					TenantID: tenantNetwork,
 					Groups: []*lokiv1.RecordingRuleGroup{
@@ -136,6 +158,9 @@ func TestRecordingRuleTenantLabels(t *testing.T) {
 				},
 			},
 			want: &lokiv1.RecordingRule{
+				ObjectMeta: metav1.ObjectMeta{
+					Namespace: "test-ns",
+				},
 				Spec: lokiv1.RecordingRuleSpec{
 					TenantID: tenantNetwork,
 					Groups: []*lokiv1.RecordingRuleGroup{
@@ -144,6 +169,9 @@ func TestRecordingRuleTenantLabels(t *testing.T) {
 							Rules: []*lokiv1.RecordingRuleGroupSpec{
 								{
 									Record: "record",
+									Labels: map[string]string{
+										ocpMonitoringGroupByLabel: "test-ns",
+									},
 								},
 							},
 						},
@@ -153,6 +181,9 @@ func TestRecordingRuleTenantLabels(t *testing.T) {
 		},
 		{
 			rule: &lokiv1.RecordingRule{
+				ObjectMeta: metav1.ObjectMeta{
+					Namespace: "test-ns",
+				},
 				Spec: lokiv1.RecordingRuleSpec{
 					TenantID: "unknown",
 					Groups: []*lokiv1.RecordingRuleGroup{
@@ -168,6 +199,9 @@ func TestRecordingRuleTenantLabels(t *testing.T) {
 				},
 			},
 			want: &lokiv1.RecordingRule{
+				ObjectMeta: metav1.ObjectMeta{
+					Namespace: "test-ns",
+				},
 				Spec: lokiv1.RecordingRuleSpec{
 					TenantID: "unknown",
 					Groups: []*lokiv1.RecordingRuleGroup{
diff --git a/operator/internal/manifests/openshift/recordngrule.go b/operator/internal/manifests/openshift/recordngrule.go
deleted file mode 100644
index e4448affeae99..0000000000000
--- a/operator/internal/manifests/openshift/recordngrule.go
+++ /dev/null
@@ -1,27 +0,0 @@
-package openshift
-
-import lokiv1 "github.com/grafana/loki/operator/apis/loki/v1"
-
-func RecordingRuleTenantLabels(r *lokiv1.RecordingRule) {
-	switch r.Spec.TenantID {
-	case tenantApplication:
-		for groupIdx, group := range r.Spec.Groups {
-			group := group
-			for ruleIdx, rule := range group.Rules {
-				rule := rule
-				if rule.Labels == nil {
-					rule.Labels = map[string]string{}
-				}
-				rule.Labels[opaDefaultLabelMatcher] = r.Namespace
-				group.Rules[ruleIdx] = rule
-			}
-			r.Spec.Groups[groupIdx] = group
-		}
-	case tenantInfrastructure, tenantAudit:
-		// Do nothing
-	case tenantNetwork:
-		// Do nothing
-	default:
-		// Do nothing
-	}
-}
diff --git a/operator/internal/manifests/var.go b/operator/internal/manifests/var.go
index 1fed3e1422982..46ba72d450850 100644
--- a/operator/internal/manifests/var.go
+++ b/operator/internal/manifests/var.go
@@ -59,7 +59,7 @@ const (
 	EnvRelatedImageGateway = "RELATED_IMAGE_GATEWAY"
 
 	// DefaultContainerImage declares the default fallback for loki image.
-	DefaultContainerImage = "docker.io/grafana/loki:3.1.1"
+	DefaultContainerImage = "docker.io/grafana/loki:3.2.0"
 
 	// DefaultLokiStackGatewayImage declares the default image for lokiStack-gateway.
 	DefaultLokiStackGatewayImage = "quay.io/observatorium/api:latest"
diff --git a/operator/internal/status/components.go b/operator/internal/status/components.go
index 2d3061ecfc500..3e59f8a198556 100644
--- a/operator/internal/status/components.go
+++ b/operator/internal/status/components.go
@@ -73,6 +73,15 @@ func appendPodStatus(ctx context.Context, k k8s.Client, component, stack, ns str
 		status := podStatus(&pod)
 		psm[status] = append(psm[status], pod.Name)
 	}
+
+	if len(psm) == 0 {
+		psm = lokiv1.PodStatusMap{
+			lokiv1.PodFailed:  []string{},
+			lokiv1.PodPending: []string{},
+			lokiv1.PodRunning: []string{},
+			lokiv1.PodReady:   []string{},
+		}
+	}
 	return psm, nil
 }
 
diff --git a/operator/internal/status/components_test.go b/operator/internal/status/components_test.go
index 111074f880031..d43bb2a293150 100644
--- a/operator/internal/status/components_test.go
+++ b/operator/internal/status/components_test.go
@@ -65,6 +65,13 @@ func setupListClient(t *testing.T, stack *lokiv1.LokiStack, componentPods map[st
 }
 
 func TestGenerateComponentStatus(t *testing.T) {
+	empty := lokiv1.PodStatusMap{
+		lokiv1.PodFailed:  []string{},
+		lokiv1.PodPending: []string{},
+		lokiv1.PodRunning: []string{},
+		lokiv1.PodReady:   []string{},
+	}
+
 	tt := []struct {
 		desc                string
 		componentPods       map[string]*corev1.PodList
@@ -83,14 +90,14 @@ func TestGenerateComponentStatus(t *testing.T) {
 				manifests.LabelGatewayComponent:       {},
 			},
 			wantComponentStatus: &lokiv1.LokiStackComponentStatus{
-				Compactor:     lokiv1.PodStatusMap{},
-				Distributor:   lokiv1.PodStatusMap{},
-				IndexGateway:  lokiv1.PodStatusMap{},
-				Ingester:      lokiv1.PodStatusMap{},
-				Querier:       lokiv1.PodStatusMap{},
-				QueryFrontend: lokiv1.PodStatusMap{},
-				Gateway:       lokiv1.PodStatusMap{},
-				Ruler:         lokiv1.PodStatusMap{},
+				Compactor:     empty,
+				Distributor:   empty,
+				IndexGateway:  empty,
+				Ingester:      empty,
+				Querier:       empty,
+				QueryFrontend: empty,
+				Gateway:       empty,
+				Ruler:         empty,
 			},
 		},
 		{
@@ -116,6 +123,29 @@ func TestGenerateComponentStatus(t *testing.T) {
 				Ruler:         lokiv1.PodStatusMap{lokiv1.PodRunning: {"ruler-pod-0"}},
 			},
 		},
+		{
+			desc: "all pods without ruler",
+			componentPods: map[string]*corev1.PodList{
+				manifests.LabelCompactorComponent:     createPodList(manifests.LabelCompactorComponent, false, corev1.PodRunning),
+				manifests.LabelDistributorComponent:   createPodList(manifests.LabelDistributorComponent, false, corev1.PodRunning),
+				manifests.LabelIngesterComponent:      createPodList(manifests.LabelIngesterComponent, false, corev1.PodRunning),
+				manifests.LabelQuerierComponent:       createPodList(manifests.LabelQuerierComponent, false, corev1.PodRunning),
+				manifests.LabelQueryFrontendComponent: createPodList(manifests.LabelQueryFrontendComponent, false, corev1.PodRunning),
+				manifests.LabelIndexGatewayComponent:  createPodList(manifests.LabelIndexGatewayComponent, false, corev1.PodRunning),
+				manifests.LabelRulerComponent:         {},
+				manifests.LabelGatewayComponent:       createPodList(manifests.LabelGatewayComponent, false, corev1.PodRunning),
+			},
+			wantComponentStatus: &lokiv1.LokiStackComponentStatus{
+				Compactor:     lokiv1.PodStatusMap{lokiv1.PodRunning: {"compactor-pod-0"}},
+				Distributor:   lokiv1.PodStatusMap{lokiv1.PodRunning: {"distributor-pod-0"}},
+				IndexGateway:  lokiv1.PodStatusMap{lokiv1.PodRunning: {"index-gateway-pod-0"}},
+				Ingester:      lokiv1.PodStatusMap{lokiv1.PodRunning: {"ingester-pod-0"}},
+				Querier:       lokiv1.PodStatusMap{lokiv1.PodRunning: {"querier-pod-0"}},
+				QueryFrontend: lokiv1.PodStatusMap{lokiv1.PodRunning: {"query-frontend-pod-0"}},
+				Gateway:       lokiv1.PodStatusMap{lokiv1.PodRunning: {"lokistack-gateway-pod-0"}},
+				Ruler:         empty,
+			},
+		},
 	}
 
 	for _, tc := range tt {
diff --git a/operator/jsonnet/jsonnetfile.json b/operator/jsonnet/jsonnetfile.json
index 7d1b707308153..a2ac102b07587 100644
--- a/operator/jsonnet/jsonnetfile.json
+++ b/operator/jsonnet/jsonnetfile.json
@@ -8,7 +8,7 @@
           "subdir": "production/loki-mixin"
         }
       },
-      "version": "v3.1.0"
+      "version": "v3.2.0"
     }
   ],
   "legacyImports": true
diff --git a/operator/jsonnet/jsonnetfile.lock.json b/operator/jsonnet/jsonnetfile.lock.json
index acc2cd702fe38..40bf4f23a2544 100644
--- a/operator/jsonnet/jsonnetfile.lock.json
+++ b/operator/jsonnet/jsonnetfile.lock.json
@@ -38,8 +38,8 @@
           "subdir": "production/loki-mixin"
         }
       },
-      "version": "935aee77ed389c825d36b8d6a85c0d83895a24d1",
-      "sum": "FsHTEwIRvbbC2qPwDuzOQx8ilfa1+gR/8r52bFsNQMU="
+      "version": "659f5421dde1dc4b27d6a0afd1a568673f50dfcc",
+      "sum": "mVKuwcuL1Wm+JMgt2MiwOAtKkmuClzE75+liMe0AGek="
     },
     {
       "source": {
diff --git a/pkg/logql/engine.go b/pkg/logql/engine.go
index c44ee06c16948..17e90520e7de5 100644
--- a/pkg/logql/engine.go
+++ b/pkg/logql/engine.go
@@ -241,11 +241,21 @@ func (q *query) Exec(ctx context.Context) (logqlmodel.Result, error) {
 
 	if q.logExecQuery {
 		queryHash := util.HashedQuery(q.params.QueryString())
+
+		logValues := []interface{}{
+			"msg", "executing query",
+			"query", q.params.QueryString(),
+			"query_hash", queryHash,
+		}
+		tags := httpreq.ExtractQueryTagsFromContext(ctx)
+		tagValues := tagsToKeyValues(tags)
 		if GetRangeType(q.params) == InstantType {
-			level.Info(logutil.WithContext(ctx, q.logger)).Log("msg", "executing query", "type", "instant", "query", q.params.QueryString(), "query_hash", queryHash)
+			logValues = append(logValues, "type", "instant")
 		} else {
-			level.Info(logutil.WithContext(ctx, q.logger)).Log("msg", "executing query", "type", "range", "query", q.params.QueryString(), "length", q.params.End().Sub(q.params.Start()), "step", q.params.Step(), "query_hash", queryHash)
+			logValues = append(logValues, "type", "range", "length", q.params.End().Sub(q.params.Start()), "step", q.params.Step())
 		}
+		logValues = append(logValues, tagValues...)
+		level.Info(logutil.WithContext(ctx, q.logger)).Log(logValues...)
 	}
 
 	rangeType := GetRangeType(q.params)
@@ -265,6 +275,7 @@ func (q *query) Exec(ctx context.Context) (logqlmodel.Result, error) {
 	sp.LogKV(statResult.KVList()...)
 
 	status, _ := server.ClientHTTPStatusAndError(err)
+
 	if q.record {
 		RecordRangeAndInstantQueryMetrics(ctx, q.logger, q.params, strconv.Itoa(status), statResult, data)
 	}
diff --git a/pkg/logql/engine_test.go b/pkg/logql/engine_test.go
index 23bcbac91e0f6..fd534ac046e8c 100644
--- a/pkg/logql/engine_test.go
+++ b/pkg/logql/engine_test.go
@@ -6,7 +6,6 @@ import (
 	"errors"
 	"fmt"
 	"math"
-	"regexp"
 	"strings"
 	"testing"
 	"time"
@@ -2618,23 +2617,14 @@ func TestHashingStability(t *testing.T) {
 		expectedQueryHash := util.HashedQuery(test.qs)
 
 		// check that both places will end up having the same query hash, even though they're emitting different log lines.
-		require.Regexp(t,
-			regexp.MustCompile(
-				fmt.Sprintf(
-					`level=info org_id=fake msg="executing query" type=range query=.* length=5s step=1m0s query_hash=%d.*`, expectedQueryHash,
-				),
-			),
-			queryWithEngine(),
-		)
-
-		require.Regexp(t,
-			regexp.MustCompile(
-				fmt.Sprintf(
-					`level=info org_id=fake latency=slow query=".*" query_hash=%d query_type=metric range_type=range.*\n`, expectedQueryHash,
-				),
-			),
-			queryDirectly(),
-		)
+		withEngine := queryWithEngine()
+		require.Contains(t, withEngine, fmt.Sprintf("query_hash=%d", expectedQueryHash))
+		require.Contains(t, withEngine, "step=1m0s")
+
+		directly := queryDirectly()
+		require.Contains(t, directly, fmt.Sprintf("query_hash=%d", expectedQueryHash))
+		require.Contains(t, directly, "length=5s")
+		require.Contains(t, directly, "latency=slow")
 	}
 }
 
diff --git a/production/helm/loki/CHANGELOG.md b/production/helm/loki/CHANGELOG.md
index 519755dbec944..18d0aca93a3d6 100644
--- a/production/helm/loki/CHANGELOG.md
+++ b/production/helm/loki/CHANGELOG.md
@@ -13,9 +13,18 @@ Entries should include a reference to the pull request that introduced the chang
 
 [//]: # (<AUTOMATED_UPDATES_LOCATOR> : do not remove this line. This locator is used by the CI pipeline to automatically create a changelog entry for each new Loki release. Add other chart versions and respective changelog entries bellow this line.)
 
+## 6.17.0
+
+- [CHANGE] Changed version of Grafana Loki to 3.2.0
+
+## 6.16.0
+
+- [ENHANCEMENT] Allow setting nodeSelector, tolerations and affinity to enterprise components (tokengen and provisioner).
+
 ## 6.15.0
 
 - [ENHANCEMENT] Allow setting annotations for memberlist and query-scheduler-discovery services
+
 ## 6.14.1
 
 - [BUGFIX] Fixed Memcached persistence options.
diff --git a/production/helm/loki/Chart.yaml b/production/helm/loki/Chart.yaml
index 56e48a535e070..07941278dfa0a 100644
--- a/production/helm/loki/Chart.yaml
+++ b/production/helm/loki/Chart.yaml
@@ -2,8 +2,8 @@ apiVersion: v2
 name: loki
 description: Helm chart for Grafana Loki and Grafana Enterprise Logs supporting both simple, scalable and distributed modes.
 type: application
-appVersion: 3.1.1
-version: 6.15.0
+appVersion: 3.2.0
+version: 6.17.0
 home: https://grafana.github.io/helm-charts
 sources:
   - https://github.com/grafana/loki
diff --git a/production/helm/loki/README.md b/production/helm/loki/README.md
index 92008e768d6bc..12ebf95948e23 100644
--- a/production/helm/loki/README.md
+++ b/production/helm/loki/README.md
@@ -1,6 +1,6 @@
 # loki
 
-![Version: 6.15.0](https://img.shields.io/badge/Version-6.15.0-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 3.1.1](https://img.shields.io/badge/AppVersion-3.1.1-informational?style=flat-square)
+![Version: 6.17.0](https://img.shields.io/badge/Version-6.17.0-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 3.2.0](https://img.shields.io/badge/AppVersion-3.2.0-informational?style=flat-square)
 
 Helm chart for Grafana Loki and Grafana Enterprise Logs supporting both simple, scalable and distributed modes.
 
@@ -19,3 +19,14 @@ Helm chart for Grafana Loki and Grafana Enterprise Logs supporting both simple,
 | https://grafana.github.io/helm-charts | rollout_operator(rollout-operator) | 0.13.0 |
 
 Find more information in the Loki Helm Chart [documentation](https://grafana.com/docs/loki/next/installation/helm).
+
+## Contributing and releasing
+
+If you made any changes to the [Chart.yaml](https://github.com/grafana/loki/blob/main/production/helm/loki/Chart.yaml) or [values.yaml](https://github.com/grafana/loki/blob/main/production/helm/loki/values.yaml) run `make helm-doc` from the root of the repository to update the documentation and commit the changed files.
+
+#### Versioning
+
+Normally contributors need _not_ bump the version nor update the [CHANGELOG.md](https://github.com/grafana/loki/blob/main/production/helm/loki/CHANGELOG.md). A new version of the Chart will follow this cadence:
+- Automatic weekly releases
+- Releases that coincide with Loki/GEL releases
+- Manual releases when necessary (ie. to address a CVE or critical bug)
diff --git a/production/helm/loki/README.md.gotmpl b/production/helm/loki/README.md.gotmpl
index dcd577845d4b3..f1af286656a24 100644
--- a/production/helm/loki/README.md.gotmpl
+++ b/production/helm/loki/README.md.gotmpl
@@ -9,3 +9,14 @@
 {{ template "chart.requirementsSection" . }}
 
 Find more information in the Loki Helm Chart [documentation](https://grafana.com/docs/loki/next/installation/helm).
+
+## Contributing and releasing
+
+If you made any changes to the [Chart.yaml](https://github.com/grafana/loki/blob/main/production/helm/loki/Chart.yaml) or [values.yaml](https://github.com/grafana/loki/blob/main/production/helm/loki/values.yaml) run `make helm-doc` from the root of the repository to update the documentation and commit the changed files.
+
+#### Versioning
+
+Normally contributors need _not_ bump the version nor update the [CHANGELOG.md](https://github.com/grafana/loki/blob/main/production/helm/loki/CHANGELOG.md). A new version of the Chart will follow this cadence:
+- Automatic weekly releases
+- Releases that coincide with Loki/GEL releases
+- Manual releases when necessary (ie. to address a CVE or critical bug)
diff --git a/production/helm/loki/templates/provisioner/job-provisioner.yaml b/production/helm/loki/templates/provisioner/job-provisioner.yaml
index 61fd2be8501ba..5a6bc063715f4 100644
--- a/production/helm/loki/templates/provisioner/job-provisioner.yaml
+++ b/production/helm/loki/templates/provisioner/job-provisioner.yaml
@@ -123,6 +123,18 @@ spec:
             {{- end }}
             - name: bootstrap
               mountPath: /bootstrap
+      {{- with .Values.enterprise.provisioner.affinity }}
+      affinity:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      {{- with .Values.enterprise.provisioner.nodeSelector }}
+      nodeSelector:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      {{- with .Values.enterprise.provisioner.tolerations }}
+      tolerations:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
       restartPolicy: OnFailure
       serviceAccount: {{ include "enterprise-logs.provisionerFullname" . }}
       serviceAccountName: {{ include "enterprise-logs.provisionerFullname" . }}
diff --git a/production/helm/loki/templates/tokengen/job-tokengen.yaml b/production/helm/loki/templates/tokengen/job-tokengen.yaml
index f9ae7374c2d17..b0950d6f19675 100644
--- a/production/helm/loki/templates/tokengen/job-tokengen.yaml
+++ b/production/helm/loki/templates/tokengen/job-tokengen.yaml
@@ -110,6 +110,14 @@ spec:
       restartPolicy: OnFailure
       serviceAccount: {{ template "enterprise-logs.tokengenFullname" . }}
       serviceAccountName: {{ template "enterprise-logs.tokengenFullname" . }}
+      {{- with .Values.enterprise.tokengen.affinity }}
+      affinity:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      {{- with .Values.enterprise.tokengen.nodeSelector }}
+      nodeSelector:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
       {{- with .Values.enterprise.tokengen.tolerations }}
       tolerations:
         {{- toYaml . | nindent 8 }}
diff --git a/production/helm/loki/values.yaml b/production/helm/loki/values.yaml
index a3ee28a805338..ab62323c95f86 100644
--- a/production/helm/loki/values.yaml
+++ b/production/helm/loki/values.yaml
@@ -50,8 +50,6 @@ loki:
     # -- Docker image repository
     repository: grafana/loki
     # -- Overrides the image tag whose default is the chart's appVersion
-    # TODO: needed for 3rd target backend functionality
-    # revert to null or latest once this behavior is relased
     tag: null
     # -- Overrides the image tag with an image digest
     digest: null
@@ -540,6 +538,10 @@ enterprise:
     labels: {}
     # -- Additional annotations for the `tokengen` Job
     annotations: {}
+    # -- Affinity for tokengen Pods
+    affinity: {}
+    # -- Node selector for tokengen Pods
+    nodeSelector: {}
     # -- Tolerations for tokengen Job
     tolerations: []
     # -- Additional volumes for Pods
@@ -575,6 +577,12 @@ enterprise:
     labels: {}
     # -- Additional annotations for the `provisioner` Job
     annotations: {}
+    # -- Affinity for tokengen Pods
+    affinity: {}
+    # -- Node selector for tokengen Pods
+    nodeSelector: {}
+    # -- Tolerations for tokengen Pods
+    tolerations: []
     # -- The name of the PriorityClass for provisioner Job
     priorityClassName: null
     # -- Run containers as user `enterprise-logs(uid=10001)`
diff --git a/production/loki-mixin-compiled-ssd/alerts.yaml b/production/loki-mixin-compiled-ssd/alerts.yaml
index 09b9b6f543412..e33c010ff0560 100644
--- a/production/loki-mixin-compiled-ssd/alerts.yaml
+++ b/production/loki-mixin-compiled-ssd/alerts.yaml
@@ -43,3 +43,36 @@ groups:
           for: 5m
           labels:
             severity: warning
+        - alert: LokiCompactorHasNotSuccessfullyRunCompaction
+          annotations:
+            description: |
+                {{ $labels.cluster }} {{ $labels.namespace }} has not run compaction in the last 3 hours since the last compaction. This may indicate a problem with the compactor.
+            summary: Loki compaction has not run in the last 3 hours since the last compaction.
+          expr: |
+            # The "last successful run" metric is updated even if the compactor owns no tenants,
+            # so this alert correctly doesn't fire if compactor has nothing to do.
+            min (
+              time() - (loki_boltdb_shipper_compact_tables_operation_last_successful_run_timestamp_seconds{} > 0)
+            )
+            by (cluster, namespace)
+            > 60 * 60 * 3
+          for: 1h
+          labels:
+            severity: critical
+        - alert: LokiCompactorHasNotSuccessfullyRunCompaction
+          annotations:
+            description: |
+                {{ $labels.cluster }} {{ $labels.namespace }} has not run compaction in the last 3h since startup. This may indicate a problem with the compactor.
+            summary: Loki compaction has not run in the last 3h since startup.
+          expr: |
+            # The "last successful run" metric is updated even if the compactor owns no tenants,
+            # so this alert correctly doesn't fire if compactor has nothing to do.
+            max(
+              max_over_time(
+                loki_boltdb_shipper_compact_tables_operation_last_successful_run_timestamp_seconds{}[3h]
+              )
+            ) by (cluster, namespace)
+            == 0
+          for: 1h
+          labels:
+            severity: critical
diff --git a/production/loki-mixin-compiled/alerts.yaml b/production/loki-mixin-compiled/alerts.yaml
index 09b9b6f543412..e33c010ff0560 100644
--- a/production/loki-mixin-compiled/alerts.yaml
+++ b/production/loki-mixin-compiled/alerts.yaml
@@ -43,3 +43,36 @@ groups:
           for: 5m
           labels:
             severity: warning
+        - alert: LokiCompactorHasNotSuccessfullyRunCompaction
+          annotations:
+            description: |
+                {{ $labels.cluster }} {{ $labels.namespace }} has not run compaction in the last 3 hours since the last compaction. This may indicate a problem with the compactor.
+            summary: Loki compaction has not run in the last 3 hours since the last compaction.
+          expr: |
+            # The "last successful run" metric is updated even if the compactor owns no tenants,
+            # so this alert correctly doesn't fire if compactor has nothing to do.
+            min (
+              time() - (loki_boltdb_shipper_compact_tables_operation_last_successful_run_timestamp_seconds{} > 0)
+            )
+            by (cluster, namespace)
+            > 60 * 60 * 3
+          for: 1h
+          labels:
+            severity: critical
+        - alert: LokiCompactorHasNotSuccessfullyRunCompaction
+          annotations:
+            description: |
+                {{ $labels.cluster }} {{ $labels.namespace }} has not run compaction in the last 3h since startup. This may indicate a problem with the compactor.
+            summary: Loki compaction has not run in the last 3h since startup.
+          expr: |
+            # The "last successful run" metric is updated even if the compactor owns no tenants,
+            # so this alert correctly doesn't fire if compactor has nothing to do.
+            max(
+              max_over_time(
+                loki_boltdb_shipper_compact_tables_operation_last_successful_run_timestamp_seconds{}[3h]
+              )
+            ) by (cluster, namespace)
+            == 0
+          for: 1h
+          labels:
+            severity: critical
diff --git a/production/loki-mixin/alerts.libsonnet b/production/loki-mixin/alerts.libsonnet
index 25fbbefaca172..0d34086d9b3a5 100644
--- a/production/loki-mixin/alerts.libsonnet
+++ b/production/loki-mixin/alerts.libsonnet
@@ -70,6 +70,53 @@
               |||, 'cluster', $._config.per_cluster_label),
             },
           },
+          {
+            // Alert if the compactor has not successfully run compaction in the last 3h since the last compaction.
+            alert: 'LokiCompactorHasNotSuccessfullyRunCompaction',
+            expr: |||
+              # The "last successful run" metric is updated even if the compactor owns no tenants,
+              # so this alert correctly doesn't fire if compactor has nothing to do.
+              min (
+                time() - (loki_boltdb_shipper_compact_tables_operation_last_successful_run_timestamp_seconds{} > 0)
+              )
+              by (%s, namespace)
+              > 60 * 60 * 3
+            ||| % $._config.per_cluster_label,
+            'for': '1h',
+            labels: {
+              severity: 'critical',
+            },
+            annotations: {
+              summary: 'Loki compaction has not run in the last 3 hours since the last compaction.',
+              description: std.strReplace(|||
+                {{ $labels.cluster }} {{ $labels.namespace }} has not run compaction in the last 3 hours since the last compaction. This may indicate a problem with the compactor.
+              |||, 'cluster', $._config.per_cluster_label),
+            },
+          },
+          {
+            // Alert if the compactor has not successfully run compaction in the last 3h since startup.
+            alert: 'LokiCompactorHasNotSuccessfullyRunCompaction',
+            expr: |||
+              # The "last successful run" metric is updated even if the compactor owns no tenants,
+              # so this alert correctly doesn't fire if compactor has nothing to do.
+              max(
+                max_over_time(
+                  loki_boltdb_shipper_compact_tables_operation_last_successful_run_timestamp_seconds{}[3h]
+                )
+              ) by (%s, namespace)
+              == 0
+            ||| % $._config.per_cluster_label,
+            'for': '1h',
+            labels: {
+              severity: 'critical',
+            },
+            annotations: {
+              summary: 'Loki compaction has not run in the last 3h since startup.',
+              description: std.strReplace(|||
+                {{ $labels.cluster }} {{ $labels.namespace }} has not run compaction in the last 3h since startup. This may indicate a problem with the compactor.
+              |||, 'cluster', $._config.per_cluster_label),
+            },
+          },
         ],
       },
     ],
diff --git a/production/terraform/modules/s3/versions.tf b/production/terraform/modules/s3/versions.tf
index ffbae7a8f9edf..06f1024a30144 100644
--- a/production/terraform/modules/s3/versions.tf
+++ b/production/terraform/modules/s3/versions.tf
@@ -2,12 +2,12 @@ terraform {
   required_providers {
     aws = {
       source  = "hashicorp/aws"
-      version = "~> 5.65.0"
+      version = "~> 5.69.0"
     }
 
     random = {
       source  = "hashicorp/random"
-      version = "3.6.2"
+      version = "3.6.3"
     }
   }
 
diff --git a/tools/gcplog/main.tf b/tools/gcplog/main.tf
index fa2f2fbe646ae..73217fd10ca68 100644
--- a/tools/gcplog/main.tf
+++ b/tools/gcplog/main.tf
@@ -2,7 +2,7 @@ terraform {
   required_providers {
     google = {
       source = "hashicorp/google"
-      version = "6.0.1"
+      version = "6.5.0"
     }
   }
 }
diff --git a/tools/lambda-promtail/go.mod b/tools/lambda-promtail/go.mod
index 42e8066c9436e..96480efeb823b 100644
--- a/tools/lambda-promtail/go.mod
+++ b/tools/lambda-promtail/go.mod
@@ -10,7 +10,7 @@ require (
 	github.com/go-kit/log v0.2.1
 	github.com/gogo/protobuf v1.3.2
 	github.com/golang/snappy v0.0.4
-	github.com/grafana/dskit v0.0.0-20240925193654-7c41a4057319
+	github.com/grafana/dskit v0.0.0-20240930165212-f52de24af9bc
 	github.com/grafana/loki/v3 v3.0.0-20240809103847-9315b3d03d79
 	github.com/prometheus/common v0.55.0
 	github.com/stretchr/testify v1.9.0
diff --git a/tools/lambda-promtail/go.sum b/tools/lambda-promtail/go.sum
index 1729b4d8e00c4..024bef722f7ae 100644
--- a/tools/lambda-promtail/go.sum
+++ b/tools/lambda-promtail/go.sum
@@ -216,8 +216,8 @@ github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
 github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/gorilla/mux v1.8.1 h1:TuBL49tXwgrFYWhqrNgrUNEY92u81SPhu7sTdzQEiWY=
 github.com/gorilla/mux v1.8.1/go.mod h1:AKf9I4AEqPTmMytcMc0KkNouC66V3BtZ4qD5fmWSiMQ=
-github.com/grafana/dskit v0.0.0-20240925193654-7c41a4057319 h1:KACpOOTqA4WqyyKF2fFPQFiaSOpZdOT5f5gg0qkPLiU=
-github.com/grafana/dskit v0.0.0-20240925193654-7c41a4057319/go.mod h1:SPLNCARd4xdjCkue0O6hvuoveuS1dGJjDnfxYe405YQ=
+github.com/grafana/dskit v0.0.0-20240930165212-f52de24af9bc h1:OLRT3mpHjvTjq4Km7L36lipZ+hTOX1U3jct0DFMADdo=
+github.com/grafana/dskit v0.0.0-20240930165212-f52de24af9bc/go.mod h1:SPLNCARd4xdjCkue0O6hvuoveuS1dGJjDnfxYe405YQ=
 github.com/grafana/gomemcache v0.0.0-20240229205252-cd6a66d6fb56 h1:X8IKQ0wu40wpvYcKfBcc5T4QnhdQjUhtUtB/1CY89lE=
 github.com/grafana/gomemcache v0.0.0-20240229205252-cd6a66d6fb56/go.mod h1:PGk3RjYHpxMM8HFPhKKo+vve3DdlPUELZLSDEFehPuU=
 github.com/grafana/jsonparser v0.0.0-20240425183733-ea80629e1a32 h1:NznuPwItog+rwdVg8hAuGKP29ndRSzJAwhxKldkP8oQ=
diff --git a/vendor/github.com/IBM/ibm-cos-sdk-go/aws/version.go b/vendor/github.com/IBM/ibm-cos-sdk-go/aws/version.go
index 12babed70e3d3..982d490db1aee 100644
--- a/vendor/github.com/IBM/ibm-cos-sdk-go/aws/version.go
+++ b/vendor/github.com/IBM/ibm-cos-sdk-go/aws/version.go
@@ -7,6 +7,6 @@ package aws
 const SDKName = "ibm-cos-sdk-go"
 
 // SDKVersion is the version of this SDK
-const SDKVersion = "1.11.0"
+const SDKVersion = "1.11.1"
 
 // IBM COS SDK Code -- END
diff --git a/vendor/github.com/baidubce/bce-sdk-go/bce/config.go b/vendor/github.com/baidubce/bce-sdk-go/bce/config.go
index 44f7b2fab77d5..f4871a3c92b5f 100644
--- a/vendor/github.com/baidubce/bce-sdk-go/bce/config.go
+++ b/vendor/github.com/baidubce/bce-sdk-go/bce/config.go
@@ -26,7 +26,7 @@ import (
 
 // Constants and default values for the package bce
 const (
-	SDK_VERSION                          = "0.9.189"
+	SDK_VERSION                          = "0.9.192"
 	URI_PREFIX                           = "/" // now support uri without prefix "v1" so just set root path
 	DEFAULT_DOMAIN                       = "baidubce.com"
 	DEFAULT_PROTOCOL                     = "http"
diff --git a/vendor/github.com/baidubce/bce-sdk-go/http/constants.go b/vendor/github.com/baidubce/bce-sdk-go/http/constants.go
index f362a19ddd0ba..fbbea847db9c4 100644
--- a/vendor/github.com/baidubce/bce-sdk-go/http/constants.go
+++ b/vendor/github.com/baidubce/bce-sdk-go/http/constants.go
@@ -85,4 +85,5 @@ const (
 	BCE_TRAFFIC_LIMIT                   = "x-bce-traffic-limit"
 	BCE_BUCKET_TYPE                     = "x-bce-bucket-type"
 	BCE_OBJECT_TAGGING                  = "x-bce-tagging"
+	BCE_FETCH_CALLBACK_ADDRESS          = "x-bce-callback-address"
 )
diff --git a/vendor/github.com/baidubce/bce-sdk-go/services/bos/api/model.go b/vendor/github.com/baidubce/bce-sdk-go/services/bos/api/model.go
index 914fa21d16d4b..8a529cbd8d65a 100644
--- a/vendor/github.com/baidubce/bce-sdk-go/services/bos/api/model.go
+++ b/vendor/github.com/baidubce/bce-sdk-go/services/bos/api/model.go
@@ -394,6 +394,7 @@ type EndMessage struct {
 type FetchObjectArgs struct {
 	FetchMode    string
 	StorageClass string
+	FetchCallBackAddress string
 }
 
 // FetchObjectResult defines the result json structure for the fetch object api.
diff --git a/vendor/github.com/baidubce/bce-sdk-go/services/bos/api/multipart.go b/vendor/github.com/baidubce/bce-sdk-go/services/bos/api/multipart.go
index 0f86fa6404515..94294093c292f 100644
--- a/vendor/github.com/baidubce/bce-sdk-go/services/bos/api/multipart.go
+++ b/vendor/github.com/baidubce/bce-sdk-go/services/bos/api/multipart.go
@@ -202,7 +202,7 @@ func UploadPartFromBytes(cli bce.Client, bucket, object, uploadId string, partNu
 	}
 	// Send request and get the result
 	resp := &bce.BceResponse{}
-	if err := cli.SendRequestFromBytes(req, resp, content); err != nil {
+	if err := SendRequestFromBytes(cli, req, resp, ctx, content); err != nil {
 		return "", err
 	}
 	if resp.IsFail() {
diff --git a/vendor/github.com/baidubce/bce-sdk-go/services/bos/api/object.go b/vendor/github.com/baidubce/bce-sdk-go/services/bos/api/object.go
index b4f2269b9ffb8..68b7ca1ec986b 100644
--- a/vendor/github.com/baidubce/bce-sdk-go/services/bos/api/object.go
+++ b/vendor/github.com/baidubce/bce-sdk-go/services/bos/api/object.go
@@ -530,6 +530,9 @@ func FetchObject(cli bce.Client, bucket, object, source string,
 					args.StorageClass)
 			}
 		}
+		if len(args.FetchCallBackAddress) != 0 {
+			req.SetHeader(http.BCE_FETCH_CALLBACK_ADDRESS, args.FetchCallBackAddress)
+		}
 	}
 
 	// Send request and get the result
diff --git a/vendor/github.com/baidubce/bce-sdk-go/services/bos/api/util.go b/vendor/github.com/baidubce/bce-sdk-go/services/bos/api/util.go
index 31ba002b7e089..13a0399503cd4 100644
--- a/vendor/github.com/baidubce/bce-sdk-go/services/bos/api/util.go
+++ b/vendor/github.com/baidubce/bce-sdk-go/services/bos/api/util.go
@@ -300,6 +300,35 @@ func SendRequest(cli bce.Client, req *bce.BceRequest, resp *bce.BceResponse, ctx
 	return err
 }
 
+func SendRequestFromBytes(cli bce.Client, req *bce.BceRequest, resp *bce.BceResponse, ctx *BosContext, content []byte) error {
+	var (
+		err        error
+		need_retry bool
+	)
+	setUriAndEndpoint(cli, req, ctx, cli.GetBceClientConfig().Endpoint)
+	if err = cli.SendRequestFromBytes(req, resp, content); err != nil {
+		if serviceErr, isServiceErr := err.(*bce.BceServiceError); isServiceErr {
+			if serviceErr.StatusCode == net_http.StatusInternalServerError ||
+				serviceErr.StatusCode == net_http.StatusBadGateway ||
+				serviceErr.StatusCode == net_http.StatusServiceUnavailable ||
+				(serviceErr.StatusCode == net_http.StatusBadRequest && serviceErr.Code == "Http400") {
+				need_retry = true
+			}
+		}
+		if _, isClientErr := err.(*bce.BceClientError); isClientErr {
+			need_retry = true
+		}
+		// retry backup endpoint
+		if need_retry && cli.GetBceClientConfig().BackupEndpoint != "" {
+			setUriAndEndpoint(cli, req, ctx, cli.GetBceClientConfig().BackupEndpoint)
+			if err = cli.SendRequestFromBytes(req, resp, content); err != nil {
+				return err
+			}
+		}
+	}
+	return err
+}
+
 func isVirtualHost(host string) bool {
 	domain := getDomainWithoutPort(host)
 	arr := strings.Split(domain, ".")
diff --git a/vendor/github.com/baidubce/bce-sdk-go/services/bos/client.go b/vendor/github.com/baidubce/bce-sdk-go/services/bos/client.go
index aec79b24b2f06..2c4f44c9bd656 100644
--- a/vendor/github.com/baidubce/bce-sdk-go/services/bos/client.go
+++ b/vendor/github.com/baidubce/bce-sdk-go/services/bos/client.go
@@ -850,7 +850,7 @@ func (c *Client) PutObject(bucket, object string, body *bce.Body,
 //     - string: etag of the uploaded object
 //     - error: the uploaded error if any occurs
 func (c *Client) BasicPutObject(bucket, object string, body *bce.Body) (string, error) {
- 	etag , _, err := api.PutObject(c, bucket, object, body, nil, c.BosContext)
+	etag, _, err := api.PutObject(c, bucket, object, body, nil, c.BosContext)
 	return etag, err
 }
 
@@ -891,7 +891,7 @@ func (c *Client) PutObjectFromString(bucket, object, content string,
 	if err != nil {
 		return "", err
 	}
-	etag , _, err := api.PutObject(c, bucket, object, body, args, c.BosContext)
+	etag, _, err := api.PutObject(c, bucket, object, body, args, c.BosContext)
 	return etag, err
 
 }
@@ -912,7 +912,7 @@ func (c *Client) PutObjectFromFile(bucket, object, fileName string,
 	if err != nil {
 		return "", err
 	}
-	etag , _, err := api.PutObject(c, bucket, object, body, args, c.BosContext)
+	etag, _, err := api.PutObject(c, bucket, object, body, args, c.BosContext)
 	return etag, err
 }
 
@@ -932,7 +932,7 @@ func (c *Client) PutObjectFromStream(bucket, object string, reader io.Reader,
 	if err != nil {
 		return "", err
 	}
-	etag , _, err := api.PutObject(c, bucket, object, body, args, c.BosContext)
+	etag, _, err := api.PutObject(c, bucket, object, body, args, c.BosContext)
 	return etag, err
 }
 
@@ -1113,7 +1113,7 @@ func (c *Client) BasicFetchObject(bucket, object, source string) (*api.FetchObje
 //     - error: any error if it occurs
 func (c *Client) SimpleFetchObject(bucket, object, source, mode,
 	storageClass string) (*api.FetchObjectResult, error) {
-	args := &api.FetchObjectArgs{mode, storageClass}
+	args := &api.FetchObjectArgs{mode, storageClass, ""}
 	return api.FetchObject(c, bucket, object, source, args, c.BosContext)
 }
 
diff --git a/vendor/github.com/felixge/fgprof/fgprof.go b/vendor/github.com/felixge/fgprof/fgprof.go
index e4af2e4052bf8..459787b6f62a9 100644
--- a/vendor/github.com/felixge/fgprof/fgprof.go
+++ b/vendor/github.com/felixge/fgprof/fgprof.go
@@ -81,6 +81,11 @@ type profiler struct {
 	selfFrame *runtime.Frame
 }
 
+// nullTerminationWorkaround deals with a regression in go1.23, see:
+// - https://github.com/felixge/fgprof/issues/33
+// - https://go-review.googlesource.com/c/go/+/609815
+var nullTerminationWorkaround = runtime.Version() == "go1.23.0"
+
 // GoroutineProfile returns the stacks of all goroutines currently managed by
 // the scheduler. This includes both goroutines that are currently running
 // (On-CPU), as well as waiting (Off-CPU).
@@ -107,6 +112,11 @@ func (p *profiler) GoroutineProfile() []runtime.StackRecord {
 	// p.stacks dynamically as well, but let's not over-engineer this until we
 	// understand those cases better.
 	for {
+		if nullTerminationWorkaround {
+			for i := range p.stacks {
+				p.stacks[i].Stack0 = [32]uintptr{}
+			}
+		}
 		n, ok := runtime.GoroutineProfile(p.stacks)
 		if !ok {
 			p.stacks = make([]runtime.StackRecord, int(float64(n)*1.1))
diff --git a/vendor/golang.org/x/crypto/argon2/blamka_amd64.s b/vendor/golang.org/x/crypto/argon2/blamka_amd64.s
index 6713accac09bf..c3895478ed0c5 100644
--- a/vendor/golang.org/x/crypto/argon2/blamka_amd64.s
+++ b/vendor/golang.org/x/crypto/argon2/blamka_amd64.s
@@ -1,243 +1,2791 @@
-// Copyright 2017 The Go Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
+// Code generated by command: go run blamka_amd64.go -out ../blamka_amd64.s -pkg argon2. DO NOT EDIT.
 
 //go:build amd64 && gc && !purego
 
 #include "textflag.h"
 
-DATA ·c40<>+0x00(SB)/8, $0x0201000706050403
-DATA ·c40<>+0x08(SB)/8, $0x0a09080f0e0d0c0b
-GLOBL ·c40<>(SB), (NOPTR+RODATA), $16
-
-DATA ·c48<>+0x00(SB)/8, $0x0100070605040302
-DATA ·c48<>+0x08(SB)/8, $0x09080f0e0d0c0b0a
-GLOBL ·c48<>(SB), (NOPTR+RODATA), $16
-
-#define SHUFFLE(v2, v3, v4, v5, v6, v7, t1, t2) \
-	MOVO       v4, t1; \
-	MOVO       v5, v4; \
-	MOVO       t1, v5; \
-	MOVO       v6, t1; \
-	PUNPCKLQDQ v6, t2; \
-	PUNPCKHQDQ v7, v6; \
-	PUNPCKHQDQ t2, v6; \
-	PUNPCKLQDQ v7, t2; \
-	MOVO       t1, v7; \
-	MOVO       v2, t1; \
-	PUNPCKHQDQ t2, v7; \
-	PUNPCKLQDQ v3, t2; \
-	PUNPCKHQDQ t2, v2; \
-	PUNPCKLQDQ t1, t2; \
-	PUNPCKHQDQ t2, v3
-
-#define SHUFFLE_INV(v2, v3, v4, v5, v6, v7, t1, t2) \
-	MOVO       v4, t1; \
-	MOVO       v5, v4; \
-	MOVO       t1, v5; \
-	MOVO       v2, t1; \
-	PUNPCKLQDQ v2, t2; \
-	PUNPCKHQDQ v3, v2; \
-	PUNPCKHQDQ t2, v2; \
-	PUNPCKLQDQ v3, t2; \
-	MOVO       t1, v3; \
-	MOVO       v6, t1; \
-	PUNPCKHQDQ t2, v3; \
-	PUNPCKLQDQ v7, t2; \
-	PUNPCKHQDQ t2, v6; \
-	PUNPCKLQDQ t1, t2; \
-	PUNPCKHQDQ t2, v7
-
-#define HALF_ROUND(v0, v1, v2, v3, v4, v5, v6, v7, t0, c40, c48) \
-	MOVO    v0, t0;        \
-	PMULULQ v2, t0;        \
-	PADDQ   v2, v0;        \
-	PADDQ   t0, v0;        \
-	PADDQ   t0, v0;        \
-	PXOR    v0, v6;        \
-	PSHUFD  $0xB1, v6, v6; \
-	MOVO    v4, t0;        \
-	PMULULQ v6, t0;        \
-	PADDQ   v6, v4;        \
-	PADDQ   t0, v4;        \
-	PADDQ   t0, v4;        \
-	PXOR    v4, v2;        \
-	PSHUFB  c40, v2;       \
-	MOVO    v0, t0;        \
-	PMULULQ v2, t0;        \
-	PADDQ   v2, v0;        \
-	PADDQ   t0, v0;        \
-	PADDQ   t0, v0;        \
-	PXOR    v0, v6;        \
-	PSHUFB  c48, v6;       \
-	MOVO    v4, t0;        \
-	PMULULQ v6, t0;        \
-	PADDQ   v6, v4;        \
-	PADDQ   t0, v4;        \
-	PADDQ   t0, v4;        \
-	PXOR    v4, v2;        \
-	MOVO    v2, t0;        \
-	PADDQ   v2, t0;        \
-	PSRLQ   $63, v2;       \
-	PXOR    t0, v2;        \
-	MOVO    v1, t0;        \
-	PMULULQ v3, t0;        \
-	PADDQ   v3, v1;        \
-	PADDQ   t0, v1;        \
-	PADDQ   t0, v1;        \
-	PXOR    v1, v7;        \
-	PSHUFD  $0xB1, v7, v7; \
-	MOVO    v5, t0;        \
-	PMULULQ v7, t0;        \
-	PADDQ   v7, v5;        \
-	PADDQ   t0, v5;        \
-	PADDQ   t0, v5;        \
-	PXOR    v5, v3;        \
-	PSHUFB  c40, v3;       \
-	MOVO    v1, t0;        \
-	PMULULQ v3, t0;        \
-	PADDQ   v3, v1;        \
-	PADDQ   t0, v1;        \
-	PADDQ   t0, v1;        \
-	PXOR    v1, v7;        \
-	PSHUFB  c48, v7;       \
-	MOVO    v5, t0;        \
-	PMULULQ v7, t0;        \
-	PADDQ   v7, v5;        \
-	PADDQ   t0, v5;        \
-	PADDQ   t0, v5;        \
-	PXOR    v5, v3;        \
-	MOVO    v3, t0;        \
-	PADDQ   v3, t0;        \
-	PSRLQ   $63, v3;       \
-	PXOR    t0, v3
-
-#define LOAD_MSG_0(block, off) \
-	MOVOU 8*(off+0)(block), X0;  \
-	MOVOU 8*(off+2)(block), X1;  \
-	MOVOU 8*(off+4)(block), X2;  \
-	MOVOU 8*(off+6)(block), X3;  \
-	MOVOU 8*(off+8)(block), X4;  \
-	MOVOU 8*(off+10)(block), X5; \
-	MOVOU 8*(off+12)(block), X6; \
-	MOVOU 8*(off+14)(block), X7
-
-#define STORE_MSG_0(block, off) \
-	MOVOU X0, 8*(off+0)(block);  \
-	MOVOU X1, 8*(off+2)(block);  \
-	MOVOU X2, 8*(off+4)(block);  \
-	MOVOU X3, 8*(off+6)(block);  \
-	MOVOU X4, 8*(off+8)(block);  \
-	MOVOU X5, 8*(off+10)(block); \
-	MOVOU X6, 8*(off+12)(block); \
-	MOVOU X7, 8*(off+14)(block)
-
-#define LOAD_MSG_1(block, off) \
-	MOVOU 8*off+0*8(block), X0;  \
-	MOVOU 8*off+16*8(block), X1; \
-	MOVOU 8*off+32*8(block), X2; \
-	MOVOU 8*off+48*8(block), X3; \
-	MOVOU 8*off+64*8(block), X4; \
-	MOVOU 8*off+80*8(block), X5; \
-	MOVOU 8*off+96*8(block), X6; \
-	MOVOU 8*off+112*8(block), X7
-
-#define STORE_MSG_1(block, off) \
-	MOVOU X0, 8*off+0*8(block);  \
-	MOVOU X1, 8*off+16*8(block); \
-	MOVOU X2, 8*off+32*8(block); \
-	MOVOU X3, 8*off+48*8(block); \
-	MOVOU X4, 8*off+64*8(block); \
-	MOVOU X5, 8*off+80*8(block); \
-	MOVOU X6, 8*off+96*8(block); \
-	MOVOU X7, 8*off+112*8(block)
-
-#define BLAMKA_ROUND_0(block, off, t0, t1, c40, c48) \
-	LOAD_MSG_0(block, off);                                   \
-	HALF_ROUND(X0, X1, X2, X3, X4, X5, X6, X7, t0, c40, c48); \
-	SHUFFLE(X2, X3, X4, X5, X6, X7, t0, t1);                  \
-	HALF_ROUND(X0, X1, X2, X3, X4, X5, X6, X7, t0, c40, c48); \
-	SHUFFLE_INV(X2, X3, X4, X5, X6, X7, t0, t1);              \
-	STORE_MSG_0(block, off)
-
-#define BLAMKA_ROUND_1(block, off, t0, t1, c40, c48) \
-	LOAD_MSG_1(block, off);                                   \
-	HALF_ROUND(X0, X1, X2, X3, X4, X5, X6, X7, t0, c40, c48); \
-	SHUFFLE(X2, X3, X4, X5, X6, X7, t0, t1);                  \
-	HALF_ROUND(X0, X1, X2, X3, X4, X5, X6, X7, t0, c40, c48); \
-	SHUFFLE_INV(X2, X3, X4, X5, X6, X7, t0, t1);              \
-	STORE_MSG_1(block, off)
-
 // func blamkaSSE4(b *block)
-TEXT ·blamkaSSE4(SB), 4, $0-8
-	MOVQ b+0(FP), AX
-
-	MOVOU ·c40<>(SB), X10
-	MOVOU ·c48<>(SB), X11
+// Requires: SSE2, SSSE3
+TEXT ·blamkaSSE4(SB), NOSPLIT, $0-8
+	MOVQ       b+0(FP), AX
+	MOVOU      ·c40<>+0(SB), X10
+	MOVOU      ·c48<>+0(SB), X11
+	MOVOU      (AX), X0
+	MOVOU      16(AX), X1
+	MOVOU      32(AX), X2
+	MOVOU      48(AX), X3
+	MOVOU      64(AX), X4
+	MOVOU      80(AX), X5
+	MOVOU      96(AX), X6
+	MOVOU      112(AX), X7
+	MOVO       X0, X8
+	PMULULQ    X2, X8
+	PADDQ      X2, X0
+	PADDQ      X8, X0
+	PADDQ      X8, X0
+	PXOR       X0, X6
+	PSHUFD     $0xb1, X6, X6
+	MOVO       X4, X8
+	PMULULQ    X6, X8
+	PADDQ      X6, X4
+	PADDQ      X8, X4
+	PADDQ      X8, X4
+	PXOR       X4, X2
+	PSHUFB     X10, X2
+	MOVO       X0, X8
+	PMULULQ    X2, X8
+	PADDQ      X2, X0
+	PADDQ      X8, X0
+	PADDQ      X8, X0
+	PXOR       X0, X6
+	PSHUFB     X11, X6
+	MOVO       X4, X8
+	PMULULQ    X6, X8
+	PADDQ      X6, X4
+	PADDQ      X8, X4
+	PADDQ      X8, X4
+	PXOR       X4, X2
+	MOVO       X2, X8
+	PADDQ      X2, X8
+	PSRLQ      $0x3f, X2
+	PXOR       X8, X2
+	MOVO       X1, X8
+	PMULULQ    X3, X8
+	PADDQ      X3, X1
+	PADDQ      X8, X1
+	PADDQ      X8, X1
+	PXOR       X1, X7
+	PSHUFD     $0xb1, X7, X7
+	MOVO       X5, X8
+	PMULULQ    X7, X8
+	PADDQ      X7, X5
+	PADDQ      X8, X5
+	PADDQ      X8, X5
+	PXOR       X5, X3
+	PSHUFB     X10, X3
+	MOVO       X1, X8
+	PMULULQ    X3, X8
+	PADDQ      X3, X1
+	PADDQ      X8, X1
+	PADDQ      X8, X1
+	PXOR       X1, X7
+	PSHUFB     X11, X7
+	MOVO       X5, X8
+	PMULULQ    X7, X8
+	PADDQ      X7, X5
+	PADDQ      X8, X5
+	PADDQ      X8, X5
+	PXOR       X5, X3
+	MOVO       X3, X8
+	PADDQ      X3, X8
+	PSRLQ      $0x3f, X3
+	PXOR       X8, X3
+	MOVO       X4, X8
+	MOVO       X5, X4
+	MOVO       X8, X5
+	MOVO       X6, X8
+	PUNPCKLQDQ X6, X9
+	PUNPCKHQDQ X7, X6
+	PUNPCKHQDQ X9, X6
+	PUNPCKLQDQ X7, X9
+	MOVO       X8, X7
+	MOVO       X2, X8
+	PUNPCKHQDQ X9, X7
+	PUNPCKLQDQ X3, X9
+	PUNPCKHQDQ X9, X2
+	PUNPCKLQDQ X8, X9
+	PUNPCKHQDQ X9, X3
+	MOVO       X0, X8
+	PMULULQ    X2, X8
+	PADDQ      X2, X0
+	PADDQ      X8, X0
+	PADDQ      X8, X0
+	PXOR       X0, X6
+	PSHUFD     $0xb1, X6, X6
+	MOVO       X4, X8
+	PMULULQ    X6, X8
+	PADDQ      X6, X4
+	PADDQ      X8, X4
+	PADDQ      X8, X4
+	PXOR       X4, X2
+	PSHUFB     X10, X2
+	MOVO       X0, X8
+	PMULULQ    X2, X8
+	PADDQ      X2, X0
+	PADDQ      X8, X0
+	PADDQ      X8, X0
+	PXOR       X0, X6
+	PSHUFB     X11, X6
+	MOVO       X4, X8
+	PMULULQ    X6, X8
+	PADDQ      X6, X4
+	PADDQ      X8, X4
+	PADDQ      X8, X4
+	PXOR       X4, X2
+	MOVO       X2, X8
+	PADDQ      X2, X8
+	PSRLQ      $0x3f, X2
+	PXOR       X8, X2
+	MOVO       X1, X8
+	PMULULQ    X3, X8
+	PADDQ      X3, X1
+	PADDQ      X8, X1
+	PADDQ      X8, X1
+	PXOR       X1, X7
+	PSHUFD     $0xb1, X7, X7
+	MOVO       X5, X8
+	PMULULQ    X7, X8
+	PADDQ      X7, X5
+	PADDQ      X8, X5
+	PADDQ      X8, X5
+	PXOR       X5, X3
+	PSHUFB     X10, X3
+	MOVO       X1, X8
+	PMULULQ    X3, X8
+	PADDQ      X3, X1
+	PADDQ      X8, X1
+	PADDQ      X8, X1
+	PXOR       X1, X7
+	PSHUFB     X11, X7
+	MOVO       X5, X8
+	PMULULQ    X7, X8
+	PADDQ      X7, X5
+	PADDQ      X8, X5
+	PADDQ      X8, X5
+	PXOR       X5, X3
+	MOVO       X3, X8
+	PADDQ      X3, X8
+	PSRLQ      $0x3f, X3
+	PXOR       X8, X3
+	MOVO       X4, X8
+	MOVO       X5, X4
+	MOVO       X8, X5
+	MOVO       X2, X8
+	PUNPCKLQDQ X2, X9
+	PUNPCKHQDQ X3, X2
+	PUNPCKHQDQ X9, X2
+	PUNPCKLQDQ X3, X9
+	MOVO       X8, X3
+	MOVO       X6, X8
+	PUNPCKHQDQ X9, X3
+	PUNPCKLQDQ X7, X9
+	PUNPCKHQDQ X9, X6
+	PUNPCKLQDQ X8, X9
+	PUNPCKHQDQ X9, X7
+	MOVOU      X0, (AX)
+	MOVOU      X1, 16(AX)
+	MOVOU      X2, 32(AX)
+	MOVOU      X3, 48(AX)
+	MOVOU      X4, 64(AX)
+	MOVOU      X5, 80(AX)
+	MOVOU      X6, 96(AX)
+	MOVOU      X7, 112(AX)
+	MOVOU      128(AX), X0
+	MOVOU      144(AX), X1
+	MOVOU      160(AX), X2
+	MOVOU      176(AX), X3
+	MOVOU      192(AX), X4
+	MOVOU      208(AX), X5
+	MOVOU      224(AX), X6
+	MOVOU      240(AX), X7
+	MOVO       X0, X8
+	PMULULQ    X2, X8
+	PADDQ      X2, X0
+	PADDQ      X8, X0
+	PADDQ      X8, X0
+	PXOR       X0, X6
+	PSHUFD     $0xb1, X6, X6
+	MOVO       X4, X8
+	PMULULQ    X6, X8
+	PADDQ      X6, X4
+	PADDQ      X8, X4
+	PADDQ      X8, X4
+	PXOR       X4, X2
+	PSHUFB     X10, X2
+	MOVO       X0, X8
+	PMULULQ    X2, X8
+	PADDQ      X2, X0
+	PADDQ      X8, X0
+	PADDQ      X8, X0
+	PXOR       X0, X6
+	PSHUFB     X11, X6
+	MOVO       X4, X8
+	PMULULQ    X6, X8
+	PADDQ      X6, X4
+	PADDQ      X8, X4
+	PADDQ      X8, X4
+	PXOR       X4, X2
+	MOVO       X2, X8
+	PADDQ      X2, X8
+	PSRLQ      $0x3f, X2
+	PXOR       X8, X2
+	MOVO       X1, X8
+	PMULULQ    X3, X8
+	PADDQ      X3, X1
+	PADDQ      X8, X1
+	PADDQ      X8, X1
+	PXOR       X1, X7
+	PSHUFD     $0xb1, X7, X7
+	MOVO       X5, X8
+	PMULULQ    X7, X8
+	PADDQ      X7, X5
+	PADDQ      X8, X5
+	PADDQ      X8, X5
+	PXOR       X5, X3
+	PSHUFB     X10, X3
+	MOVO       X1, X8
+	PMULULQ    X3, X8
+	PADDQ      X3, X1
+	PADDQ      X8, X1
+	PADDQ      X8, X1
+	PXOR       X1, X7
+	PSHUFB     X11, X7
+	MOVO       X5, X8
+	PMULULQ    X7, X8
+	PADDQ      X7, X5
+	PADDQ      X8, X5
+	PADDQ      X8, X5
+	PXOR       X5, X3
+	MOVO       X3, X8
+	PADDQ      X3, X8
+	PSRLQ      $0x3f, X3
+	PXOR       X8, X3
+	MOVO       X4, X8
+	MOVO       X5, X4
+	MOVO       X8, X5
+	MOVO       X6, X8
+	PUNPCKLQDQ X6, X9
+	PUNPCKHQDQ X7, X6
+	PUNPCKHQDQ X9, X6
+	PUNPCKLQDQ X7, X9
+	MOVO       X8, X7
+	MOVO       X2, X8
+	PUNPCKHQDQ X9, X7
+	PUNPCKLQDQ X3, X9
+	PUNPCKHQDQ X9, X2
+	PUNPCKLQDQ X8, X9
+	PUNPCKHQDQ X9, X3
+	MOVO       X0, X8
+	PMULULQ    X2, X8
+	PADDQ      X2, X0
+	PADDQ      X8, X0
+	PADDQ      X8, X0
+	PXOR       X0, X6
+	PSHUFD     $0xb1, X6, X6
+	MOVO       X4, X8
+	PMULULQ    X6, X8
+	PADDQ      X6, X4
+	PADDQ      X8, X4
+	PADDQ      X8, X4
+	PXOR       X4, X2
+	PSHUFB     X10, X2
+	MOVO       X0, X8
+	PMULULQ    X2, X8
+	PADDQ      X2, X0
+	PADDQ      X8, X0
+	PADDQ      X8, X0
+	PXOR       X0, X6
+	PSHUFB     X11, X6
+	MOVO       X4, X8
+	PMULULQ    X6, X8
+	PADDQ      X6, X4
+	PADDQ      X8, X4
+	PADDQ      X8, X4
+	PXOR       X4, X2
+	MOVO       X2, X8
+	PADDQ      X2, X8
+	PSRLQ      $0x3f, X2
+	PXOR       X8, X2
+	MOVO       X1, X8
+	PMULULQ    X3, X8
+	PADDQ      X3, X1
+	PADDQ      X8, X1
+	PADDQ      X8, X1
+	PXOR       X1, X7
+	PSHUFD     $0xb1, X7, X7
+	MOVO       X5, X8
+	PMULULQ    X7, X8
+	PADDQ      X7, X5
+	PADDQ      X8, X5
+	PADDQ      X8, X5
+	PXOR       X5, X3
+	PSHUFB     X10, X3
+	MOVO       X1, X8
+	PMULULQ    X3, X8
+	PADDQ      X3, X1
+	PADDQ      X8, X1
+	PADDQ      X8, X1
+	PXOR       X1, X7
+	PSHUFB     X11, X7
+	MOVO       X5, X8
+	PMULULQ    X7, X8
+	PADDQ      X7, X5
+	PADDQ      X8, X5
+	PADDQ      X8, X5
+	PXOR       X5, X3
+	MOVO       X3, X8
+	PADDQ      X3, X8
+	PSRLQ      $0x3f, X3
+	PXOR       X8, X3
+	MOVO       X4, X8
+	MOVO       X5, X4
+	MOVO       X8, X5
+	MOVO       X2, X8
+	PUNPCKLQDQ X2, X9
+	PUNPCKHQDQ X3, X2
+	PUNPCKHQDQ X9, X2
+	PUNPCKLQDQ X3, X9
+	MOVO       X8, X3
+	MOVO       X6, X8
+	PUNPCKHQDQ X9, X3
+	PUNPCKLQDQ X7, X9
+	PUNPCKHQDQ X9, X6
+	PUNPCKLQDQ X8, X9
+	PUNPCKHQDQ X9, X7
+	MOVOU      X0, 128(AX)
+	MOVOU      X1, 144(AX)
+	MOVOU      X2, 160(AX)
+	MOVOU      X3, 176(AX)
+	MOVOU      X4, 192(AX)
+	MOVOU      X5, 208(AX)
+	MOVOU      X6, 224(AX)
+	MOVOU      X7, 240(AX)
+	MOVOU      256(AX), X0
+	MOVOU      272(AX), X1
+	MOVOU      288(AX), X2
+	MOVOU      304(AX), X3
+	MOVOU      320(AX), X4
+	MOVOU      336(AX), X5
+	MOVOU      352(AX), X6
+	MOVOU      368(AX), X7
+	MOVO       X0, X8
+	PMULULQ    X2, X8
+	PADDQ      X2, X0
+	PADDQ      X8, X0
+	PADDQ      X8, X0
+	PXOR       X0, X6
+	PSHUFD     $0xb1, X6, X6
+	MOVO       X4, X8
+	PMULULQ    X6, X8
+	PADDQ      X6, X4
+	PADDQ      X8, X4
+	PADDQ      X8, X4
+	PXOR       X4, X2
+	PSHUFB     X10, X2
+	MOVO       X0, X8
+	PMULULQ    X2, X8
+	PADDQ      X2, X0
+	PADDQ      X8, X0
+	PADDQ      X8, X0
+	PXOR       X0, X6
+	PSHUFB     X11, X6
+	MOVO       X4, X8
+	PMULULQ    X6, X8
+	PADDQ      X6, X4
+	PADDQ      X8, X4
+	PADDQ      X8, X4
+	PXOR       X4, X2
+	MOVO       X2, X8
+	PADDQ      X2, X8
+	PSRLQ      $0x3f, X2
+	PXOR       X8, X2
+	MOVO       X1, X8
+	PMULULQ    X3, X8
+	PADDQ      X3, X1
+	PADDQ      X8, X1
+	PADDQ      X8, X1
+	PXOR       X1, X7
+	PSHUFD     $0xb1, X7, X7
+	MOVO       X5, X8
+	PMULULQ    X7, X8
+	PADDQ      X7, X5
+	PADDQ      X8, X5
+	PADDQ      X8, X5
+	PXOR       X5, X3
+	PSHUFB     X10, X3
+	MOVO       X1, X8
+	PMULULQ    X3, X8
+	PADDQ      X3, X1
+	PADDQ      X8, X1
+	PADDQ      X8, X1
+	PXOR       X1, X7
+	PSHUFB     X11, X7
+	MOVO       X5, X8
+	PMULULQ    X7, X8
+	PADDQ      X7, X5
+	PADDQ      X8, X5
+	PADDQ      X8, X5
+	PXOR       X5, X3
+	MOVO       X3, X8
+	PADDQ      X3, X8
+	PSRLQ      $0x3f, X3
+	PXOR       X8, X3
+	MOVO       X4, X8
+	MOVO       X5, X4
+	MOVO       X8, X5
+	MOVO       X6, X8
+	PUNPCKLQDQ X6, X9
+	PUNPCKHQDQ X7, X6
+	PUNPCKHQDQ X9, X6
+	PUNPCKLQDQ X7, X9
+	MOVO       X8, X7
+	MOVO       X2, X8
+	PUNPCKHQDQ X9, X7
+	PUNPCKLQDQ X3, X9
+	PUNPCKHQDQ X9, X2
+	PUNPCKLQDQ X8, X9
+	PUNPCKHQDQ X9, X3
+	MOVO       X0, X8
+	PMULULQ    X2, X8
+	PADDQ      X2, X0
+	PADDQ      X8, X0
+	PADDQ      X8, X0
+	PXOR       X0, X6
+	PSHUFD     $0xb1, X6, X6
+	MOVO       X4, X8
+	PMULULQ    X6, X8
+	PADDQ      X6, X4
+	PADDQ      X8, X4
+	PADDQ      X8, X4
+	PXOR       X4, X2
+	PSHUFB     X10, X2
+	MOVO       X0, X8
+	PMULULQ    X2, X8
+	PADDQ      X2, X0
+	PADDQ      X8, X0
+	PADDQ      X8, X0
+	PXOR       X0, X6
+	PSHUFB     X11, X6
+	MOVO       X4, X8
+	PMULULQ    X6, X8
+	PADDQ      X6, X4
+	PADDQ      X8, X4
+	PADDQ      X8, X4
+	PXOR       X4, X2
+	MOVO       X2, X8
+	PADDQ      X2, X8
+	PSRLQ      $0x3f, X2
+	PXOR       X8, X2
+	MOVO       X1, X8
+	PMULULQ    X3, X8
+	PADDQ      X3, X1
+	PADDQ      X8, X1
+	PADDQ      X8, X1
+	PXOR       X1, X7
+	PSHUFD     $0xb1, X7, X7
+	MOVO       X5, X8
+	PMULULQ    X7, X8
+	PADDQ      X7, X5
+	PADDQ      X8, X5
+	PADDQ      X8, X5
+	PXOR       X5, X3
+	PSHUFB     X10, X3
+	MOVO       X1, X8
+	PMULULQ    X3, X8
+	PADDQ      X3, X1
+	PADDQ      X8, X1
+	PADDQ      X8, X1
+	PXOR       X1, X7
+	PSHUFB     X11, X7
+	MOVO       X5, X8
+	PMULULQ    X7, X8
+	PADDQ      X7, X5
+	PADDQ      X8, X5
+	PADDQ      X8, X5
+	PXOR       X5, X3
+	MOVO       X3, X8
+	PADDQ      X3, X8
+	PSRLQ      $0x3f, X3
+	PXOR       X8, X3
+	MOVO       X4, X8
+	MOVO       X5, X4
+	MOVO       X8, X5
+	MOVO       X2, X8
+	PUNPCKLQDQ X2, X9
+	PUNPCKHQDQ X3, X2
+	PUNPCKHQDQ X9, X2
+	PUNPCKLQDQ X3, X9
+	MOVO       X8, X3
+	MOVO       X6, X8
+	PUNPCKHQDQ X9, X3
+	PUNPCKLQDQ X7, X9
+	PUNPCKHQDQ X9, X6
+	PUNPCKLQDQ X8, X9
+	PUNPCKHQDQ X9, X7
+	MOVOU      X0, 256(AX)
+	MOVOU      X1, 272(AX)
+	MOVOU      X2, 288(AX)
+	MOVOU      X3, 304(AX)
+	MOVOU      X4, 320(AX)
+	MOVOU      X5, 336(AX)
+	MOVOU      X6, 352(AX)
+	MOVOU      X7, 368(AX)
+	MOVOU      384(AX), X0
+	MOVOU      400(AX), X1
+	MOVOU      416(AX), X2
+	MOVOU      432(AX), X3
+	MOVOU      448(AX), X4
+	MOVOU      464(AX), X5
+	MOVOU      480(AX), X6
+	MOVOU      496(AX), X7
+	MOVO       X0, X8
+	PMULULQ    X2, X8
+	PADDQ      X2, X0
+	PADDQ      X8, X0
+	PADDQ      X8, X0
+	PXOR       X0, X6
+	PSHUFD     $0xb1, X6, X6
+	MOVO       X4, X8
+	PMULULQ    X6, X8
+	PADDQ      X6, X4
+	PADDQ      X8, X4
+	PADDQ      X8, X4
+	PXOR       X4, X2
+	PSHUFB     X10, X2
+	MOVO       X0, X8
+	PMULULQ    X2, X8
+	PADDQ      X2, X0
+	PADDQ      X8, X0
+	PADDQ      X8, X0
+	PXOR       X0, X6
+	PSHUFB     X11, X6
+	MOVO       X4, X8
+	PMULULQ    X6, X8
+	PADDQ      X6, X4
+	PADDQ      X8, X4
+	PADDQ      X8, X4
+	PXOR       X4, X2
+	MOVO       X2, X8
+	PADDQ      X2, X8
+	PSRLQ      $0x3f, X2
+	PXOR       X8, X2
+	MOVO       X1, X8
+	PMULULQ    X3, X8
+	PADDQ      X3, X1
+	PADDQ      X8, X1
+	PADDQ      X8, X1
+	PXOR       X1, X7
+	PSHUFD     $0xb1, X7, X7
+	MOVO       X5, X8
+	PMULULQ    X7, X8
+	PADDQ      X7, X5
+	PADDQ      X8, X5
+	PADDQ      X8, X5
+	PXOR       X5, X3
+	PSHUFB     X10, X3
+	MOVO       X1, X8
+	PMULULQ    X3, X8
+	PADDQ      X3, X1
+	PADDQ      X8, X1
+	PADDQ      X8, X1
+	PXOR       X1, X7
+	PSHUFB     X11, X7
+	MOVO       X5, X8
+	PMULULQ    X7, X8
+	PADDQ      X7, X5
+	PADDQ      X8, X5
+	PADDQ      X8, X5
+	PXOR       X5, X3
+	MOVO       X3, X8
+	PADDQ      X3, X8
+	PSRLQ      $0x3f, X3
+	PXOR       X8, X3
+	MOVO       X4, X8
+	MOVO       X5, X4
+	MOVO       X8, X5
+	MOVO       X6, X8
+	PUNPCKLQDQ X6, X9
+	PUNPCKHQDQ X7, X6
+	PUNPCKHQDQ X9, X6
+	PUNPCKLQDQ X7, X9
+	MOVO       X8, X7
+	MOVO       X2, X8
+	PUNPCKHQDQ X9, X7
+	PUNPCKLQDQ X3, X9
+	PUNPCKHQDQ X9, X2
+	PUNPCKLQDQ X8, X9
+	PUNPCKHQDQ X9, X3
+	MOVO       X0, X8
+	PMULULQ    X2, X8
+	PADDQ      X2, X0
+	PADDQ      X8, X0
+	PADDQ      X8, X0
+	PXOR       X0, X6
+	PSHUFD     $0xb1, X6, X6
+	MOVO       X4, X8
+	PMULULQ    X6, X8
+	PADDQ      X6, X4
+	PADDQ      X8, X4
+	PADDQ      X8, X4
+	PXOR       X4, X2
+	PSHUFB     X10, X2
+	MOVO       X0, X8
+	PMULULQ    X2, X8
+	PADDQ      X2, X0
+	PADDQ      X8, X0
+	PADDQ      X8, X0
+	PXOR       X0, X6
+	PSHUFB     X11, X6
+	MOVO       X4, X8
+	PMULULQ    X6, X8
+	PADDQ      X6, X4
+	PADDQ      X8, X4
+	PADDQ      X8, X4
+	PXOR       X4, X2
+	MOVO       X2, X8
+	PADDQ      X2, X8
+	PSRLQ      $0x3f, X2
+	PXOR       X8, X2
+	MOVO       X1, X8
+	PMULULQ    X3, X8
+	PADDQ      X3, X1
+	PADDQ      X8, X1
+	PADDQ      X8, X1
+	PXOR       X1, X7
+	PSHUFD     $0xb1, X7, X7
+	MOVO       X5, X8
+	PMULULQ    X7, X8
+	PADDQ      X7, X5
+	PADDQ      X8, X5
+	PADDQ      X8, X5
+	PXOR       X5, X3
+	PSHUFB     X10, X3
+	MOVO       X1, X8
+	PMULULQ    X3, X8
+	PADDQ      X3, X1
+	PADDQ      X8, X1
+	PADDQ      X8, X1
+	PXOR       X1, X7
+	PSHUFB     X11, X7
+	MOVO       X5, X8
+	PMULULQ    X7, X8
+	PADDQ      X7, X5
+	PADDQ      X8, X5
+	PADDQ      X8, X5
+	PXOR       X5, X3
+	MOVO       X3, X8
+	PADDQ      X3, X8
+	PSRLQ      $0x3f, X3
+	PXOR       X8, X3
+	MOVO       X4, X8
+	MOVO       X5, X4
+	MOVO       X8, X5
+	MOVO       X2, X8
+	PUNPCKLQDQ X2, X9
+	PUNPCKHQDQ X3, X2
+	PUNPCKHQDQ X9, X2
+	PUNPCKLQDQ X3, X9
+	MOVO       X8, X3
+	MOVO       X6, X8
+	PUNPCKHQDQ X9, X3
+	PUNPCKLQDQ X7, X9
+	PUNPCKHQDQ X9, X6
+	PUNPCKLQDQ X8, X9
+	PUNPCKHQDQ X9, X7
+	MOVOU      X0, 384(AX)
+	MOVOU      X1, 400(AX)
+	MOVOU      X2, 416(AX)
+	MOVOU      X3, 432(AX)
+	MOVOU      X4, 448(AX)
+	MOVOU      X5, 464(AX)
+	MOVOU      X6, 480(AX)
+	MOVOU      X7, 496(AX)
+	MOVOU      512(AX), X0
+	MOVOU      528(AX), X1
+	MOVOU      544(AX), X2
+	MOVOU      560(AX), X3
+	MOVOU      576(AX), X4
+	MOVOU      592(AX), X5
+	MOVOU      608(AX), X6
+	MOVOU      624(AX), X7
+	MOVO       X0, X8
+	PMULULQ    X2, X8
+	PADDQ      X2, X0
+	PADDQ      X8, X0
+	PADDQ      X8, X0
+	PXOR       X0, X6
+	PSHUFD     $0xb1, X6, X6
+	MOVO       X4, X8
+	PMULULQ    X6, X8
+	PADDQ      X6, X4
+	PADDQ      X8, X4
+	PADDQ      X8, X4
+	PXOR       X4, X2
+	PSHUFB     X10, X2
+	MOVO       X0, X8
+	PMULULQ    X2, X8
+	PADDQ      X2, X0
+	PADDQ      X8, X0
+	PADDQ      X8, X0
+	PXOR       X0, X6
+	PSHUFB     X11, X6
+	MOVO       X4, X8
+	PMULULQ    X6, X8
+	PADDQ      X6, X4
+	PADDQ      X8, X4
+	PADDQ      X8, X4
+	PXOR       X4, X2
+	MOVO       X2, X8
+	PADDQ      X2, X8
+	PSRLQ      $0x3f, X2
+	PXOR       X8, X2
+	MOVO       X1, X8
+	PMULULQ    X3, X8
+	PADDQ      X3, X1
+	PADDQ      X8, X1
+	PADDQ      X8, X1
+	PXOR       X1, X7
+	PSHUFD     $0xb1, X7, X7
+	MOVO       X5, X8
+	PMULULQ    X7, X8
+	PADDQ      X7, X5
+	PADDQ      X8, X5
+	PADDQ      X8, X5
+	PXOR       X5, X3
+	PSHUFB     X10, X3
+	MOVO       X1, X8
+	PMULULQ    X3, X8
+	PADDQ      X3, X1
+	PADDQ      X8, X1
+	PADDQ      X8, X1
+	PXOR       X1, X7
+	PSHUFB     X11, X7
+	MOVO       X5, X8
+	PMULULQ    X7, X8
+	PADDQ      X7, X5
+	PADDQ      X8, X5
+	PADDQ      X8, X5
+	PXOR       X5, X3
+	MOVO       X3, X8
+	PADDQ      X3, X8
+	PSRLQ      $0x3f, X3
+	PXOR       X8, X3
+	MOVO       X4, X8
+	MOVO       X5, X4
+	MOVO       X8, X5
+	MOVO       X6, X8
+	PUNPCKLQDQ X6, X9
+	PUNPCKHQDQ X7, X6
+	PUNPCKHQDQ X9, X6
+	PUNPCKLQDQ X7, X9
+	MOVO       X8, X7
+	MOVO       X2, X8
+	PUNPCKHQDQ X9, X7
+	PUNPCKLQDQ X3, X9
+	PUNPCKHQDQ X9, X2
+	PUNPCKLQDQ X8, X9
+	PUNPCKHQDQ X9, X3
+	MOVO       X0, X8
+	PMULULQ    X2, X8
+	PADDQ      X2, X0
+	PADDQ      X8, X0
+	PADDQ      X8, X0
+	PXOR       X0, X6
+	PSHUFD     $0xb1, X6, X6
+	MOVO       X4, X8
+	PMULULQ    X6, X8
+	PADDQ      X6, X4
+	PADDQ      X8, X4
+	PADDQ      X8, X4
+	PXOR       X4, X2
+	PSHUFB     X10, X2
+	MOVO       X0, X8
+	PMULULQ    X2, X8
+	PADDQ      X2, X0
+	PADDQ      X8, X0
+	PADDQ      X8, X0
+	PXOR       X0, X6
+	PSHUFB     X11, X6
+	MOVO       X4, X8
+	PMULULQ    X6, X8
+	PADDQ      X6, X4
+	PADDQ      X8, X4
+	PADDQ      X8, X4
+	PXOR       X4, X2
+	MOVO       X2, X8
+	PADDQ      X2, X8
+	PSRLQ      $0x3f, X2
+	PXOR       X8, X2
+	MOVO       X1, X8
+	PMULULQ    X3, X8
+	PADDQ      X3, X1
+	PADDQ      X8, X1
+	PADDQ      X8, X1
+	PXOR       X1, X7
+	PSHUFD     $0xb1, X7, X7
+	MOVO       X5, X8
+	PMULULQ    X7, X8
+	PADDQ      X7, X5
+	PADDQ      X8, X5
+	PADDQ      X8, X5
+	PXOR       X5, X3
+	PSHUFB     X10, X3
+	MOVO       X1, X8
+	PMULULQ    X3, X8
+	PADDQ      X3, X1
+	PADDQ      X8, X1
+	PADDQ      X8, X1
+	PXOR       X1, X7
+	PSHUFB     X11, X7
+	MOVO       X5, X8
+	PMULULQ    X7, X8
+	PADDQ      X7, X5
+	PADDQ      X8, X5
+	PADDQ      X8, X5
+	PXOR       X5, X3
+	MOVO       X3, X8
+	PADDQ      X3, X8
+	PSRLQ      $0x3f, X3
+	PXOR       X8, X3
+	MOVO       X4, X8
+	MOVO       X5, X4
+	MOVO       X8, X5
+	MOVO       X2, X8
+	PUNPCKLQDQ X2, X9
+	PUNPCKHQDQ X3, X2
+	PUNPCKHQDQ X9, X2
+	PUNPCKLQDQ X3, X9
+	MOVO       X8, X3
+	MOVO       X6, X8
+	PUNPCKHQDQ X9, X3
+	PUNPCKLQDQ X7, X9
+	PUNPCKHQDQ X9, X6
+	PUNPCKLQDQ X8, X9
+	PUNPCKHQDQ X9, X7
+	MOVOU      X0, 512(AX)
+	MOVOU      X1, 528(AX)
+	MOVOU      X2, 544(AX)
+	MOVOU      X3, 560(AX)
+	MOVOU      X4, 576(AX)
+	MOVOU      X5, 592(AX)
+	MOVOU      X6, 608(AX)
+	MOVOU      X7, 624(AX)
+	MOVOU      640(AX), X0
+	MOVOU      656(AX), X1
+	MOVOU      672(AX), X2
+	MOVOU      688(AX), X3
+	MOVOU      704(AX), X4
+	MOVOU      720(AX), X5
+	MOVOU      736(AX), X6
+	MOVOU      752(AX), X7
+	MOVO       X0, X8
+	PMULULQ    X2, X8
+	PADDQ      X2, X0
+	PADDQ      X8, X0
+	PADDQ      X8, X0
+	PXOR       X0, X6
+	PSHUFD     $0xb1, X6, X6
+	MOVO       X4, X8
+	PMULULQ    X6, X8
+	PADDQ      X6, X4
+	PADDQ      X8, X4
+	PADDQ      X8, X4
+	PXOR       X4, X2
+	PSHUFB     X10, X2
+	MOVO       X0, X8
+	PMULULQ    X2, X8
+	PADDQ      X2, X0
+	PADDQ      X8, X0
+	PADDQ      X8, X0
+	PXOR       X0, X6
+	PSHUFB     X11, X6
+	MOVO       X4, X8
+	PMULULQ    X6, X8
+	PADDQ      X6, X4
+	PADDQ      X8, X4
+	PADDQ      X8, X4
+	PXOR       X4, X2
+	MOVO       X2, X8
+	PADDQ      X2, X8
+	PSRLQ      $0x3f, X2
+	PXOR       X8, X2
+	MOVO       X1, X8
+	PMULULQ    X3, X8
+	PADDQ      X3, X1
+	PADDQ      X8, X1
+	PADDQ      X8, X1
+	PXOR       X1, X7
+	PSHUFD     $0xb1, X7, X7
+	MOVO       X5, X8
+	PMULULQ    X7, X8
+	PADDQ      X7, X5
+	PADDQ      X8, X5
+	PADDQ      X8, X5
+	PXOR       X5, X3
+	PSHUFB     X10, X3
+	MOVO       X1, X8
+	PMULULQ    X3, X8
+	PADDQ      X3, X1
+	PADDQ      X8, X1
+	PADDQ      X8, X1
+	PXOR       X1, X7
+	PSHUFB     X11, X7
+	MOVO       X5, X8
+	PMULULQ    X7, X8
+	PADDQ      X7, X5
+	PADDQ      X8, X5
+	PADDQ      X8, X5
+	PXOR       X5, X3
+	MOVO       X3, X8
+	PADDQ      X3, X8
+	PSRLQ      $0x3f, X3
+	PXOR       X8, X3
+	MOVO       X4, X8
+	MOVO       X5, X4
+	MOVO       X8, X5
+	MOVO       X6, X8
+	PUNPCKLQDQ X6, X9
+	PUNPCKHQDQ X7, X6
+	PUNPCKHQDQ X9, X6
+	PUNPCKLQDQ X7, X9
+	MOVO       X8, X7
+	MOVO       X2, X8
+	PUNPCKHQDQ X9, X7
+	PUNPCKLQDQ X3, X9
+	PUNPCKHQDQ X9, X2
+	PUNPCKLQDQ X8, X9
+	PUNPCKHQDQ X9, X3
+	MOVO       X0, X8
+	PMULULQ    X2, X8
+	PADDQ      X2, X0
+	PADDQ      X8, X0
+	PADDQ      X8, X0
+	PXOR       X0, X6
+	PSHUFD     $0xb1, X6, X6
+	MOVO       X4, X8
+	PMULULQ    X6, X8
+	PADDQ      X6, X4
+	PADDQ      X8, X4
+	PADDQ      X8, X4
+	PXOR       X4, X2
+	PSHUFB     X10, X2
+	MOVO       X0, X8
+	PMULULQ    X2, X8
+	PADDQ      X2, X0
+	PADDQ      X8, X0
+	PADDQ      X8, X0
+	PXOR       X0, X6
+	PSHUFB     X11, X6
+	MOVO       X4, X8
+	PMULULQ    X6, X8
+	PADDQ      X6, X4
+	PADDQ      X8, X4
+	PADDQ      X8, X4
+	PXOR       X4, X2
+	MOVO       X2, X8
+	PADDQ      X2, X8
+	PSRLQ      $0x3f, X2
+	PXOR       X8, X2
+	MOVO       X1, X8
+	PMULULQ    X3, X8
+	PADDQ      X3, X1
+	PADDQ      X8, X1
+	PADDQ      X8, X1
+	PXOR       X1, X7
+	PSHUFD     $0xb1, X7, X7
+	MOVO       X5, X8
+	PMULULQ    X7, X8
+	PADDQ      X7, X5
+	PADDQ      X8, X5
+	PADDQ      X8, X5
+	PXOR       X5, X3
+	PSHUFB     X10, X3
+	MOVO       X1, X8
+	PMULULQ    X3, X8
+	PADDQ      X3, X1
+	PADDQ      X8, X1
+	PADDQ      X8, X1
+	PXOR       X1, X7
+	PSHUFB     X11, X7
+	MOVO       X5, X8
+	PMULULQ    X7, X8
+	PADDQ      X7, X5
+	PADDQ      X8, X5
+	PADDQ      X8, X5
+	PXOR       X5, X3
+	MOVO       X3, X8
+	PADDQ      X3, X8
+	PSRLQ      $0x3f, X3
+	PXOR       X8, X3
+	MOVO       X4, X8
+	MOVO       X5, X4
+	MOVO       X8, X5
+	MOVO       X2, X8
+	PUNPCKLQDQ X2, X9
+	PUNPCKHQDQ X3, X2
+	PUNPCKHQDQ X9, X2
+	PUNPCKLQDQ X3, X9
+	MOVO       X8, X3
+	MOVO       X6, X8
+	PUNPCKHQDQ X9, X3
+	PUNPCKLQDQ X7, X9
+	PUNPCKHQDQ X9, X6
+	PUNPCKLQDQ X8, X9
+	PUNPCKHQDQ X9, X7
+	MOVOU      X0, 640(AX)
+	MOVOU      X1, 656(AX)
+	MOVOU      X2, 672(AX)
+	MOVOU      X3, 688(AX)
+	MOVOU      X4, 704(AX)
+	MOVOU      X5, 720(AX)
+	MOVOU      X6, 736(AX)
+	MOVOU      X7, 752(AX)
+	MOVOU      768(AX), X0
+	MOVOU      784(AX), X1
+	MOVOU      800(AX), X2
+	MOVOU      816(AX), X3
+	MOVOU      832(AX), X4
+	MOVOU      848(AX), X5
+	MOVOU      864(AX), X6
+	MOVOU      880(AX), X7
+	MOVO       X0, X8
+	PMULULQ    X2, X8
+	PADDQ      X2, X0
+	PADDQ      X8, X0
+	PADDQ      X8, X0
+	PXOR       X0, X6
+	PSHUFD     $0xb1, X6, X6
+	MOVO       X4, X8
+	PMULULQ    X6, X8
+	PADDQ      X6, X4
+	PADDQ      X8, X4
+	PADDQ      X8, X4
+	PXOR       X4, X2
+	PSHUFB     X10, X2
+	MOVO       X0, X8
+	PMULULQ    X2, X8
+	PADDQ      X2, X0
+	PADDQ      X8, X0
+	PADDQ      X8, X0
+	PXOR       X0, X6
+	PSHUFB     X11, X6
+	MOVO       X4, X8
+	PMULULQ    X6, X8
+	PADDQ      X6, X4
+	PADDQ      X8, X4
+	PADDQ      X8, X4
+	PXOR       X4, X2
+	MOVO       X2, X8
+	PADDQ      X2, X8
+	PSRLQ      $0x3f, X2
+	PXOR       X8, X2
+	MOVO       X1, X8
+	PMULULQ    X3, X8
+	PADDQ      X3, X1
+	PADDQ      X8, X1
+	PADDQ      X8, X1
+	PXOR       X1, X7
+	PSHUFD     $0xb1, X7, X7
+	MOVO       X5, X8
+	PMULULQ    X7, X8
+	PADDQ      X7, X5
+	PADDQ      X8, X5
+	PADDQ      X8, X5
+	PXOR       X5, X3
+	PSHUFB     X10, X3
+	MOVO       X1, X8
+	PMULULQ    X3, X8
+	PADDQ      X3, X1
+	PADDQ      X8, X1
+	PADDQ      X8, X1
+	PXOR       X1, X7
+	PSHUFB     X11, X7
+	MOVO       X5, X8
+	PMULULQ    X7, X8
+	PADDQ      X7, X5
+	PADDQ      X8, X5
+	PADDQ      X8, X5
+	PXOR       X5, X3
+	MOVO       X3, X8
+	PADDQ      X3, X8
+	PSRLQ      $0x3f, X3
+	PXOR       X8, X3
+	MOVO       X4, X8
+	MOVO       X5, X4
+	MOVO       X8, X5
+	MOVO       X6, X8
+	PUNPCKLQDQ X6, X9
+	PUNPCKHQDQ X7, X6
+	PUNPCKHQDQ X9, X6
+	PUNPCKLQDQ X7, X9
+	MOVO       X8, X7
+	MOVO       X2, X8
+	PUNPCKHQDQ X9, X7
+	PUNPCKLQDQ X3, X9
+	PUNPCKHQDQ X9, X2
+	PUNPCKLQDQ X8, X9
+	PUNPCKHQDQ X9, X3
+	MOVO       X0, X8
+	PMULULQ    X2, X8
+	PADDQ      X2, X0
+	PADDQ      X8, X0
+	PADDQ      X8, X0
+	PXOR       X0, X6
+	PSHUFD     $0xb1, X6, X6
+	MOVO       X4, X8
+	PMULULQ    X6, X8
+	PADDQ      X6, X4
+	PADDQ      X8, X4
+	PADDQ      X8, X4
+	PXOR       X4, X2
+	PSHUFB     X10, X2
+	MOVO       X0, X8
+	PMULULQ    X2, X8
+	PADDQ      X2, X0
+	PADDQ      X8, X0
+	PADDQ      X8, X0
+	PXOR       X0, X6
+	PSHUFB     X11, X6
+	MOVO       X4, X8
+	PMULULQ    X6, X8
+	PADDQ      X6, X4
+	PADDQ      X8, X4
+	PADDQ      X8, X4
+	PXOR       X4, X2
+	MOVO       X2, X8
+	PADDQ      X2, X8
+	PSRLQ      $0x3f, X2
+	PXOR       X8, X2
+	MOVO       X1, X8
+	PMULULQ    X3, X8
+	PADDQ      X3, X1
+	PADDQ      X8, X1
+	PADDQ      X8, X1
+	PXOR       X1, X7
+	PSHUFD     $0xb1, X7, X7
+	MOVO       X5, X8
+	PMULULQ    X7, X8
+	PADDQ      X7, X5
+	PADDQ      X8, X5
+	PADDQ      X8, X5
+	PXOR       X5, X3
+	PSHUFB     X10, X3
+	MOVO       X1, X8
+	PMULULQ    X3, X8
+	PADDQ      X3, X1
+	PADDQ      X8, X1
+	PADDQ      X8, X1
+	PXOR       X1, X7
+	PSHUFB     X11, X7
+	MOVO       X5, X8
+	PMULULQ    X7, X8
+	PADDQ      X7, X5
+	PADDQ      X8, X5
+	PADDQ      X8, X5
+	PXOR       X5, X3
+	MOVO       X3, X8
+	PADDQ      X3, X8
+	PSRLQ      $0x3f, X3
+	PXOR       X8, X3
+	MOVO       X4, X8
+	MOVO       X5, X4
+	MOVO       X8, X5
+	MOVO       X2, X8
+	PUNPCKLQDQ X2, X9
+	PUNPCKHQDQ X3, X2
+	PUNPCKHQDQ X9, X2
+	PUNPCKLQDQ X3, X9
+	MOVO       X8, X3
+	MOVO       X6, X8
+	PUNPCKHQDQ X9, X3
+	PUNPCKLQDQ X7, X9
+	PUNPCKHQDQ X9, X6
+	PUNPCKLQDQ X8, X9
+	PUNPCKHQDQ X9, X7
+	MOVOU      X0, 768(AX)
+	MOVOU      X1, 784(AX)
+	MOVOU      X2, 800(AX)
+	MOVOU      X3, 816(AX)
+	MOVOU      X4, 832(AX)
+	MOVOU      X5, 848(AX)
+	MOVOU      X6, 864(AX)
+	MOVOU      X7, 880(AX)
+	MOVOU      896(AX), X0
+	MOVOU      912(AX), X1
+	MOVOU      928(AX), X2
+	MOVOU      944(AX), X3
+	MOVOU      960(AX), X4
+	MOVOU      976(AX), X5
+	MOVOU      992(AX), X6
+	MOVOU      1008(AX), X7
+	MOVO       X0, X8
+	PMULULQ    X2, X8
+	PADDQ      X2, X0
+	PADDQ      X8, X0
+	PADDQ      X8, X0
+	PXOR       X0, X6
+	PSHUFD     $0xb1, X6, X6
+	MOVO       X4, X8
+	PMULULQ    X6, X8
+	PADDQ      X6, X4
+	PADDQ      X8, X4
+	PADDQ      X8, X4
+	PXOR       X4, X2
+	PSHUFB     X10, X2
+	MOVO       X0, X8
+	PMULULQ    X2, X8
+	PADDQ      X2, X0
+	PADDQ      X8, X0
+	PADDQ      X8, X0
+	PXOR       X0, X6
+	PSHUFB     X11, X6
+	MOVO       X4, X8
+	PMULULQ    X6, X8
+	PADDQ      X6, X4
+	PADDQ      X8, X4
+	PADDQ      X8, X4
+	PXOR       X4, X2
+	MOVO       X2, X8
+	PADDQ      X2, X8
+	PSRLQ      $0x3f, X2
+	PXOR       X8, X2
+	MOVO       X1, X8
+	PMULULQ    X3, X8
+	PADDQ      X3, X1
+	PADDQ      X8, X1
+	PADDQ      X8, X1
+	PXOR       X1, X7
+	PSHUFD     $0xb1, X7, X7
+	MOVO       X5, X8
+	PMULULQ    X7, X8
+	PADDQ      X7, X5
+	PADDQ      X8, X5
+	PADDQ      X8, X5
+	PXOR       X5, X3
+	PSHUFB     X10, X3
+	MOVO       X1, X8
+	PMULULQ    X3, X8
+	PADDQ      X3, X1
+	PADDQ      X8, X1
+	PADDQ      X8, X1
+	PXOR       X1, X7
+	PSHUFB     X11, X7
+	MOVO       X5, X8
+	PMULULQ    X7, X8
+	PADDQ      X7, X5
+	PADDQ      X8, X5
+	PADDQ      X8, X5
+	PXOR       X5, X3
+	MOVO       X3, X8
+	PADDQ      X3, X8
+	PSRLQ      $0x3f, X3
+	PXOR       X8, X3
+	MOVO       X4, X8
+	MOVO       X5, X4
+	MOVO       X8, X5
+	MOVO       X6, X8
+	PUNPCKLQDQ X6, X9
+	PUNPCKHQDQ X7, X6
+	PUNPCKHQDQ X9, X6
+	PUNPCKLQDQ X7, X9
+	MOVO       X8, X7
+	MOVO       X2, X8
+	PUNPCKHQDQ X9, X7
+	PUNPCKLQDQ X3, X9
+	PUNPCKHQDQ X9, X2
+	PUNPCKLQDQ X8, X9
+	PUNPCKHQDQ X9, X3
+	MOVO       X0, X8
+	PMULULQ    X2, X8
+	PADDQ      X2, X0
+	PADDQ      X8, X0
+	PADDQ      X8, X0
+	PXOR       X0, X6
+	PSHUFD     $0xb1, X6, X6
+	MOVO       X4, X8
+	PMULULQ    X6, X8
+	PADDQ      X6, X4
+	PADDQ      X8, X4
+	PADDQ      X8, X4
+	PXOR       X4, X2
+	PSHUFB     X10, X2
+	MOVO       X0, X8
+	PMULULQ    X2, X8
+	PADDQ      X2, X0
+	PADDQ      X8, X0
+	PADDQ      X8, X0
+	PXOR       X0, X6
+	PSHUFB     X11, X6
+	MOVO       X4, X8
+	PMULULQ    X6, X8
+	PADDQ      X6, X4
+	PADDQ      X8, X4
+	PADDQ      X8, X4
+	PXOR       X4, X2
+	MOVO       X2, X8
+	PADDQ      X2, X8
+	PSRLQ      $0x3f, X2
+	PXOR       X8, X2
+	MOVO       X1, X8
+	PMULULQ    X3, X8
+	PADDQ      X3, X1
+	PADDQ      X8, X1
+	PADDQ      X8, X1
+	PXOR       X1, X7
+	PSHUFD     $0xb1, X7, X7
+	MOVO       X5, X8
+	PMULULQ    X7, X8
+	PADDQ      X7, X5
+	PADDQ      X8, X5
+	PADDQ      X8, X5
+	PXOR       X5, X3
+	PSHUFB     X10, X3
+	MOVO       X1, X8
+	PMULULQ    X3, X8
+	PADDQ      X3, X1
+	PADDQ      X8, X1
+	PADDQ      X8, X1
+	PXOR       X1, X7
+	PSHUFB     X11, X7
+	MOVO       X5, X8
+	PMULULQ    X7, X8
+	PADDQ      X7, X5
+	PADDQ      X8, X5
+	PADDQ      X8, X5
+	PXOR       X5, X3
+	MOVO       X3, X8
+	PADDQ      X3, X8
+	PSRLQ      $0x3f, X3
+	PXOR       X8, X3
+	MOVO       X4, X8
+	MOVO       X5, X4
+	MOVO       X8, X5
+	MOVO       X2, X8
+	PUNPCKLQDQ X2, X9
+	PUNPCKHQDQ X3, X2
+	PUNPCKHQDQ X9, X2
+	PUNPCKLQDQ X3, X9
+	MOVO       X8, X3
+	MOVO       X6, X8
+	PUNPCKHQDQ X9, X3
+	PUNPCKLQDQ X7, X9
+	PUNPCKHQDQ X9, X6
+	PUNPCKLQDQ X8, X9
+	PUNPCKHQDQ X9, X7
+	MOVOU      X0, 896(AX)
+	MOVOU      X1, 912(AX)
+	MOVOU      X2, 928(AX)
+	MOVOU      X3, 944(AX)
+	MOVOU      X4, 960(AX)
+	MOVOU      X5, 976(AX)
+	MOVOU      X6, 992(AX)
+	MOVOU      X7, 1008(AX)
+	MOVOU      (AX), X0
+	MOVOU      128(AX), X1
+	MOVOU      256(AX), X2
+	MOVOU      384(AX), X3
+	MOVOU      512(AX), X4
+	MOVOU      640(AX), X5
+	MOVOU      768(AX), X6
+	MOVOU      896(AX), X7
+	MOVO       X0, X8
+	PMULULQ    X2, X8
+	PADDQ      X2, X0
+	PADDQ      X8, X0
+	PADDQ      X8, X0
+	PXOR       X0, X6
+	PSHUFD     $0xb1, X6, X6
+	MOVO       X4, X8
+	PMULULQ    X6, X8
+	PADDQ      X6, X4
+	PADDQ      X8, X4
+	PADDQ      X8, X4
+	PXOR       X4, X2
+	PSHUFB     X10, X2
+	MOVO       X0, X8
+	PMULULQ    X2, X8
+	PADDQ      X2, X0
+	PADDQ      X8, X0
+	PADDQ      X8, X0
+	PXOR       X0, X6
+	PSHUFB     X11, X6
+	MOVO       X4, X8
+	PMULULQ    X6, X8
+	PADDQ      X6, X4
+	PADDQ      X8, X4
+	PADDQ      X8, X4
+	PXOR       X4, X2
+	MOVO       X2, X8
+	PADDQ      X2, X8
+	PSRLQ      $0x3f, X2
+	PXOR       X8, X2
+	MOVO       X1, X8
+	PMULULQ    X3, X8
+	PADDQ      X3, X1
+	PADDQ      X8, X1
+	PADDQ      X8, X1
+	PXOR       X1, X7
+	PSHUFD     $0xb1, X7, X7
+	MOVO       X5, X8
+	PMULULQ    X7, X8
+	PADDQ      X7, X5
+	PADDQ      X8, X5
+	PADDQ      X8, X5
+	PXOR       X5, X3
+	PSHUFB     X10, X3
+	MOVO       X1, X8
+	PMULULQ    X3, X8
+	PADDQ      X3, X1
+	PADDQ      X8, X1
+	PADDQ      X8, X1
+	PXOR       X1, X7
+	PSHUFB     X11, X7
+	MOVO       X5, X8
+	PMULULQ    X7, X8
+	PADDQ      X7, X5
+	PADDQ      X8, X5
+	PADDQ      X8, X5
+	PXOR       X5, X3
+	MOVO       X3, X8
+	PADDQ      X3, X8
+	PSRLQ      $0x3f, X3
+	PXOR       X8, X3
+	MOVO       X4, X8
+	MOVO       X5, X4
+	MOVO       X8, X5
+	MOVO       X6, X8
+	PUNPCKLQDQ X6, X9
+	PUNPCKHQDQ X7, X6
+	PUNPCKHQDQ X9, X6
+	PUNPCKLQDQ X7, X9
+	MOVO       X8, X7
+	MOVO       X2, X8
+	PUNPCKHQDQ X9, X7
+	PUNPCKLQDQ X3, X9
+	PUNPCKHQDQ X9, X2
+	PUNPCKLQDQ X8, X9
+	PUNPCKHQDQ X9, X3
+	MOVO       X0, X8
+	PMULULQ    X2, X8
+	PADDQ      X2, X0
+	PADDQ      X8, X0
+	PADDQ      X8, X0
+	PXOR       X0, X6
+	PSHUFD     $0xb1, X6, X6
+	MOVO       X4, X8
+	PMULULQ    X6, X8
+	PADDQ      X6, X4
+	PADDQ      X8, X4
+	PADDQ      X8, X4
+	PXOR       X4, X2
+	PSHUFB     X10, X2
+	MOVO       X0, X8
+	PMULULQ    X2, X8
+	PADDQ      X2, X0
+	PADDQ      X8, X0
+	PADDQ      X8, X0
+	PXOR       X0, X6
+	PSHUFB     X11, X6
+	MOVO       X4, X8
+	PMULULQ    X6, X8
+	PADDQ      X6, X4
+	PADDQ      X8, X4
+	PADDQ      X8, X4
+	PXOR       X4, X2
+	MOVO       X2, X8
+	PADDQ      X2, X8
+	PSRLQ      $0x3f, X2
+	PXOR       X8, X2
+	MOVO       X1, X8
+	PMULULQ    X3, X8
+	PADDQ      X3, X1
+	PADDQ      X8, X1
+	PADDQ      X8, X1
+	PXOR       X1, X7
+	PSHUFD     $0xb1, X7, X7
+	MOVO       X5, X8
+	PMULULQ    X7, X8
+	PADDQ      X7, X5
+	PADDQ      X8, X5
+	PADDQ      X8, X5
+	PXOR       X5, X3
+	PSHUFB     X10, X3
+	MOVO       X1, X8
+	PMULULQ    X3, X8
+	PADDQ      X3, X1
+	PADDQ      X8, X1
+	PADDQ      X8, X1
+	PXOR       X1, X7
+	PSHUFB     X11, X7
+	MOVO       X5, X8
+	PMULULQ    X7, X8
+	PADDQ      X7, X5
+	PADDQ      X8, X5
+	PADDQ      X8, X5
+	PXOR       X5, X3
+	MOVO       X3, X8
+	PADDQ      X3, X8
+	PSRLQ      $0x3f, X3
+	PXOR       X8, X3
+	MOVO       X4, X8
+	MOVO       X5, X4
+	MOVO       X8, X5
+	MOVO       X2, X8
+	PUNPCKLQDQ X2, X9
+	PUNPCKHQDQ X3, X2
+	PUNPCKHQDQ X9, X2
+	PUNPCKLQDQ X3, X9
+	MOVO       X8, X3
+	MOVO       X6, X8
+	PUNPCKHQDQ X9, X3
+	PUNPCKLQDQ X7, X9
+	PUNPCKHQDQ X9, X6
+	PUNPCKLQDQ X8, X9
+	PUNPCKHQDQ X9, X7
+	MOVOU      X0, (AX)
+	MOVOU      X1, 128(AX)
+	MOVOU      X2, 256(AX)
+	MOVOU      X3, 384(AX)
+	MOVOU      X4, 512(AX)
+	MOVOU      X5, 640(AX)
+	MOVOU      X6, 768(AX)
+	MOVOU      X7, 896(AX)
+	MOVOU      16(AX), X0
+	MOVOU      144(AX), X1
+	MOVOU      272(AX), X2
+	MOVOU      400(AX), X3
+	MOVOU      528(AX), X4
+	MOVOU      656(AX), X5
+	MOVOU      784(AX), X6
+	MOVOU      912(AX), X7
+	MOVO       X0, X8
+	PMULULQ    X2, X8
+	PADDQ      X2, X0
+	PADDQ      X8, X0
+	PADDQ      X8, X0
+	PXOR       X0, X6
+	PSHUFD     $0xb1, X6, X6
+	MOVO       X4, X8
+	PMULULQ    X6, X8
+	PADDQ      X6, X4
+	PADDQ      X8, X4
+	PADDQ      X8, X4
+	PXOR       X4, X2
+	PSHUFB     X10, X2
+	MOVO       X0, X8
+	PMULULQ    X2, X8
+	PADDQ      X2, X0
+	PADDQ      X8, X0
+	PADDQ      X8, X0
+	PXOR       X0, X6
+	PSHUFB     X11, X6
+	MOVO       X4, X8
+	PMULULQ    X6, X8
+	PADDQ      X6, X4
+	PADDQ      X8, X4
+	PADDQ      X8, X4
+	PXOR       X4, X2
+	MOVO       X2, X8
+	PADDQ      X2, X8
+	PSRLQ      $0x3f, X2
+	PXOR       X8, X2
+	MOVO       X1, X8
+	PMULULQ    X3, X8
+	PADDQ      X3, X1
+	PADDQ      X8, X1
+	PADDQ      X8, X1
+	PXOR       X1, X7
+	PSHUFD     $0xb1, X7, X7
+	MOVO       X5, X8
+	PMULULQ    X7, X8
+	PADDQ      X7, X5
+	PADDQ      X8, X5
+	PADDQ      X8, X5
+	PXOR       X5, X3
+	PSHUFB     X10, X3
+	MOVO       X1, X8
+	PMULULQ    X3, X8
+	PADDQ      X3, X1
+	PADDQ      X8, X1
+	PADDQ      X8, X1
+	PXOR       X1, X7
+	PSHUFB     X11, X7
+	MOVO       X5, X8
+	PMULULQ    X7, X8
+	PADDQ      X7, X5
+	PADDQ      X8, X5
+	PADDQ      X8, X5
+	PXOR       X5, X3
+	MOVO       X3, X8
+	PADDQ      X3, X8
+	PSRLQ      $0x3f, X3
+	PXOR       X8, X3
+	MOVO       X4, X8
+	MOVO       X5, X4
+	MOVO       X8, X5
+	MOVO       X6, X8
+	PUNPCKLQDQ X6, X9
+	PUNPCKHQDQ X7, X6
+	PUNPCKHQDQ X9, X6
+	PUNPCKLQDQ X7, X9
+	MOVO       X8, X7
+	MOVO       X2, X8
+	PUNPCKHQDQ X9, X7
+	PUNPCKLQDQ X3, X9
+	PUNPCKHQDQ X9, X2
+	PUNPCKLQDQ X8, X9
+	PUNPCKHQDQ X9, X3
+	MOVO       X0, X8
+	PMULULQ    X2, X8
+	PADDQ      X2, X0
+	PADDQ      X8, X0
+	PADDQ      X8, X0
+	PXOR       X0, X6
+	PSHUFD     $0xb1, X6, X6
+	MOVO       X4, X8
+	PMULULQ    X6, X8
+	PADDQ      X6, X4
+	PADDQ      X8, X4
+	PADDQ      X8, X4
+	PXOR       X4, X2
+	PSHUFB     X10, X2
+	MOVO       X0, X8
+	PMULULQ    X2, X8
+	PADDQ      X2, X0
+	PADDQ      X8, X0
+	PADDQ      X8, X0
+	PXOR       X0, X6
+	PSHUFB     X11, X6
+	MOVO       X4, X8
+	PMULULQ    X6, X8
+	PADDQ      X6, X4
+	PADDQ      X8, X4
+	PADDQ      X8, X4
+	PXOR       X4, X2
+	MOVO       X2, X8
+	PADDQ      X2, X8
+	PSRLQ      $0x3f, X2
+	PXOR       X8, X2
+	MOVO       X1, X8
+	PMULULQ    X3, X8
+	PADDQ      X3, X1
+	PADDQ      X8, X1
+	PADDQ      X8, X1
+	PXOR       X1, X7
+	PSHUFD     $0xb1, X7, X7
+	MOVO       X5, X8
+	PMULULQ    X7, X8
+	PADDQ      X7, X5
+	PADDQ      X8, X5
+	PADDQ      X8, X5
+	PXOR       X5, X3
+	PSHUFB     X10, X3
+	MOVO       X1, X8
+	PMULULQ    X3, X8
+	PADDQ      X3, X1
+	PADDQ      X8, X1
+	PADDQ      X8, X1
+	PXOR       X1, X7
+	PSHUFB     X11, X7
+	MOVO       X5, X8
+	PMULULQ    X7, X8
+	PADDQ      X7, X5
+	PADDQ      X8, X5
+	PADDQ      X8, X5
+	PXOR       X5, X3
+	MOVO       X3, X8
+	PADDQ      X3, X8
+	PSRLQ      $0x3f, X3
+	PXOR       X8, X3
+	MOVO       X4, X8
+	MOVO       X5, X4
+	MOVO       X8, X5
+	MOVO       X2, X8
+	PUNPCKLQDQ X2, X9
+	PUNPCKHQDQ X3, X2
+	PUNPCKHQDQ X9, X2
+	PUNPCKLQDQ X3, X9
+	MOVO       X8, X3
+	MOVO       X6, X8
+	PUNPCKHQDQ X9, X3
+	PUNPCKLQDQ X7, X9
+	PUNPCKHQDQ X9, X6
+	PUNPCKLQDQ X8, X9
+	PUNPCKHQDQ X9, X7
+	MOVOU      X0, 16(AX)
+	MOVOU      X1, 144(AX)
+	MOVOU      X2, 272(AX)
+	MOVOU      X3, 400(AX)
+	MOVOU      X4, 528(AX)
+	MOVOU      X5, 656(AX)
+	MOVOU      X6, 784(AX)
+	MOVOU      X7, 912(AX)
+	MOVOU      32(AX), X0
+	MOVOU      160(AX), X1
+	MOVOU      288(AX), X2
+	MOVOU      416(AX), X3
+	MOVOU      544(AX), X4
+	MOVOU      672(AX), X5
+	MOVOU      800(AX), X6
+	MOVOU      928(AX), X7
+	MOVO       X0, X8
+	PMULULQ    X2, X8
+	PADDQ      X2, X0
+	PADDQ      X8, X0
+	PADDQ      X8, X0
+	PXOR       X0, X6
+	PSHUFD     $0xb1, X6, X6
+	MOVO       X4, X8
+	PMULULQ    X6, X8
+	PADDQ      X6, X4
+	PADDQ      X8, X4
+	PADDQ      X8, X4
+	PXOR       X4, X2
+	PSHUFB     X10, X2
+	MOVO       X0, X8
+	PMULULQ    X2, X8
+	PADDQ      X2, X0
+	PADDQ      X8, X0
+	PADDQ      X8, X0
+	PXOR       X0, X6
+	PSHUFB     X11, X6
+	MOVO       X4, X8
+	PMULULQ    X6, X8
+	PADDQ      X6, X4
+	PADDQ      X8, X4
+	PADDQ      X8, X4
+	PXOR       X4, X2
+	MOVO       X2, X8
+	PADDQ      X2, X8
+	PSRLQ      $0x3f, X2
+	PXOR       X8, X2
+	MOVO       X1, X8
+	PMULULQ    X3, X8
+	PADDQ      X3, X1
+	PADDQ      X8, X1
+	PADDQ      X8, X1
+	PXOR       X1, X7
+	PSHUFD     $0xb1, X7, X7
+	MOVO       X5, X8
+	PMULULQ    X7, X8
+	PADDQ      X7, X5
+	PADDQ      X8, X5
+	PADDQ      X8, X5
+	PXOR       X5, X3
+	PSHUFB     X10, X3
+	MOVO       X1, X8
+	PMULULQ    X3, X8
+	PADDQ      X3, X1
+	PADDQ      X8, X1
+	PADDQ      X8, X1
+	PXOR       X1, X7
+	PSHUFB     X11, X7
+	MOVO       X5, X8
+	PMULULQ    X7, X8
+	PADDQ      X7, X5
+	PADDQ      X8, X5
+	PADDQ      X8, X5
+	PXOR       X5, X3
+	MOVO       X3, X8
+	PADDQ      X3, X8
+	PSRLQ      $0x3f, X3
+	PXOR       X8, X3
+	MOVO       X4, X8
+	MOVO       X5, X4
+	MOVO       X8, X5
+	MOVO       X6, X8
+	PUNPCKLQDQ X6, X9
+	PUNPCKHQDQ X7, X6
+	PUNPCKHQDQ X9, X6
+	PUNPCKLQDQ X7, X9
+	MOVO       X8, X7
+	MOVO       X2, X8
+	PUNPCKHQDQ X9, X7
+	PUNPCKLQDQ X3, X9
+	PUNPCKHQDQ X9, X2
+	PUNPCKLQDQ X8, X9
+	PUNPCKHQDQ X9, X3
+	MOVO       X0, X8
+	PMULULQ    X2, X8
+	PADDQ      X2, X0
+	PADDQ      X8, X0
+	PADDQ      X8, X0
+	PXOR       X0, X6
+	PSHUFD     $0xb1, X6, X6
+	MOVO       X4, X8
+	PMULULQ    X6, X8
+	PADDQ      X6, X4
+	PADDQ      X8, X4
+	PADDQ      X8, X4
+	PXOR       X4, X2
+	PSHUFB     X10, X2
+	MOVO       X0, X8
+	PMULULQ    X2, X8
+	PADDQ      X2, X0
+	PADDQ      X8, X0
+	PADDQ      X8, X0
+	PXOR       X0, X6
+	PSHUFB     X11, X6
+	MOVO       X4, X8
+	PMULULQ    X6, X8
+	PADDQ      X6, X4
+	PADDQ      X8, X4
+	PADDQ      X8, X4
+	PXOR       X4, X2
+	MOVO       X2, X8
+	PADDQ      X2, X8
+	PSRLQ      $0x3f, X2
+	PXOR       X8, X2
+	MOVO       X1, X8
+	PMULULQ    X3, X8
+	PADDQ      X3, X1
+	PADDQ      X8, X1
+	PADDQ      X8, X1
+	PXOR       X1, X7
+	PSHUFD     $0xb1, X7, X7
+	MOVO       X5, X8
+	PMULULQ    X7, X8
+	PADDQ      X7, X5
+	PADDQ      X8, X5
+	PADDQ      X8, X5
+	PXOR       X5, X3
+	PSHUFB     X10, X3
+	MOVO       X1, X8
+	PMULULQ    X3, X8
+	PADDQ      X3, X1
+	PADDQ      X8, X1
+	PADDQ      X8, X1
+	PXOR       X1, X7
+	PSHUFB     X11, X7
+	MOVO       X5, X8
+	PMULULQ    X7, X8
+	PADDQ      X7, X5
+	PADDQ      X8, X5
+	PADDQ      X8, X5
+	PXOR       X5, X3
+	MOVO       X3, X8
+	PADDQ      X3, X8
+	PSRLQ      $0x3f, X3
+	PXOR       X8, X3
+	MOVO       X4, X8
+	MOVO       X5, X4
+	MOVO       X8, X5
+	MOVO       X2, X8
+	PUNPCKLQDQ X2, X9
+	PUNPCKHQDQ X3, X2
+	PUNPCKHQDQ X9, X2
+	PUNPCKLQDQ X3, X9
+	MOVO       X8, X3
+	MOVO       X6, X8
+	PUNPCKHQDQ X9, X3
+	PUNPCKLQDQ X7, X9
+	PUNPCKHQDQ X9, X6
+	PUNPCKLQDQ X8, X9
+	PUNPCKHQDQ X9, X7
+	MOVOU      X0, 32(AX)
+	MOVOU      X1, 160(AX)
+	MOVOU      X2, 288(AX)
+	MOVOU      X3, 416(AX)
+	MOVOU      X4, 544(AX)
+	MOVOU      X5, 672(AX)
+	MOVOU      X6, 800(AX)
+	MOVOU      X7, 928(AX)
+	MOVOU      48(AX), X0
+	MOVOU      176(AX), X1
+	MOVOU      304(AX), X2
+	MOVOU      432(AX), X3
+	MOVOU      560(AX), X4
+	MOVOU      688(AX), X5
+	MOVOU      816(AX), X6
+	MOVOU      944(AX), X7
+	MOVO       X0, X8
+	PMULULQ    X2, X8
+	PADDQ      X2, X0
+	PADDQ      X8, X0
+	PADDQ      X8, X0
+	PXOR       X0, X6
+	PSHUFD     $0xb1, X6, X6
+	MOVO       X4, X8
+	PMULULQ    X6, X8
+	PADDQ      X6, X4
+	PADDQ      X8, X4
+	PADDQ      X8, X4
+	PXOR       X4, X2
+	PSHUFB     X10, X2
+	MOVO       X0, X8
+	PMULULQ    X2, X8
+	PADDQ      X2, X0
+	PADDQ      X8, X0
+	PADDQ      X8, X0
+	PXOR       X0, X6
+	PSHUFB     X11, X6
+	MOVO       X4, X8
+	PMULULQ    X6, X8
+	PADDQ      X6, X4
+	PADDQ      X8, X4
+	PADDQ      X8, X4
+	PXOR       X4, X2
+	MOVO       X2, X8
+	PADDQ      X2, X8
+	PSRLQ      $0x3f, X2
+	PXOR       X8, X2
+	MOVO       X1, X8
+	PMULULQ    X3, X8
+	PADDQ      X3, X1
+	PADDQ      X8, X1
+	PADDQ      X8, X1
+	PXOR       X1, X7
+	PSHUFD     $0xb1, X7, X7
+	MOVO       X5, X8
+	PMULULQ    X7, X8
+	PADDQ      X7, X5
+	PADDQ      X8, X5
+	PADDQ      X8, X5
+	PXOR       X5, X3
+	PSHUFB     X10, X3
+	MOVO       X1, X8
+	PMULULQ    X3, X8
+	PADDQ      X3, X1
+	PADDQ      X8, X1
+	PADDQ      X8, X1
+	PXOR       X1, X7
+	PSHUFB     X11, X7
+	MOVO       X5, X8
+	PMULULQ    X7, X8
+	PADDQ      X7, X5
+	PADDQ      X8, X5
+	PADDQ      X8, X5
+	PXOR       X5, X3
+	MOVO       X3, X8
+	PADDQ      X3, X8
+	PSRLQ      $0x3f, X3
+	PXOR       X8, X3
+	MOVO       X4, X8
+	MOVO       X5, X4
+	MOVO       X8, X5
+	MOVO       X6, X8
+	PUNPCKLQDQ X6, X9
+	PUNPCKHQDQ X7, X6
+	PUNPCKHQDQ X9, X6
+	PUNPCKLQDQ X7, X9
+	MOVO       X8, X7
+	MOVO       X2, X8
+	PUNPCKHQDQ X9, X7
+	PUNPCKLQDQ X3, X9
+	PUNPCKHQDQ X9, X2
+	PUNPCKLQDQ X8, X9
+	PUNPCKHQDQ X9, X3
+	MOVO       X0, X8
+	PMULULQ    X2, X8
+	PADDQ      X2, X0
+	PADDQ      X8, X0
+	PADDQ      X8, X0
+	PXOR       X0, X6
+	PSHUFD     $0xb1, X6, X6
+	MOVO       X4, X8
+	PMULULQ    X6, X8
+	PADDQ      X6, X4
+	PADDQ      X8, X4
+	PADDQ      X8, X4
+	PXOR       X4, X2
+	PSHUFB     X10, X2
+	MOVO       X0, X8
+	PMULULQ    X2, X8
+	PADDQ      X2, X0
+	PADDQ      X8, X0
+	PADDQ      X8, X0
+	PXOR       X0, X6
+	PSHUFB     X11, X6
+	MOVO       X4, X8
+	PMULULQ    X6, X8
+	PADDQ      X6, X4
+	PADDQ      X8, X4
+	PADDQ      X8, X4
+	PXOR       X4, X2
+	MOVO       X2, X8
+	PADDQ      X2, X8
+	PSRLQ      $0x3f, X2
+	PXOR       X8, X2
+	MOVO       X1, X8
+	PMULULQ    X3, X8
+	PADDQ      X3, X1
+	PADDQ      X8, X1
+	PADDQ      X8, X1
+	PXOR       X1, X7
+	PSHUFD     $0xb1, X7, X7
+	MOVO       X5, X8
+	PMULULQ    X7, X8
+	PADDQ      X7, X5
+	PADDQ      X8, X5
+	PADDQ      X8, X5
+	PXOR       X5, X3
+	PSHUFB     X10, X3
+	MOVO       X1, X8
+	PMULULQ    X3, X8
+	PADDQ      X3, X1
+	PADDQ      X8, X1
+	PADDQ      X8, X1
+	PXOR       X1, X7
+	PSHUFB     X11, X7
+	MOVO       X5, X8
+	PMULULQ    X7, X8
+	PADDQ      X7, X5
+	PADDQ      X8, X5
+	PADDQ      X8, X5
+	PXOR       X5, X3
+	MOVO       X3, X8
+	PADDQ      X3, X8
+	PSRLQ      $0x3f, X3
+	PXOR       X8, X3
+	MOVO       X4, X8
+	MOVO       X5, X4
+	MOVO       X8, X5
+	MOVO       X2, X8
+	PUNPCKLQDQ X2, X9
+	PUNPCKHQDQ X3, X2
+	PUNPCKHQDQ X9, X2
+	PUNPCKLQDQ X3, X9
+	MOVO       X8, X3
+	MOVO       X6, X8
+	PUNPCKHQDQ X9, X3
+	PUNPCKLQDQ X7, X9
+	PUNPCKHQDQ X9, X6
+	PUNPCKLQDQ X8, X9
+	PUNPCKHQDQ X9, X7
+	MOVOU      X0, 48(AX)
+	MOVOU      X1, 176(AX)
+	MOVOU      X2, 304(AX)
+	MOVOU      X3, 432(AX)
+	MOVOU      X4, 560(AX)
+	MOVOU      X5, 688(AX)
+	MOVOU      X6, 816(AX)
+	MOVOU      X7, 944(AX)
+	MOVOU      64(AX), X0
+	MOVOU      192(AX), X1
+	MOVOU      320(AX), X2
+	MOVOU      448(AX), X3
+	MOVOU      576(AX), X4
+	MOVOU      704(AX), X5
+	MOVOU      832(AX), X6
+	MOVOU      960(AX), X7
+	MOVO       X0, X8
+	PMULULQ    X2, X8
+	PADDQ      X2, X0
+	PADDQ      X8, X0
+	PADDQ      X8, X0
+	PXOR       X0, X6
+	PSHUFD     $0xb1, X6, X6
+	MOVO       X4, X8
+	PMULULQ    X6, X8
+	PADDQ      X6, X4
+	PADDQ      X8, X4
+	PADDQ      X8, X4
+	PXOR       X4, X2
+	PSHUFB     X10, X2
+	MOVO       X0, X8
+	PMULULQ    X2, X8
+	PADDQ      X2, X0
+	PADDQ      X8, X0
+	PADDQ      X8, X0
+	PXOR       X0, X6
+	PSHUFB     X11, X6
+	MOVO       X4, X8
+	PMULULQ    X6, X8
+	PADDQ      X6, X4
+	PADDQ      X8, X4
+	PADDQ      X8, X4
+	PXOR       X4, X2
+	MOVO       X2, X8
+	PADDQ      X2, X8
+	PSRLQ      $0x3f, X2
+	PXOR       X8, X2
+	MOVO       X1, X8
+	PMULULQ    X3, X8
+	PADDQ      X3, X1
+	PADDQ      X8, X1
+	PADDQ      X8, X1
+	PXOR       X1, X7
+	PSHUFD     $0xb1, X7, X7
+	MOVO       X5, X8
+	PMULULQ    X7, X8
+	PADDQ      X7, X5
+	PADDQ      X8, X5
+	PADDQ      X8, X5
+	PXOR       X5, X3
+	PSHUFB     X10, X3
+	MOVO       X1, X8
+	PMULULQ    X3, X8
+	PADDQ      X3, X1
+	PADDQ      X8, X1
+	PADDQ      X8, X1
+	PXOR       X1, X7
+	PSHUFB     X11, X7
+	MOVO       X5, X8
+	PMULULQ    X7, X8
+	PADDQ      X7, X5
+	PADDQ      X8, X5
+	PADDQ      X8, X5
+	PXOR       X5, X3
+	MOVO       X3, X8
+	PADDQ      X3, X8
+	PSRLQ      $0x3f, X3
+	PXOR       X8, X3
+	MOVO       X4, X8
+	MOVO       X5, X4
+	MOVO       X8, X5
+	MOVO       X6, X8
+	PUNPCKLQDQ X6, X9
+	PUNPCKHQDQ X7, X6
+	PUNPCKHQDQ X9, X6
+	PUNPCKLQDQ X7, X9
+	MOVO       X8, X7
+	MOVO       X2, X8
+	PUNPCKHQDQ X9, X7
+	PUNPCKLQDQ X3, X9
+	PUNPCKHQDQ X9, X2
+	PUNPCKLQDQ X8, X9
+	PUNPCKHQDQ X9, X3
+	MOVO       X0, X8
+	PMULULQ    X2, X8
+	PADDQ      X2, X0
+	PADDQ      X8, X0
+	PADDQ      X8, X0
+	PXOR       X0, X6
+	PSHUFD     $0xb1, X6, X6
+	MOVO       X4, X8
+	PMULULQ    X6, X8
+	PADDQ      X6, X4
+	PADDQ      X8, X4
+	PADDQ      X8, X4
+	PXOR       X4, X2
+	PSHUFB     X10, X2
+	MOVO       X0, X8
+	PMULULQ    X2, X8
+	PADDQ      X2, X0
+	PADDQ      X8, X0
+	PADDQ      X8, X0
+	PXOR       X0, X6
+	PSHUFB     X11, X6
+	MOVO       X4, X8
+	PMULULQ    X6, X8
+	PADDQ      X6, X4
+	PADDQ      X8, X4
+	PADDQ      X8, X4
+	PXOR       X4, X2
+	MOVO       X2, X8
+	PADDQ      X2, X8
+	PSRLQ      $0x3f, X2
+	PXOR       X8, X2
+	MOVO       X1, X8
+	PMULULQ    X3, X8
+	PADDQ      X3, X1
+	PADDQ      X8, X1
+	PADDQ      X8, X1
+	PXOR       X1, X7
+	PSHUFD     $0xb1, X7, X7
+	MOVO       X5, X8
+	PMULULQ    X7, X8
+	PADDQ      X7, X5
+	PADDQ      X8, X5
+	PADDQ      X8, X5
+	PXOR       X5, X3
+	PSHUFB     X10, X3
+	MOVO       X1, X8
+	PMULULQ    X3, X8
+	PADDQ      X3, X1
+	PADDQ      X8, X1
+	PADDQ      X8, X1
+	PXOR       X1, X7
+	PSHUFB     X11, X7
+	MOVO       X5, X8
+	PMULULQ    X7, X8
+	PADDQ      X7, X5
+	PADDQ      X8, X5
+	PADDQ      X8, X5
+	PXOR       X5, X3
+	MOVO       X3, X8
+	PADDQ      X3, X8
+	PSRLQ      $0x3f, X3
+	PXOR       X8, X3
+	MOVO       X4, X8
+	MOVO       X5, X4
+	MOVO       X8, X5
+	MOVO       X2, X8
+	PUNPCKLQDQ X2, X9
+	PUNPCKHQDQ X3, X2
+	PUNPCKHQDQ X9, X2
+	PUNPCKLQDQ X3, X9
+	MOVO       X8, X3
+	MOVO       X6, X8
+	PUNPCKHQDQ X9, X3
+	PUNPCKLQDQ X7, X9
+	PUNPCKHQDQ X9, X6
+	PUNPCKLQDQ X8, X9
+	PUNPCKHQDQ X9, X7
+	MOVOU      X0, 64(AX)
+	MOVOU      X1, 192(AX)
+	MOVOU      X2, 320(AX)
+	MOVOU      X3, 448(AX)
+	MOVOU      X4, 576(AX)
+	MOVOU      X5, 704(AX)
+	MOVOU      X6, 832(AX)
+	MOVOU      X7, 960(AX)
+	MOVOU      80(AX), X0
+	MOVOU      208(AX), X1
+	MOVOU      336(AX), X2
+	MOVOU      464(AX), X3
+	MOVOU      592(AX), X4
+	MOVOU      720(AX), X5
+	MOVOU      848(AX), X6
+	MOVOU      976(AX), X7
+	MOVO       X0, X8
+	PMULULQ    X2, X8
+	PADDQ      X2, X0
+	PADDQ      X8, X0
+	PADDQ      X8, X0
+	PXOR       X0, X6
+	PSHUFD     $0xb1, X6, X6
+	MOVO       X4, X8
+	PMULULQ    X6, X8
+	PADDQ      X6, X4
+	PADDQ      X8, X4
+	PADDQ      X8, X4
+	PXOR       X4, X2
+	PSHUFB     X10, X2
+	MOVO       X0, X8
+	PMULULQ    X2, X8
+	PADDQ      X2, X0
+	PADDQ      X8, X0
+	PADDQ      X8, X0
+	PXOR       X0, X6
+	PSHUFB     X11, X6
+	MOVO       X4, X8
+	PMULULQ    X6, X8
+	PADDQ      X6, X4
+	PADDQ      X8, X4
+	PADDQ      X8, X4
+	PXOR       X4, X2
+	MOVO       X2, X8
+	PADDQ      X2, X8
+	PSRLQ      $0x3f, X2
+	PXOR       X8, X2
+	MOVO       X1, X8
+	PMULULQ    X3, X8
+	PADDQ      X3, X1
+	PADDQ      X8, X1
+	PADDQ      X8, X1
+	PXOR       X1, X7
+	PSHUFD     $0xb1, X7, X7
+	MOVO       X5, X8
+	PMULULQ    X7, X8
+	PADDQ      X7, X5
+	PADDQ      X8, X5
+	PADDQ      X8, X5
+	PXOR       X5, X3
+	PSHUFB     X10, X3
+	MOVO       X1, X8
+	PMULULQ    X3, X8
+	PADDQ      X3, X1
+	PADDQ      X8, X1
+	PADDQ      X8, X1
+	PXOR       X1, X7
+	PSHUFB     X11, X7
+	MOVO       X5, X8
+	PMULULQ    X7, X8
+	PADDQ      X7, X5
+	PADDQ      X8, X5
+	PADDQ      X8, X5
+	PXOR       X5, X3
+	MOVO       X3, X8
+	PADDQ      X3, X8
+	PSRLQ      $0x3f, X3
+	PXOR       X8, X3
+	MOVO       X4, X8
+	MOVO       X5, X4
+	MOVO       X8, X5
+	MOVO       X6, X8
+	PUNPCKLQDQ X6, X9
+	PUNPCKHQDQ X7, X6
+	PUNPCKHQDQ X9, X6
+	PUNPCKLQDQ X7, X9
+	MOVO       X8, X7
+	MOVO       X2, X8
+	PUNPCKHQDQ X9, X7
+	PUNPCKLQDQ X3, X9
+	PUNPCKHQDQ X9, X2
+	PUNPCKLQDQ X8, X9
+	PUNPCKHQDQ X9, X3
+	MOVO       X0, X8
+	PMULULQ    X2, X8
+	PADDQ      X2, X0
+	PADDQ      X8, X0
+	PADDQ      X8, X0
+	PXOR       X0, X6
+	PSHUFD     $0xb1, X6, X6
+	MOVO       X4, X8
+	PMULULQ    X6, X8
+	PADDQ      X6, X4
+	PADDQ      X8, X4
+	PADDQ      X8, X4
+	PXOR       X4, X2
+	PSHUFB     X10, X2
+	MOVO       X0, X8
+	PMULULQ    X2, X8
+	PADDQ      X2, X0
+	PADDQ      X8, X0
+	PADDQ      X8, X0
+	PXOR       X0, X6
+	PSHUFB     X11, X6
+	MOVO       X4, X8
+	PMULULQ    X6, X8
+	PADDQ      X6, X4
+	PADDQ      X8, X4
+	PADDQ      X8, X4
+	PXOR       X4, X2
+	MOVO       X2, X8
+	PADDQ      X2, X8
+	PSRLQ      $0x3f, X2
+	PXOR       X8, X2
+	MOVO       X1, X8
+	PMULULQ    X3, X8
+	PADDQ      X3, X1
+	PADDQ      X8, X1
+	PADDQ      X8, X1
+	PXOR       X1, X7
+	PSHUFD     $0xb1, X7, X7
+	MOVO       X5, X8
+	PMULULQ    X7, X8
+	PADDQ      X7, X5
+	PADDQ      X8, X5
+	PADDQ      X8, X5
+	PXOR       X5, X3
+	PSHUFB     X10, X3
+	MOVO       X1, X8
+	PMULULQ    X3, X8
+	PADDQ      X3, X1
+	PADDQ      X8, X1
+	PADDQ      X8, X1
+	PXOR       X1, X7
+	PSHUFB     X11, X7
+	MOVO       X5, X8
+	PMULULQ    X7, X8
+	PADDQ      X7, X5
+	PADDQ      X8, X5
+	PADDQ      X8, X5
+	PXOR       X5, X3
+	MOVO       X3, X8
+	PADDQ      X3, X8
+	PSRLQ      $0x3f, X3
+	PXOR       X8, X3
+	MOVO       X4, X8
+	MOVO       X5, X4
+	MOVO       X8, X5
+	MOVO       X2, X8
+	PUNPCKLQDQ X2, X9
+	PUNPCKHQDQ X3, X2
+	PUNPCKHQDQ X9, X2
+	PUNPCKLQDQ X3, X9
+	MOVO       X8, X3
+	MOVO       X6, X8
+	PUNPCKHQDQ X9, X3
+	PUNPCKLQDQ X7, X9
+	PUNPCKHQDQ X9, X6
+	PUNPCKLQDQ X8, X9
+	PUNPCKHQDQ X9, X7
+	MOVOU      X0, 80(AX)
+	MOVOU      X1, 208(AX)
+	MOVOU      X2, 336(AX)
+	MOVOU      X3, 464(AX)
+	MOVOU      X4, 592(AX)
+	MOVOU      X5, 720(AX)
+	MOVOU      X6, 848(AX)
+	MOVOU      X7, 976(AX)
+	MOVOU      96(AX), X0
+	MOVOU      224(AX), X1
+	MOVOU      352(AX), X2
+	MOVOU      480(AX), X3
+	MOVOU      608(AX), X4
+	MOVOU      736(AX), X5
+	MOVOU      864(AX), X6
+	MOVOU      992(AX), X7
+	MOVO       X0, X8
+	PMULULQ    X2, X8
+	PADDQ      X2, X0
+	PADDQ      X8, X0
+	PADDQ      X8, X0
+	PXOR       X0, X6
+	PSHUFD     $0xb1, X6, X6
+	MOVO       X4, X8
+	PMULULQ    X6, X8
+	PADDQ      X6, X4
+	PADDQ      X8, X4
+	PADDQ      X8, X4
+	PXOR       X4, X2
+	PSHUFB     X10, X2
+	MOVO       X0, X8
+	PMULULQ    X2, X8
+	PADDQ      X2, X0
+	PADDQ      X8, X0
+	PADDQ      X8, X0
+	PXOR       X0, X6
+	PSHUFB     X11, X6
+	MOVO       X4, X8
+	PMULULQ    X6, X8
+	PADDQ      X6, X4
+	PADDQ      X8, X4
+	PADDQ      X8, X4
+	PXOR       X4, X2
+	MOVO       X2, X8
+	PADDQ      X2, X8
+	PSRLQ      $0x3f, X2
+	PXOR       X8, X2
+	MOVO       X1, X8
+	PMULULQ    X3, X8
+	PADDQ      X3, X1
+	PADDQ      X8, X1
+	PADDQ      X8, X1
+	PXOR       X1, X7
+	PSHUFD     $0xb1, X7, X7
+	MOVO       X5, X8
+	PMULULQ    X7, X8
+	PADDQ      X7, X5
+	PADDQ      X8, X5
+	PADDQ      X8, X5
+	PXOR       X5, X3
+	PSHUFB     X10, X3
+	MOVO       X1, X8
+	PMULULQ    X3, X8
+	PADDQ      X3, X1
+	PADDQ      X8, X1
+	PADDQ      X8, X1
+	PXOR       X1, X7
+	PSHUFB     X11, X7
+	MOVO       X5, X8
+	PMULULQ    X7, X8
+	PADDQ      X7, X5
+	PADDQ      X8, X5
+	PADDQ      X8, X5
+	PXOR       X5, X3
+	MOVO       X3, X8
+	PADDQ      X3, X8
+	PSRLQ      $0x3f, X3
+	PXOR       X8, X3
+	MOVO       X4, X8
+	MOVO       X5, X4
+	MOVO       X8, X5
+	MOVO       X6, X8
+	PUNPCKLQDQ X6, X9
+	PUNPCKHQDQ X7, X6
+	PUNPCKHQDQ X9, X6
+	PUNPCKLQDQ X7, X9
+	MOVO       X8, X7
+	MOVO       X2, X8
+	PUNPCKHQDQ X9, X7
+	PUNPCKLQDQ X3, X9
+	PUNPCKHQDQ X9, X2
+	PUNPCKLQDQ X8, X9
+	PUNPCKHQDQ X9, X3
+	MOVO       X0, X8
+	PMULULQ    X2, X8
+	PADDQ      X2, X0
+	PADDQ      X8, X0
+	PADDQ      X8, X0
+	PXOR       X0, X6
+	PSHUFD     $0xb1, X6, X6
+	MOVO       X4, X8
+	PMULULQ    X6, X8
+	PADDQ      X6, X4
+	PADDQ      X8, X4
+	PADDQ      X8, X4
+	PXOR       X4, X2
+	PSHUFB     X10, X2
+	MOVO       X0, X8
+	PMULULQ    X2, X8
+	PADDQ      X2, X0
+	PADDQ      X8, X0
+	PADDQ      X8, X0
+	PXOR       X0, X6
+	PSHUFB     X11, X6
+	MOVO       X4, X8
+	PMULULQ    X6, X8
+	PADDQ      X6, X4
+	PADDQ      X8, X4
+	PADDQ      X8, X4
+	PXOR       X4, X2
+	MOVO       X2, X8
+	PADDQ      X2, X8
+	PSRLQ      $0x3f, X2
+	PXOR       X8, X2
+	MOVO       X1, X8
+	PMULULQ    X3, X8
+	PADDQ      X3, X1
+	PADDQ      X8, X1
+	PADDQ      X8, X1
+	PXOR       X1, X7
+	PSHUFD     $0xb1, X7, X7
+	MOVO       X5, X8
+	PMULULQ    X7, X8
+	PADDQ      X7, X5
+	PADDQ      X8, X5
+	PADDQ      X8, X5
+	PXOR       X5, X3
+	PSHUFB     X10, X3
+	MOVO       X1, X8
+	PMULULQ    X3, X8
+	PADDQ      X3, X1
+	PADDQ      X8, X1
+	PADDQ      X8, X1
+	PXOR       X1, X7
+	PSHUFB     X11, X7
+	MOVO       X5, X8
+	PMULULQ    X7, X8
+	PADDQ      X7, X5
+	PADDQ      X8, X5
+	PADDQ      X8, X5
+	PXOR       X5, X3
+	MOVO       X3, X8
+	PADDQ      X3, X8
+	PSRLQ      $0x3f, X3
+	PXOR       X8, X3
+	MOVO       X4, X8
+	MOVO       X5, X4
+	MOVO       X8, X5
+	MOVO       X2, X8
+	PUNPCKLQDQ X2, X9
+	PUNPCKHQDQ X3, X2
+	PUNPCKHQDQ X9, X2
+	PUNPCKLQDQ X3, X9
+	MOVO       X8, X3
+	MOVO       X6, X8
+	PUNPCKHQDQ X9, X3
+	PUNPCKLQDQ X7, X9
+	PUNPCKHQDQ X9, X6
+	PUNPCKLQDQ X8, X9
+	PUNPCKHQDQ X9, X7
+	MOVOU      X0, 96(AX)
+	MOVOU      X1, 224(AX)
+	MOVOU      X2, 352(AX)
+	MOVOU      X3, 480(AX)
+	MOVOU      X4, 608(AX)
+	MOVOU      X5, 736(AX)
+	MOVOU      X6, 864(AX)
+	MOVOU      X7, 992(AX)
+	MOVOU      112(AX), X0
+	MOVOU      240(AX), X1
+	MOVOU      368(AX), X2
+	MOVOU      496(AX), X3
+	MOVOU      624(AX), X4
+	MOVOU      752(AX), X5
+	MOVOU      880(AX), X6
+	MOVOU      1008(AX), X7
+	MOVO       X0, X8
+	PMULULQ    X2, X8
+	PADDQ      X2, X0
+	PADDQ      X8, X0
+	PADDQ      X8, X0
+	PXOR       X0, X6
+	PSHUFD     $0xb1, X6, X6
+	MOVO       X4, X8
+	PMULULQ    X6, X8
+	PADDQ      X6, X4
+	PADDQ      X8, X4
+	PADDQ      X8, X4
+	PXOR       X4, X2
+	PSHUFB     X10, X2
+	MOVO       X0, X8
+	PMULULQ    X2, X8
+	PADDQ      X2, X0
+	PADDQ      X8, X0
+	PADDQ      X8, X0
+	PXOR       X0, X6
+	PSHUFB     X11, X6
+	MOVO       X4, X8
+	PMULULQ    X6, X8
+	PADDQ      X6, X4
+	PADDQ      X8, X4
+	PADDQ      X8, X4
+	PXOR       X4, X2
+	MOVO       X2, X8
+	PADDQ      X2, X8
+	PSRLQ      $0x3f, X2
+	PXOR       X8, X2
+	MOVO       X1, X8
+	PMULULQ    X3, X8
+	PADDQ      X3, X1
+	PADDQ      X8, X1
+	PADDQ      X8, X1
+	PXOR       X1, X7
+	PSHUFD     $0xb1, X7, X7
+	MOVO       X5, X8
+	PMULULQ    X7, X8
+	PADDQ      X7, X5
+	PADDQ      X8, X5
+	PADDQ      X8, X5
+	PXOR       X5, X3
+	PSHUFB     X10, X3
+	MOVO       X1, X8
+	PMULULQ    X3, X8
+	PADDQ      X3, X1
+	PADDQ      X8, X1
+	PADDQ      X8, X1
+	PXOR       X1, X7
+	PSHUFB     X11, X7
+	MOVO       X5, X8
+	PMULULQ    X7, X8
+	PADDQ      X7, X5
+	PADDQ      X8, X5
+	PADDQ      X8, X5
+	PXOR       X5, X3
+	MOVO       X3, X8
+	PADDQ      X3, X8
+	PSRLQ      $0x3f, X3
+	PXOR       X8, X3
+	MOVO       X4, X8
+	MOVO       X5, X4
+	MOVO       X8, X5
+	MOVO       X6, X8
+	PUNPCKLQDQ X6, X9
+	PUNPCKHQDQ X7, X6
+	PUNPCKHQDQ X9, X6
+	PUNPCKLQDQ X7, X9
+	MOVO       X8, X7
+	MOVO       X2, X8
+	PUNPCKHQDQ X9, X7
+	PUNPCKLQDQ X3, X9
+	PUNPCKHQDQ X9, X2
+	PUNPCKLQDQ X8, X9
+	PUNPCKHQDQ X9, X3
+	MOVO       X0, X8
+	PMULULQ    X2, X8
+	PADDQ      X2, X0
+	PADDQ      X8, X0
+	PADDQ      X8, X0
+	PXOR       X0, X6
+	PSHUFD     $0xb1, X6, X6
+	MOVO       X4, X8
+	PMULULQ    X6, X8
+	PADDQ      X6, X4
+	PADDQ      X8, X4
+	PADDQ      X8, X4
+	PXOR       X4, X2
+	PSHUFB     X10, X2
+	MOVO       X0, X8
+	PMULULQ    X2, X8
+	PADDQ      X2, X0
+	PADDQ      X8, X0
+	PADDQ      X8, X0
+	PXOR       X0, X6
+	PSHUFB     X11, X6
+	MOVO       X4, X8
+	PMULULQ    X6, X8
+	PADDQ      X6, X4
+	PADDQ      X8, X4
+	PADDQ      X8, X4
+	PXOR       X4, X2
+	MOVO       X2, X8
+	PADDQ      X2, X8
+	PSRLQ      $0x3f, X2
+	PXOR       X8, X2
+	MOVO       X1, X8
+	PMULULQ    X3, X8
+	PADDQ      X3, X1
+	PADDQ      X8, X1
+	PADDQ      X8, X1
+	PXOR       X1, X7
+	PSHUFD     $0xb1, X7, X7
+	MOVO       X5, X8
+	PMULULQ    X7, X8
+	PADDQ      X7, X5
+	PADDQ      X8, X5
+	PADDQ      X8, X5
+	PXOR       X5, X3
+	PSHUFB     X10, X3
+	MOVO       X1, X8
+	PMULULQ    X3, X8
+	PADDQ      X3, X1
+	PADDQ      X8, X1
+	PADDQ      X8, X1
+	PXOR       X1, X7
+	PSHUFB     X11, X7
+	MOVO       X5, X8
+	PMULULQ    X7, X8
+	PADDQ      X7, X5
+	PADDQ      X8, X5
+	PADDQ      X8, X5
+	PXOR       X5, X3
+	MOVO       X3, X8
+	PADDQ      X3, X8
+	PSRLQ      $0x3f, X3
+	PXOR       X8, X3
+	MOVO       X4, X8
+	MOVO       X5, X4
+	MOVO       X8, X5
+	MOVO       X2, X8
+	PUNPCKLQDQ X2, X9
+	PUNPCKHQDQ X3, X2
+	PUNPCKHQDQ X9, X2
+	PUNPCKLQDQ X3, X9
+	MOVO       X8, X3
+	MOVO       X6, X8
+	PUNPCKHQDQ X9, X3
+	PUNPCKLQDQ X7, X9
+	PUNPCKHQDQ X9, X6
+	PUNPCKLQDQ X8, X9
+	PUNPCKHQDQ X9, X7
+	MOVOU      X0, 112(AX)
+	MOVOU      X1, 240(AX)
+	MOVOU      X2, 368(AX)
+	MOVOU      X3, 496(AX)
+	MOVOU      X4, 624(AX)
+	MOVOU      X5, 752(AX)
+	MOVOU      X6, 880(AX)
+	MOVOU      X7, 1008(AX)
+	RET
 
-	BLAMKA_ROUND_0(AX, 0, X8, X9, X10, X11)
-	BLAMKA_ROUND_0(AX, 16, X8, X9, X10, X11)
-	BLAMKA_ROUND_0(AX, 32, X8, X9, X10, X11)
-	BLAMKA_ROUND_0(AX, 48, X8, X9, X10, X11)
-	BLAMKA_ROUND_0(AX, 64, X8, X9, X10, X11)
-	BLAMKA_ROUND_0(AX, 80, X8, X9, X10, X11)
-	BLAMKA_ROUND_0(AX, 96, X8, X9, X10, X11)
-	BLAMKA_ROUND_0(AX, 112, X8, X9, X10, X11)
+DATA ·c40<>+0(SB)/8, $0x0201000706050403
+DATA ·c40<>+8(SB)/8, $0x0a09080f0e0d0c0b
+GLOBL ·c40<>(SB), RODATA|NOPTR, $16
 
-	BLAMKA_ROUND_1(AX, 0, X8, X9, X10, X11)
-	BLAMKA_ROUND_1(AX, 2, X8, X9, X10, X11)
-	BLAMKA_ROUND_1(AX, 4, X8, X9, X10, X11)
-	BLAMKA_ROUND_1(AX, 6, X8, X9, X10, X11)
-	BLAMKA_ROUND_1(AX, 8, X8, X9, X10, X11)
-	BLAMKA_ROUND_1(AX, 10, X8, X9, X10, X11)
-	BLAMKA_ROUND_1(AX, 12, X8, X9, X10, X11)
-	BLAMKA_ROUND_1(AX, 14, X8, X9, X10, X11)
-	RET
+DATA ·c48<>+0(SB)/8, $0x0100070605040302
+DATA ·c48<>+8(SB)/8, $0x09080f0e0d0c0b0a
+GLOBL ·c48<>(SB), RODATA|NOPTR, $16
 
-// func mixBlocksSSE2(out, a, b, c *block)
-TEXT ·mixBlocksSSE2(SB), 4, $0-32
+// func mixBlocksSSE2(out *block, a *block, b *block, c *block)
+// Requires: SSE2
+TEXT ·mixBlocksSSE2(SB), NOSPLIT, $0-32
 	MOVQ out+0(FP), DX
 	MOVQ a+8(FP), AX
 	MOVQ b+16(FP), BX
 	MOVQ c+24(FP), CX
-	MOVQ $128, DI
+	MOVQ $0x00000080, DI
 
 loop:
-	MOVOU 0(AX), X0
-	MOVOU 0(BX), X1
-	MOVOU 0(CX), X2
+	MOVOU (AX), X0
+	MOVOU (BX), X1
+	MOVOU (CX), X2
 	PXOR  X1, X0
 	PXOR  X2, X0
-	MOVOU X0, 0(DX)
-	ADDQ  $16, AX
-	ADDQ  $16, BX
-	ADDQ  $16, CX
-	ADDQ  $16, DX
-	SUBQ  $2, DI
+	MOVOU X0, (DX)
+	ADDQ  $0x10, AX
+	ADDQ  $0x10, BX
+	ADDQ  $0x10, CX
+	ADDQ  $0x10, DX
+	SUBQ  $0x02, DI
 	JA    loop
 	RET
 
-// func xorBlocksSSE2(out, a, b, c *block)
-TEXT ·xorBlocksSSE2(SB), 4, $0-32
+// func xorBlocksSSE2(out *block, a *block, b *block, c *block)
+// Requires: SSE2
+TEXT ·xorBlocksSSE2(SB), NOSPLIT, $0-32
 	MOVQ out+0(FP), DX
 	MOVQ a+8(FP), AX
 	MOVQ b+16(FP), BX
 	MOVQ c+24(FP), CX
-	MOVQ $128, DI
+	MOVQ $0x00000080, DI
 
 loop:
-	MOVOU 0(AX), X0
-	MOVOU 0(BX), X1
-	MOVOU 0(CX), X2
-	MOVOU 0(DX), X3
+	MOVOU (AX), X0
+	MOVOU (BX), X1
+	MOVOU (CX), X2
+	MOVOU (DX), X3
 	PXOR  X1, X0
 	PXOR  X2, X0
 	PXOR  X3, X0
-	MOVOU X0, 0(DX)
-	ADDQ  $16, AX
-	ADDQ  $16, BX
-	ADDQ  $16, CX
-	ADDQ  $16, DX
-	SUBQ  $2, DI
+	MOVOU X0, (DX)
+	ADDQ  $0x10, AX
+	ADDQ  $0x10, BX
+	ADDQ  $0x10, CX
+	ADDQ  $0x10, DX
+	SUBQ  $0x02, DI
 	JA    loop
 	RET
diff --git a/vendor/golang.org/x/crypto/blake2b/blake2bAVX2_amd64.s b/vendor/golang.org/x/crypto/blake2b/blake2bAVX2_amd64.s
index 9ae8206c20165..f75162e039c10 100644
--- a/vendor/golang.org/x/crypto/blake2b/blake2bAVX2_amd64.s
+++ b/vendor/golang.org/x/crypto/blake2b/blake2bAVX2_amd64.s
@@ -1,722 +1,4517 @@
-// Copyright 2016 The Go Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
+// Code generated by command: go run blake2bAVX2_amd64_asm.go -out ../../blake2bAVX2_amd64.s -pkg blake2b. DO NOT EDIT.
 
 //go:build amd64 && gc && !purego
 
 #include "textflag.h"
 
-DATA ·AVX2_iv0<>+0x00(SB)/8, $0x6a09e667f3bcc908
-DATA ·AVX2_iv0<>+0x08(SB)/8, $0xbb67ae8584caa73b
-DATA ·AVX2_iv0<>+0x10(SB)/8, $0x3c6ef372fe94f82b
-DATA ·AVX2_iv0<>+0x18(SB)/8, $0xa54ff53a5f1d36f1
-GLOBL ·AVX2_iv0<>(SB), (NOPTR+RODATA), $32
-
-DATA ·AVX2_iv1<>+0x00(SB)/8, $0x510e527fade682d1
-DATA ·AVX2_iv1<>+0x08(SB)/8, $0x9b05688c2b3e6c1f
-DATA ·AVX2_iv1<>+0x10(SB)/8, $0x1f83d9abfb41bd6b
-DATA ·AVX2_iv1<>+0x18(SB)/8, $0x5be0cd19137e2179
-GLOBL ·AVX2_iv1<>(SB), (NOPTR+RODATA), $32
-
-DATA ·AVX2_c40<>+0x00(SB)/8, $0x0201000706050403
-DATA ·AVX2_c40<>+0x08(SB)/8, $0x0a09080f0e0d0c0b
-DATA ·AVX2_c40<>+0x10(SB)/8, $0x0201000706050403
-DATA ·AVX2_c40<>+0x18(SB)/8, $0x0a09080f0e0d0c0b
-GLOBL ·AVX2_c40<>(SB), (NOPTR+RODATA), $32
-
-DATA ·AVX2_c48<>+0x00(SB)/8, $0x0100070605040302
-DATA ·AVX2_c48<>+0x08(SB)/8, $0x09080f0e0d0c0b0a
-DATA ·AVX2_c48<>+0x10(SB)/8, $0x0100070605040302
-DATA ·AVX2_c48<>+0x18(SB)/8, $0x09080f0e0d0c0b0a
-GLOBL ·AVX2_c48<>(SB), (NOPTR+RODATA), $32
-
-DATA ·AVX_iv0<>+0x00(SB)/8, $0x6a09e667f3bcc908
-DATA ·AVX_iv0<>+0x08(SB)/8, $0xbb67ae8584caa73b
-GLOBL ·AVX_iv0<>(SB), (NOPTR+RODATA), $16
-
-DATA ·AVX_iv1<>+0x00(SB)/8, $0x3c6ef372fe94f82b
-DATA ·AVX_iv1<>+0x08(SB)/8, $0xa54ff53a5f1d36f1
-GLOBL ·AVX_iv1<>(SB), (NOPTR+RODATA), $16
-
-DATA ·AVX_iv2<>+0x00(SB)/8, $0x510e527fade682d1
-DATA ·AVX_iv2<>+0x08(SB)/8, $0x9b05688c2b3e6c1f
-GLOBL ·AVX_iv2<>(SB), (NOPTR+RODATA), $16
-
-DATA ·AVX_iv3<>+0x00(SB)/8, $0x1f83d9abfb41bd6b
-DATA ·AVX_iv3<>+0x08(SB)/8, $0x5be0cd19137e2179
-GLOBL ·AVX_iv3<>(SB), (NOPTR+RODATA), $16
-
-DATA ·AVX_c40<>+0x00(SB)/8, $0x0201000706050403
-DATA ·AVX_c40<>+0x08(SB)/8, $0x0a09080f0e0d0c0b
-GLOBL ·AVX_c40<>(SB), (NOPTR+RODATA), $16
-
-DATA ·AVX_c48<>+0x00(SB)/8, $0x0100070605040302
-DATA ·AVX_c48<>+0x08(SB)/8, $0x09080f0e0d0c0b0a
-GLOBL ·AVX_c48<>(SB), (NOPTR+RODATA), $16
-
-#define VPERMQ_0x39_Y1_Y1 BYTE $0xc4; BYTE $0xe3; BYTE $0xfd; BYTE $0x00; BYTE $0xc9; BYTE $0x39
-#define VPERMQ_0x93_Y1_Y1 BYTE $0xc4; BYTE $0xe3; BYTE $0xfd; BYTE $0x00; BYTE $0xc9; BYTE $0x93
-#define VPERMQ_0x4E_Y2_Y2 BYTE $0xc4; BYTE $0xe3; BYTE $0xfd; BYTE $0x00; BYTE $0xd2; BYTE $0x4e
-#define VPERMQ_0x93_Y3_Y3 BYTE $0xc4; BYTE $0xe3; BYTE $0xfd; BYTE $0x00; BYTE $0xdb; BYTE $0x93
-#define VPERMQ_0x39_Y3_Y3 BYTE $0xc4; BYTE $0xe3; BYTE $0xfd; BYTE $0x00; BYTE $0xdb; BYTE $0x39
-
-#define ROUND_AVX2(m0, m1, m2, m3, t, c40, c48) \
-	VPADDQ  m0, Y0, Y0;   \
-	VPADDQ  Y1, Y0, Y0;   \
-	VPXOR   Y0, Y3, Y3;   \
-	VPSHUFD $-79, Y3, Y3; \
-	VPADDQ  Y3, Y2, Y2;   \
-	VPXOR   Y2, Y1, Y1;   \
-	VPSHUFB c40, Y1, Y1;  \
-	VPADDQ  m1, Y0, Y0;   \
-	VPADDQ  Y1, Y0, Y0;   \
-	VPXOR   Y0, Y3, Y3;   \
-	VPSHUFB c48, Y3, Y3;  \
-	VPADDQ  Y3, Y2, Y2;   \
-	VPXOR   Y2, Y1, Y1;   \
-	VPADDQ  Y1, Y1, t;    \
-	VPSRLQ  $63, Y1, Y1;  \
-	VPXOR   t, Y1, Y1;    \
-	VPERMQ_0x39_Y1_Y1;    \
-	VPERMQ_0x4E_Y2_Y2;    \
-	VPERMQ_0x93_Y3_Y3;    \
-	VPADDQ  m2, Y0, Y0;   \
-	VPADDQ  Y1, Y0, Y0;   \
-	VPXOR   Y0, Y3, Y3;   \
-	VPSHUFD $-79, Y3, Y3; \
-	VPADDQ  Y3, Y2, Y2;   \
-	VPXOR   Y2, Y1, Y1;   \
-	VPSHUFB c40, Y1, Y1;  \
-	VPADDQ  m3, Y0, Y0;   \
-	VPADDQ  Y1, Y0, Y0;   \
-	VPXOR   Y0, Y3, Y3;   \
-	VPSHUFB c48, Y3, Y3;  \
-	VPADDQ  Y3, Y2, Y2;   \
-	VPXOR   Y2, Y1, Y1;   \
-	VPADDQ  Y1, Y1, t;    \
-	VPSRLQ  $63, Y1, Y1;  \
-	VPXOR   t, Y1, Y1;    \
-	VPERMQ_0x39_Y3_Y3;    \
-	VPERMQ_0x4E_Y2_Y2;    \
-	VPERMQ_0x93_Y1_Y1
-
-#define VMOVQ_SI_X11_0 BYTE $0xC5; BYTE $0x7A; BYTE $0x7E; BYTE $0x1E
-#define VMOVQ_SI_X12_0 BYTE $0xC5; BYTE $0x7A; BYTE $0x7E; BYTE $0x26
-#define VMOVQ_SI_X13_0 BYTE $0xC5; BYTE $0x7A; BYTE $0x7E; BYTE $0x2E
-#define VMOVQ_SI_X14_0 BYTE $0xC5; BYTE $0x7A; BYTE $0x7E; BYTE $0x36
-#define VMOVQ_SI_X15_0 BYTE $0xC5; BYTE $0x7A; BYTE $0x7E; BYTE $0x3E
-
-#define VMOVQ_SI_X11(n) BYTE $0xC5; BYTE $0x7A; BYTE $0x7E; BYTE $0x5E; BYTE $n
-#define VMOVQ_SI_X12(n) BYTE $0xC5; BYTE $0x7A; BYTE $0x7E; BYTE $0x66; BYTE $n
-#define VMOVQ_SI_X13(n) BYTE $0xC5; BYTE $0x7A; BYTE $0x7E; BYTE $0x6E; BYTE $n
-#define VMOVQ_SI_X14(n) BYTE $0xC5; BYTE $0x7A; BYTE $0x7E; BYTE $0x76; BYTE $n
-#define VMOVQ_SI_X15(n) BYTE $0xC5; BYTE $0x7A; BYTE $0x7E; BYTE $0x7E; BYTE $n
-
-#define VPINSRQ_1_SI_X11_0 BYTE $0xC4; BYTE $0x63; BYTE $0xA1; BYTE $0x22; BYTE $0x1E; BYTE $0x01
-#define VPINSRQ_1_SI_X12_0 BYTE $0xC4; BYTE $0x63; BYTE $0x99; BYTE $0x22; BYTE $0x26; BYTE $0x01
-#define VPINSRQ_1_SI_X13_0 BYTE $0xC4; BYTE $0x63; BYTE $0x91; BYTE $0x22; BYTE $0x2E; BYTE $0x01
-#define VPINSRQ_1_SI_X14_0 BYTE $0xC4; BYTE $0x63; BYTE $0x89; BYTE $0x22; BYTE $0x36; BYTE $0x01
-#define VPINSRQ_1_SI_X15_0 BYTE $0xC4; BYTE $0x63; BYTE $0x81; BYTE $0x22; BYTE $0x3E; BYTE $0x01
-
-#define VPINSRQ_1_SI_X11(n) BYTE $0xC4; BYTE $0x63; BYTE $0xA1; BYTE $0x22; BYTE $0x5E; BYTE $n; BYTE $0x01
-#define VPINSRQ_1_SI_X12(n) BYTE $0xC4; BYTE $0x63; BYTE $0x99; BYTE $0x22; BYTE $0x66; BYTE $n; BYTE $0x01
-#define VPINSRQ_1_SI_X13(n) BYTE $0xC4; BYTE $0x63; BYTE $0x91; BYTE $0x22; BYTE $0x6E; BYTE $n; BYTE $0x01
-#define VPINSRQ_1_SI_X14(n) BYTE $0xC4; BYTE $0x63; BYTE $0x89; BYTE $0x22; BYTE $0x76; BYTE $n; BYTE $0x01
-#define VPINSRQ_1_SI_X15(n) BYTE $0xC4; BYTE $0x63; BYTE $0x81; BYTE $0x22; BYTE $0x7E; BYTE $n; BYTE $0x01
-
-#define VMOVQ_R8_X15 BYTE $0xC4; BYTE $0x41; BYTE $0xF9; BYTE $0x6E; BYTE $0xF8
-#define VPINSRQ_1_R9_X15 BYTE $0xC4; BYTE $0x43; BYTE $0x81; BYTE $0x22; BYTE $0xF9; BYTE $0x01
-
-// load msg: Y12 = (i0, i1, i2, i3)
-// i0, i1, i2, i3 must not be 0
-#define LOAD_MSG_AVX2_Y12(i0, i1, i2, i3) \
-	VMOVQ_SI_X12(i0*8);           \
-	VMOVQ_SI_X11(i2*8);           \
-	VPINSRQ_1_SI_X12(i1*8);       \
-	VPINSRQ_1_SI_X11(i3*8);       \
-	VINSERTI128 $1, X11, Y12, Y12
-
-// load msg: Y13 = (i0, i1, i2, i3)
-// i0, i1, i2, i3 must not be 0
-#define LOAD_MSG_AVX2_Y13(i0, i1, i2, i3) \
-	VMOVQ_SI_X13(i0*8);           \
-	VMOVQ_SI_X11(i2*8);           \
-	VPINSRQ_1_SI_X13(i1*8);       \
-	VPINSRQ_1_SI_X11(i3*8);       \
-	VINSERTI128 $1, X11, Y13, Y13
-
-// load msg: Y14 = (i0, i1, i2, i3)
-// i0, i1, i2, i3 must not be 0
-#define LOAD_MSG_AVX2_Y14(i0, i1, i2, i3) \
-	VMOVQ_SI_X14(i0*8);           \
-	VMOVQ_SI_X11(i2*8);           \
-	VPINSRQ_1_SI_X14(i1*8);       \
-	VPINSRQ_1_SI_X11(i3*8);       \
-	VINSERTI128 $1, X11, Y14, Y14
-
-// load msg: Y15 = (i0, i1, i2, i3)
-// i0, i1, i2, i3 must not be 0
-#define LOAD_MSG_AVX2_Y15(i0, i1, i2, i3) \
-	VMOVQ_SI_X15(i0*8);           \
-	VMOVQ_SI_X11(i2*8);           \
-	VPINSRQ_1_SI_X15(i1*8);       \
-	VPINSRQ_1_SI_X11(i3*8);       \
-	VINSERTI128 $1, X11, Y15, Y15
-
-#define LOAD_MSG_AVX2_0_2_4_6_1_3_5_7_8_10_12_14_9_11_13_15() \
-	VMOVQ_SI_X12_0;                   \
-	VMOVQ_SI_X11(4*8);                \
-	VPINSRQ_1_SI_X12(2*8);            \
-	VPINSRQ_1_SI_X11(6*8);            \
-	VINSERTI128 $1, X11, Y12, Y12;    \
-	LOAD_MSG_AVX2_Y13(1, 3, 5, 7);    \
-	LOAD_MSG_AVX2_Y14(8, 10, 12, 14); \
-	LOAD_MSG_AVX2_Y15(9, 11, 13, 15)
-
-#define LOAD_MSG_AVX2_14_4_9_13_10_8_15_6_1_0_11_5_12_2_7_3() \
-	LOAD_MSG_AVX2_Y12(14, 4, 9, 13); \
-	LOAD_MSG_AVX2_Y13(10, 8, 15, 6); \
-	VMOVQ_SI_X11(11*8);              \
-	VPSHUFD     $0x4E, 0*8(SI), X14; \
-	VPINSRQ_1_SI_X11(5*8);           \
-	VINSERTI128 $1, X11, Y14, Y14;   \
-	LOAD_MSG_AVX2_Y15(12, 2, 7, 3)
-
-#define LOAD_MSG_AVX2_11_12_5_15_8_0_2_13_10_3_7_9_14_6_1_4() \
-	VMOVQ_SI_X11(5*8);              \
-	VMOVDQU     11*8(SI), X12;      \
-	VPINSRQ_1_SI_X11(15*8);         \
-	VINSERTI128 $1, X11, Y12, Y12;  \
-	VMOVQ_SI_X13(8*8);              \
-	VMOVQ_SI_X11(2*8);              \
-	VPINSRQ_1_SI_X13_0;             \
-	VPINSRQ_1_SI_X11(13*8);         \
-	VINSERTI128 $1, X11, Y13, Y13;  \
-	LOAD_MSG_AVX2_Y14(10, 3, 7, 9); \
-	LOAD_MSG_AVX2_Y15(14, 6, 1, 4)
-
-#define LOAD_MSG_AVX2_7_3_13_11_9_1_12_14_2_5_4_15_6_10_0_8() \
-	LOAD_MSG_AVX2_Y12(7, 3, 13, 11); \
-	LOAD_MSG_AVX2_Y13(9, 1, 12, 14); \
-	LOAD_MSG_AVX2_Y14(2, 5, 4, 15);  \
-	VMOVQ_SI_X15(6*8);               \
-	VMOVQ_SI_X11_0;                  \
-	VPINSRQ_1_SI_X15(10*8);          \
-	VPINSRQ_1_SI_X11(8*8);           \
-	VINSERTI128 $1, X11, Y15, Y15
-
-#define LOAD_MSG_AVX2_9_5_2_10_0_7_4_15_14_11_6_3_1_12_8_13() \
-	LOAD_MSG_AVX2_Y12(9, 5, 2, 10);  \
-	VMOVQ_SI_X13_0;                  \
-	VMOVQ_SI_X11(4*8);               \
-	VPINSRQ_1_SI_X13(7*8);           \
-	VPINSRQ_1_SI_X11(15*8);          \
-	VINSERTI128 $1, X11, Y13, Y13;   \
-	LOAD_MSG_AVX2_Y14(14, 11, 6, 3); \
-	LOAD_MSG_AVX2_Y15(1, 12, 8, 13)
-
-#define LOAD_MSG_AVX2_2_6_0_8_12_10_11_3_4_7_15_1_13_5_14_9() \
-	VMOVQ_SI_X12(2*8);                \
-	VMOVQ_SI_X11_0;                   \
-	VPINSRQ_1_SI_X12(6*8);            \
-	VPINSRQ_1_SI_X11(8*8);            \
-	VINSERTI128 $1, X11, Y12, Y12;    \
-	LOAD_MSG_AVX2_Y13(12, 10, 11, 3); \
-	LOAD_MSG_AVX2_Y14(4, 7, 15, 1);   \
-	LOAD_MSG_AVX2_Y15(13, 5, 14, 9)
-
-#define LOAD_MSG_AVX2_12_1_14_4_5_15_13_10_0_6_9_8_7_3_2_11() \
-	LOAD_MSG_AVX2_Y12(12, 1, 14, 4);  \
-	LOAD_MSG_AVX2_Y13(5, 15, 13, 10); \
-	VMOVQ_SI_X14_0;                   \
-	VPSHUFD     $0x4E, 8*8(SI), X11;  \
-	VPINSRQ_1_SI_X14(6*8);            \
-	VINSERTI128 $1, X11, Y14, Y14;    \
-	LOAD_MSG_AVX2_Y15(7, 3, 2, 11)
-
-#define LOAD_MSG_AVX2_13_7_12_3_11_14_1_9_5_15_8_2_0_4_6_10() \
-	LOAD_MSG_AVX2_Y12(13, 7, 12, 3); \
-	LOAD_MSG_AVX2_Y13(11, 14, 1, 9); \
-	LOAD_MSG_AVX2_Y14(5, 15, 8, 2);  \
-	VMOVQ_SI_X15_0;                  \
-	VMOVQ_SI_X11(6*8);               \
-	VPINSRQ_1_SI_X15(4*8);           \
-	VPINSRQ_1_SI_X11(10*8);          \
-	VINSERTI128 $1, X11, Y15, Y15
-
-#define LOAD_MSG_AVX2_6_14_11_0_15_9_3_8_12_13_1_10_2_7_4_5() \
-	VMOVQ_SI_X12(6*8);              \
-	VMOVQ_SI_X11(11*8);             \
-	VPINSRQ_1_SI_X12(14*8);         \
-	VPINSRQ_1_SI_X11_0;             \
-	VINSERTI128 $1, X11, Y12, Y12;  \
-	LOAD_MSG_AVX2_Y13(15, 9, 3, 8); \
-	VMOVQ_SI_X11(1*8);              \
-	VMOVDQU     12*8(SI), X14;      \
-	VPINSRQ_1_SI_X11(10*8);         \
-	VINSERTI128 $1, X11, Y14, Y14;  \
-	VMOVQ_SI_X15(2*8);              \
-	VMOVDQU     4*8(SI), X11;       \
-	VPINSRQ_1_SI_X15(7*8);          \
-	VINSERTI128 $1, X11, Y15, Y15
-
-#define LOAD_MSG_AVX2_10_8_7_1_2_4_6_5_15_9_3_13_11_14_12_0() \
-	LOAD_MSG_AVX2_Y12(10, 8, 7, 1);  \
-	VMOVQ_SI_X13(2*8);               \
-	VPSHUFD     $0x4E, 5*8(SI), X11; \
-	VPINSRQ_1_SI_X13(4*8);           \
-	VINSERTI128 $1, X11, Y13, Y13;   \
-	LOAD_MSG_AVX2_Y14(15, 9, 3, 13); \
-	VMOVQ_SI_X15(11*8);              \
-	VMOVQ_SI_X11(12*8);              \
-	VPINSRQ_1_SI_X15(14*8);          \
-	VPINSRQ_1_SI_X11_0;              \
-	VINSERTI128 $1, X11, Y15, Y15
-
 // func hashBlocksAVX2(h *[8]uint64, c *[2]uint64, flag uint64, blocks []byte)
-TEXT ·hashBlocksAVX2(SB), 4, $320-48 // frame size = 288 + 32 byte alignment
-	MOVQ h+0(FP), AX
-	MOVQ c+8(FP), BX
-	MOVQ flag+16(FP), CX
-	MOVQ blocks_base+24(FP), SI
-	MOVQ blocks_len+32(FP), DI
-
-	MOVQ SP, DX
-	ADDQ $31, DX
-	ANDQ $~31, DX
-
-	MOVQ CX, 16(DX)
-	XORQ CX, CX
-	MOVQ CX, 24(DX)
-
-	VMOVDQU ·AVX2_c40<>(SB), Y4
-	VMOVDQU ·AVX2_c48<>(SB), Y5
-
-	VMOVDQU 0(AX), Y8
+// Requires: AVX, AVX2
+TEXT ·hashBlocksAVX2(SB), NOSPLIT, $320-48
+	MOVQ    h+0(FP), AX
+	MOVQ    c+8(FP), BX
+	MOVQ    flag+16(FP), CX
+	MOVQ    blocks_base+24(FP), SI
+	MOVQ    blocks_len+32(FP), DI
+	MOVQ    SP, DX
+	ADDQ    $+31, DX
+	ANDQ    $-32, DX
+	MOVQ    CX, 16(DX)
+	XORQ    CX, CX
+	MOVQ    CX, 24(DX)
+	VMOVDQU ·AVX2_c40<>+0(SB), Y4
+	VMOVDQU ·AVX2_c48<>+0(SB), Y5
+	VMOVDQU (AX), Y8
 	VMOVDQU 32(AX), Y9
-	VMOVDQU ·AVX2_iv0<>(SB), Y6
-	VMOVDQU ·AVX2_iv1<>(SB), Y7
-
-	MOVQ 0(BX), R8
-	MOVQ 8(BX), R9
-	MOVQ R9, 8(DX)
+	VMOVDQU ·AVX2_iv0<>+0(SB), Y6
+	VMOVDQU ·AVX2_iv1<>+0(SB), Y7
+	MOVQ    (BX), R8
+	MOVQ    8(BX), R9
+	MOVQ    R9, 8(DX)
 
 loop:
-	ADDQ $128, R8
-	MOVQ R8, 0(DX)
-	CMPQ R8, $128
+	ADDQ $0x80, R8
+	MOVQ R8, (DX)
+	CMPQ R8, $0x80
 	JGE  noinc
 	INCQ R9
 	MOVQ R9, 8(DX)
 
 noinc:
-	VMOVDQA Y8, Y0
-	VMOVDQA Y9, Y1
-	VMOVDQA Y6, Y2
-	VPXOR   0(DX), Y7, Y3
-
-	LOAD_MSG_AVX2_0_2_4_6_1_3_5_7_8_10_12_14_9_11_13_15()
-	VMOVDQA Y12, 32(DX)
-	VMOVDQA Y13, 64(DX)
-	VMOVDQA Y14, 96(DX)
-	VMOVDQA Y15, 128(DX)
-	ROUND_AVX2(Y12, Y13, Y14, Y15, Y10, Y4, Y5)
-	LOAD_MSG_AVX2_14_4_9_13_10_8_15_6_1_0_11_5_12_2_7_3()
-	VMOVDQA Y12, 160(DX)
-	VMOVDQA Y13, 192(DX)
-	VMOVDQA Y14, 224(DX)
-	VMOVDQA Y15, 256(DX)
-
-	ROUND_AVX2(Y12, Y13, Y14, Y15, Y10, Y4, Y5)
-	LOAD_MSG_AVX2_11_12_5_15_8_0_2_13_10_3_7_9_14_6_1_4()
-	ROUND_AVX2(Y12, Y13, Y14, Y15, Y10, Y4, Y5)
-	LOAD_MSG_AVX2_7_3_13_11_9_1_12_14_2_5_4_15_6_10_0_8()
-	ROUND_AVX2(Y12, Y13, Y14, Y15, Y10, Y4, Y5)
-	LOAD_MSG_AVX2_9_5_2_10_0_7_4_15_14_11_6_3_1_12_8_13()
-	ROUND_AVX2(Y12, Y13, Y14, Y15, Y10, Y4, Y5)
-	LOAD_MSG_AVX2_2_6_0_8_12_10_11_3_4_7_15_1_13_5_14_9()
-	ROUND_AVX2(Y12, Y13, Y14, Y15, Y10, Y4, Y5)
-	LOAD_MSG_AVX2_12_1_14_4_5_15_13_10_0_6_9_8_7_3_2_11()
-	ROUND_AVX2(Y12, Y13, Y14, Y15, Y10, Y4, Y5)
-	LOAD_MSG_AVX2_13_7_12_3_11_14_1_9_5_15_8_2_0_4_6_10()
-	ROUND_AVX2(Y12, Y13, Y14, Y15, Y10, Y4, Y5)
-	LOAD_MSG_AVX2_6_14_11_0_15_9_3_8_12_13_1_10_2_7_4_5()
-	ROUND_AVX2(Y12, Y13, Y14, Y15, Y10, Y4, Y5)
-	LOAD_MSG_AVX2_10_8_7_1_2_4_6_5_15_9_3_13_11_14_12_0()
-	ROUND_AVX2(Y12, Y13, Y14, Y15, Y10, Y4, Y5)
-
-	ROUND_AVX2(32(DX), 64(DX), 96(DX), 128(DX), Y10, Y4, Y5)
-	ROUND_AVX2(160(DX), 192(DX), 224(DX), 256(DX), Y10, Y4, Y5)
-
-	VPXOR Y0, Y8, Y8
-	VPXOR Y1, Y9, Y9
-	VPXOR Y2, Y8, Y8
-	VPXOR Y3, Y9, Y9
-
-	LEAQ 128(SI), SI
-	SUBQ $128, DI
-	JNE  loop
-
-	MOVQ R8, 0(BX)
-	MOVQ R9, 8(BX)
-
-	VMOVDQU Y8, 0(AX)
-	VMOVDQU Y9, 32(AX)
+	VMOVDQA     Y8, Y0
+	VMOVDQA     Y9, Y1
+	VMOVDQA     Y6, Y2
+	VPXOR       (DX), Y7, Y3
+	BYTE        $0xc5
+	BYTE        $0x7a
+	BYTE        $0x7e
+	BYTE        $0x26
+	BYTE        $0xc5
+	BYTE        $0x7a
+	BYTE        $0x7e
+	BYTE        $0x5e
+	BYTE        $0x20
+	BYTE        $0xc4
+	BYTE        $0x63
+	BYTE        $0x99
+	BYTE        $0x22
+	BYTE        $0x66
+	BYTE        $0x10
+	BYTE        $0x01
+	BYTE        $0xc4
+	BYTE        $0x63
+	BYTE        $0xa1
+	BYTE        $0x22
+	BYTE        $0x5e
+	BYTE        $0x30
+	BYTE        $0x01
+	VINSERTI128 $0x01, X11, Y12, Y12
+	BYTE        $0xc5
+	BYTE        $0x7a
+	BYTE        $0x7e
+	BYTE        $0x6e
+	BYTE        $0x08
+	BYTE        $0xc5
+	BYTE        $0x7a
+	BYTE        $0x7e
+	BYTE        $0x5e
+	BYTE        $0x28
+	BYTE        $0xc4
+	BYTE        $0x63
+	BYTE        $0x91
+	BYTE        $0x22
+	BYTE        $0x6e
+	BYTE        $0x18
+	BYTE        $0x01
+	BYTE        $0xc4
+	BYTE        $0x63
+	BYTE        $0xa1
+	BYTE        $0x22
+	BYTE        $0x5e
+	BYTE        $0x38
+	BYTE        $0x01
+	VINSERTI128 $0x01, X11, Y13, Y13
+	BYTE        $0xc5
+	BYTE        $0x7a
+	BYTE        $0x7e
+	BYTE        $0x76
+	BYTE        $0x40
+	BYTE        $0xc5
+	BYTE        $0x7a
+	BYTE        $0x7e
+	BYTE        $0x5e
+	BYTE        $0x60
+	BYTE        $0xc4
+	BYTE        $0x63
+	BYTE        $0x89
+	BYTE        $0x22
+	BYTE        $0x76
+	BYTE        $0x50
+	BYTE        $0x01
+	BYTE        $0xc4
+	BYTE        $0x63
+	BYTE        $0xa1
+	BYTE        $0x22
+	BYTE        $0x5e
+	BYTE        $0x70
+	BYTE        $0x01
+	VINSERTI128 $0x01, X11, Y14, Y14
+	BYTE        $0xc5
+	BYTE        $0x7a
+	BYTE        $0x7e
+	BYTE        $0x7e
+	BYTE        $0x48
+	BYTE        $0xc5
+	BYTE        $0x7a
+	BYTE        $0x7e
+	BYTE        $0x5e
+	BYTE        $0x68
+	BYTE        $0xc4
+	BYTE        $0x63
+	BYTE        $0x81
+	BYTE        $0x22
+	BYTE        $0x7e
+	BYTE        $0x58
+	BYTE        $0x01
+	BYTE        $0xc4
+	BYTE        $0x63
+	BYTE        $0xa1
+	BYTE        $0x22
+	BYTE        $0x5e
+	BYTE        $0x78
+	BYTE        $0x01
+	VINSERTI128 $0x01, X11, Y15, Y15
+	VMOVDQA     Y12, 32(DX)
+	VMOVDQA     Y13, 64(DX)
+	VMOVDQA     Y14, 96(DX)
+	VMOVDQA     Y15, 128(DX)
+	VPADDQ      Y12, Y0, Y0
+	VPADDQ      Y1, Y0, Y0
+	VPXOR       Y0, Y3, Y3
+	VPSHUFD     $-79, Y3, Y3
+	VPADDQ      Y3, Y2, Y2
+	VPXOR       Y2, Y1, Y1
+	VPSHUFB     Y4, Y1, Y1
+	VPADDQ      Y13, Y0, Y0
+	VPADDQ      Y1, Y0, Y0
+	VPXOR       Y0, Y3, Y3
+	VPSHUFB     Y5, Y3, Y3
+	VPADDQ      Y3, Y2, Y2
+	VPXOR       Y2, Y1, Y1
+	VPADDQ      Y1, Y1, Y10
+	VPSRLQ      $0x3f, Y1, Y1
+	VPXOR       Y10, Y1, Y1
+	BYTE        $0xc4
+	BYTE        $0xe3
+	BYTE        $0xfd
+	BYTE        $0x00
+	BYTE        $0xc9
+	BYTE        $0x39
+	BYTE        $0xc4
+	BYTE        $0xe3
+	BYTE        $0xfd
+	BYTE        $0x00
+	BYTE        $0xd2
+	BYTE        $0x4e
+	BYTE        $0xc4
+	BYTE        $0xe3
+	BYTE        $0xfd
+	BYTE        $0x00
+	BYTE        $0xdb
+	BYTE        $0x93
+	VPADDQ      Y14, Y0, Y0
+	VPADDQ      Y1, Y0, Y0
+	VPXOR       Y0, Y3, Y3
+	VPSHUFD     $-79, Y3, Y3
+	VPADDQ      Y3, Y2, Y2
+	VPXOR       Y2, Y1, Y1
+	VPSHUFB     Y4, Y1, Y1
+	VPADDQ      Y15, Y0, Y0
+	VPADDQ      Y1, Y0, Y0
+	VPXOR       Y0, Y3, Y3
+	VPSHUFB     Y5, Y3, Y3
+	VPADDQ      Y3, Y2, Y2
+	VPXOR       Y2, Y1, Y1
+	VPADDQ      Y1, Y1, Y10
+	VPSRLQ      $0x3f, Y1, Y1
+	VPXOR       Y10, Y1, Y1
+	BYTE        $0xc4
+	BYTE        $0xe3
+	BYTE        $0xfd
+	BYTE        $0x00
+	BYTE        $0xdb
+	BYTE        $0x39
+	BYTE        $0xc4
+	BYTE        $0xe3
+	BYTE        $0xfd
+	BYTE        $0x00
+	BYTE        $0xd2
+	BYTE        $0x4e
+	BYTE        $0xc4
+	BYTE        $0xe3
+	BYTE        $0xfd
+	BYTE        $0x00
+	BYTE        $0xc9
+	BYTE        $0x93
+	BYTE        $0xc5
+	BYTE        $0x7a
+	BYTE        $0x7e
+	BYTE        $0x66
+	BYTE        $0x70
+	BYTE        $0xc5
+	BYTE        $0x7a
+	BYTE        $0x7e
+	BYTE        $0x5e
+	BYTE        $0x48
+	BYTE        $0xc4
+	BYTE        $0x63
+	BYTE        $0x99
+	BYTE        $0x22
+	BYTE        $0x66
+	BYTE        $0x20
+	BYTE        $0x01
+	BYTE        $0xc4
+	BYTE        $0x63
+	BYTE        $0xa1
+	BYTE        $0x22
+	BYTE        $0x5e
+	BYTE        $0x68
+	BYTE        $0x01
+	VINSERTI128 $0x01, X11, Y12, Y12
+	BYTE        $0xc5
+	BYTE        $0x7a
+	BYTE        $0x7e
+	BYTE        $0x6e
+	BYTE        $0x50
+	BYTE        $0xc5
+	BYTE        $0x7a
+	BYTE        $0x7e
+	BYTE        $0x5e
+	BYTE        $0x78
+	BYTE        $0xc4
+	BYTE        $0x63
+	BYTE        $0x91
+	BYTE        $0x22
+	BYTE        $0x6e
+	BYTE        $0x40
+	BYTE        $0x01
+	BYTE        $0xc4
+	BYTE        $0x63
+	BYTE        $0xa1
+	BYTE        $0x22
+	BYTE        $0x5e
+	BYTE        $0x30
+	BYTE        $0x01
+	VINSERTI128 $0x01, X11, Y13, Y13
+	BYTE        $0xc5
+	BYTE        $0x7a
+	BYTE        $0x7e
+	BYTE        $0x5e
+	BYTE        $0x58
+	VPSHUFD     $0x4e, (SI), X14
+	BYTE        $0xc4
+	BYTE        $0x63
+	BYTE        $0xa1
+	BYTE        $0x22
+	BYTE        $0x5e
+	BYTE        $0x28
+	BYTE        $0x01
+	VINSERTI128 $0x01, X11, Y14, Y14
+	BYTE        $0xc5
+	BYTE        $0x7a
+	BYTE        $0x7e
+	BYTE        $0x7e
+	BYTE        $0x60
+	BYTE        $0xc5
+	BYTE        $0x7a
+	BYTE        $0x7e
+	BYTE        $0x5e
+	BYTE        $0x38
+	BYTE        $0xc4
+	BYTE        $0x63
+	BYTE        $0x81
+	BYTE        $0x22
+	BYTE        $0x7e
+	BYTE        $0x10
+	BYTE        $0x01
+	BYTE        $0xc4
+	BYTE        $0x63
+	BYTE        $0xa1
+	BYTE        $0x22
+	BYTE        $0x5e
+	BYTE        $0x18
+	BYTE        $0x01
+	VINSERTI128 $0x01, X11, Y15, Y15
+	VMOVDQA     Y12, 160(DX)
+	VMOVDQA     Y13, 192(DX)
+	VMOVDQA     Y14, 224(DX)
+	VMOVDQA     Y15, 256(DX)
+	VPADDQ      Y12, Y0, Y0
+	VPADDQ      Y1, Y0, Y0
+	VPXOR       Y0, Y3, Y3
+	VPSHUFD     $-79, Y3, Y3
+	VPADDQ      Y3, Y2, Y2
+	VPXOR       Y2, Y1, Y1
+	VPSHUFB     Y4, Y1, Y1
+	VPADDQ      Y13, Y0, Y0
+	VPADDQ      Y1, Y0, Y0
+	VPXOR       Y0, Y3, Y3
+	VPSHUFB     Y5, Y3, Y3
+	VPADDQ      Y3, Y2, Y2
+	VPXOR       Y2, Y1, Y1
+	VPADDQ      Y1, Y1, Y10
+	VPSRLQ      $0x3f, Y1, Y1
+	VPXOR       Y10, Y1, Y1
+	BYTE        $0xc4
+	BYTE        $0xe3
+	BYTE        $0xfd
+	BYTE        $0x00
+	BYTE        $0xc9
+	BYTE        $0x39
+	BYTE        $0xc4
+	BYTE        $0xe3
+	BYTE        $0xfd
+	BYTE        $0x00
+	BYTE        $0xd2
+	BYTE        $0x4e
+	BYTE        $0xc4
+	BYTE        $0xe3
+	BYTE        $0xfd
+	BYTE        $0x00
+	BYTE        $0xdb
+	BYTE        $0x93
+	VPADDQ      Y14, Y0, Y0
+	VPADDQ      Y1, Y0, Y0
+	VPXOR       Y0, Y3, Y3
+	VPSHUFD     $-79, Y3, Y3
+	VPADDQ      Y3, Y2, Y2
+	VPXOR       Y2, Y1, Y1
+	VPSHUFB     Y4, Y1, Y1
+	VPADDQ      Y15, Y0, Y0
+	VPADDQ      Y1, Y0, Y0
+	VPXOR       Y0, Y3, Y3
+	VPSHUFB     Y5, Y3, Y3
+	VPADDQ      Y3, Y2, Y2
+	VPXOR       Y2, Y1, Y1
+	VPADDQ      Y1, Y1, Y10
+	VPSRLQ      $0x3f, Y1, Y1
+	VPXOR       Y10, Y1, Y1
+	BYTE        $0xc4
+	BYTE        $0xe3
+	BYTE        $0xfd
+	BYTE        $0x00
+	BYTE        $0xdb
+	BYTE        $0x39
+	BYTE        $0xc4
+	BYTE        $0xe3
+	BYTE        $0xfd
+	BYTE        $0x00
+	BYTE        $0xd2
+	BYTE        $0x4e
+	BYTE        $0xc4
+	BYTE        $0xe3
+	BYTE        $0xfd
+	BYTE        $0x00
+	BYTE        $0xc9
+	BYTE        $0x93
+	BYTE        $0xc5
+	BYTE        $0x7a
+	BYTE        $0x7e
+	BYTE        $0x5e
+	BYTE        $0x28
+	VMOVDQU     88(SI), X12
+	BYTE        $0xc4
+	BYTE        $0x63
+	BYTE        $0xa1
+	BYTE        $0x22
+	BYTE        $0x5e
+	BYTE        $0x78
+	BYTE        $0x01
+	VINSERTI128 $0x01, X11, Y12, Y12
+	BYTE        $0xc5
+	BYTE        $0x7a
+	BYTE        $0x7e
+	BYTE        $0x6e
+	BYTE        $0x40
+	BYTE        $0xc5
+	BYTE        $0x7a
+	BYTE        $0x7e
+	BYTE        $0x5e
+	BYTE        $0x10
+	BYTE        $0xc4
+	BYTE        $0x63
+	BYTE        $0x91
+	BYTE        $0x22
+	BYTE        $0x2e
+	BYTE        $0x01
+	BYTE        $0xc4
+	BYTE        $0x63
+	BYTE        $0xa1
+	BYTE        $0x22
+	BYTE        $0x5e
+	BYTE        $0x68
+	BYTE        $0x01
+	VINSERTI128 $0x01, X11, Y13, Y13
+	BYTE        $0xc5
+	BYTE        $0x7a
+	BYTE        $0x7e
+	BYTE        $0x76
+	BYTE        $0x50
+	BYTE        $0xc5
+	BYTE        $0x7a
+	BYTE        $0x7e
+	BYTE        $0x5e
+	BYTE        $0x38
+	BYTE        $0xc4
+	BYTE        $0x63
+	BYTE        $0x89
+	BYTE        $0x22
+	BYTE        $0x76
+	BYTE        $0x18
+	BYTE        $0x01
+	BYTE        $0xc4
+	BYTE        $0x63
+	BYTE        $0xa1
+	BYTE        $0x22
+	BYTE        $0x5e
+	BYTE        $0x48
+	BYTE        $0x01
+	VINSERTI128 $0x01, X11, Y14, Y14
+	BYTE        $0xc5
+	BYTE        $0x7a
+	BYTE        $0x7e
+	BYTE        $0x7e
+	BYTE        $0x70
+	BYTE        $0xc5
+	BYTE        $0x7a
+	BYTE        $0x7e
+	BYTE        $0x5e
+	BYTE        $0x08
+	BYTE        $0xc4
+	BYTE        $0x63
+	BYTE        $0x81
+	BYTE        $0x22
+	BYTE        $0x7e
+	BYTE        $0x30
+	BYTE        $0x01
+	BYTE        $0xc4
+	BYTE        $0x63
+	BYTE        $0xa1
+	BYTE        $0x22
+	BYTE        $0x5e
+	BYTE        $0x20
+	BYTE        $0x01
+	VINSERTI128 $0x01, X11, Y15, Y15
+	VPADDQ      Y12, Y0, Y0
+	VPADDQ      Y1, Y0, Y0
+	VPXOR       Y0, Y3, Y3
+	VPSHUFD     $-79, Y3, Y3
+	VPADDQ      Y3, Y2, Y2
+	VPXOR       Y2, Y1, Y1
+	VPSHUFB     Y4, Y1, Y1
+	VPADDQ      Y13, Y0, Y0
+	VPADDQ      Y1, Y0, Y0
+	VPXOR       Y0, Y3, Y3
+	VPSHUFB     Y5, Y3, Y3
+	VPADDQ      Y3, Y2, Y2
+	VPXOR       Y2, Y1, Y1
+	VPADDQ      Y1, Y1, Y10
+	VPSRLQ      $0x3f, Y1, Y1
+	VPXOR       Y10, Y1, Y1
+	BYTE        $0xc4
+	BYTE        $0xe3
+	BYTE        $0xfd
+	BYTE        $0x00
+	BYTE        $0xc9
+	BYTE        $0x39
+	BYTE        $0xc4
+	BYTE        $0xe3
+	BYTE        $0xfd
+	BYTE        $0x00
+	BYTE        $0xd2
+	BYTE        $0x4e
+	BYTE        $0xc4
+	BYTE        $0xe3
+	BYTE        $0xfd
+	BYTE        $0x00
+	BYTE        $0xdb
+	BYTE        $0x93
+	VPADDQ      Y14, Y0, Y0
+	VPADDQ      Y1, Y0, Y0
+	VPXOR       Y0, Y3, Y3
+	VPSHUFD     $-79, Y3, Y3
+	VPADDQ      Y3, Y2, Y2
+	VPXOR       Y2, Y1, Y1
+	VPSHUFB     Y4, Y1, Y1
+	VPADDQ      Y15, Y0, Y0
+	VPADDQ      Y1, Y0, Y0
+	VPXOR       Y0, Y3, Y3
+	VPSHUFB     Y5, Y3, Y3
+	VPADDQ      Y3, Y2, Y2
+	VPXOR       Y2, Y1, Y1
+	VPADDQ      Y1, Y1, Y10
+	VPSRLQ      $0x3f, Y1, Y1
+	VPXOR       Y10, Y1, Y1
+	BYTE        $0xc4
+	BYTE        $0xe3
+	BYTE        $0xfd
+	BYTE        $0x00
+	BYTE        $0xdb
+	BYTE        $0x39
+	BYTE        $0xc4
+	BYTE        $0xe3
+	BYTE        $0xfd
+	BYTE        $0x00
+	BYTE        $0xd2
+	BYTE        $0x4e
+	BYTE        $0xc4
+	BYTE        $0xe3
+	BYTE        $0xfd
+	BYTE        $0x00
+	BYTE        $0xc9
+	BYTE        $0x93
+	BYTE        $0xc5
+	BYTE        $0x7a
+	BYTE        $0x7e
+	BYTE        $0x66
+	BYTE        $0x38
+	BYTE        $0xc5
+	BYTE        $0x7a
+	BYTE        $0x7e
+	BYTE        $0x5e
+	BYTE        $0x68
+	BYTE        $0xc4
+	BYTE        $0x63
+	BYTE        $0x99
+	BYTE        $0x22
+	BYTE        $0x66
+	BYTE        $0x18
+	BYTE        $0x01
+	BYTE        $0xc4
+	BYTE        $0x63
+	BYTE        $0xa1
+	BYTE        $0x22
+	BYTE        $0x5e
+	BYTE        $0x58
+	BYTE        $0x01
+	VINSERTI128 $0x01, X11, Y12, Y12
+	BYTE        $0xc5
+	BYTE        $0x7a
+	BYTE        $0x7e
+	BYTE        $0x6e
+	BYTE        $0x48
+	BYTE        $0xc5
+	BYTE        $0x7a
+	BYTE        $0x7e
+	BYTE        $0x5e
+	BYTE        $0x60
+	BYTE        $0xc4
+	BYTE        $0x63
+	BYTE        $0x91
+	BYTE        $0x22
+	BYTE        $0x6e
+	BYTE        $0x08
+	BYTE        $0x01
+	BYTE        $0xc4
+	BYTE        $0x63
+	BYTE        $0xa1
+	BYTE        $0x22
+	BYTE        $0x5e
+	BYTE        $0x70
+	BYTE        $0x01
+	VINSERTI128 $0x01, X11, Y13, Y13
+	BYTE        $0xc5
+	BYTE        $0x7a
+	BYTE        $0x7e
+	BYTE        $0x76
+	BYTE        $0x10
+	BYTE        $0xc5
+	BYTE        $0x7a
+	BYTE        $0x7e
+	BYTE        $0x5e
+	BYTE        $0x20
+	BYTE        $0xc4
+	BYTE        $0x63
+	BYTE        $0x89
+	BYTE        $0x22
+	BYTE        $0x76
+	BYTE        $0x28
+	BYTE        $0x01
+	BYTE        $0xc4
+	BYTE        $0x63
+	BYTE        $0xa1
+	BYTE        $0x22
+	BYTE        $0x5e
+	BYTE        $0x78
+	BYTE        $0x01
+	VINSERTI128 $0x01, X11, Y14, Y14
+	BYTE        $0xc5
+	BYTE        $0x7a
+	BYTE        $0x7e
+	BYTE        $0x7e
+	BYTE        $0x30
+	BYTE        $0xc5
+	BYTE        $0x7a
+	BYTE        $0x7e
+	BYTE        $0x1e
+	BYTE        $0xc4
+	BYTE        $0x63
+	BYTE        $0x81
+	BYTE        $0x22
+	BYTE        $0x7e
+	BYTE        $0x50
+	BYTE        $0x01
+	BYTE        $0xc4
+	BYTE        $0x63
+	BYTE        $0xa1
+	BYTE        $0x22
+	BYTE        $0x5e
+	BYTE        $0x40
+	BYTE        $0x01
+	VINSERTI128 $0x01, X11, Y15, Y15
+	VPADDQ      Y12, Y0, Y0
+	VPADDQ      Y1, Y0, Y0
+	VPXOR       Y0, Y3, Y3
+	VPSHUFD     $-79, Y3, Y3
+	VPADDQ      Y3, Y2, Y2
+	VPXOR       Y2, Y1, Y1
+	VPSHUFB     Y4, Y1, Y1
+	VPADDQ      Y13, Y0, Y0
+	VPADDQ      Y1, Y0, Y0
+	VPXOR       Y0, Y3, Y3
+	VPSHUFB     Y5, Y3, Y3
+	VPADDQ      Y3, Y2, Y2
+	VPXOR       Y2, Y1, Y1
+	VPADDQ      Y1, Y1, Y10
+	VPSRLQ      $0x3f, Y1, Y1
+	VPXOR       Y10, Y1, Y1
+	BYTE        $0xc4
+	BYTE        $0xe3
+	BYTE        $0xfd
+	BYTE        $0x00
+	BYTE        $0xc9
+	BYTE        $0x39
+	BYTE        $0xc4
+	BYTE        $0xe3
+	BYTE        $0xfd
+	BYTE        $0x00
+	BYTE        $0xd2
+	BYTE        $0x4e
+	BYTE        $0xc4
+	BYTE        $0xe3
+	BYTE        $0xfd
+	BYTE        $0x00
+	BYTE        $0xdb
+	BYTE        $0x93
+	VPADDQ      Y14, Y0, Y0
+	VPADDQ      Y1, Y0, Y0
+	VPXOR       Y0, Y3, Y3
+	VPSHUFD     $-79, Y3, Y3
+	VPADDQ      Y3, Y2, Y2
+	VPXOR       Y2, Y1, Y1
+	VPSHUFB     Y4, Y1, Y1
+	VPADDQ      Y15, Y0, Y0
+	VPADDQ      Y1, Y0, Y0
+	VPXOR       Y0, Y3, Y3
+	VPSHUFB     Y5, Y3, Y3
+	VPADDQ      Y3, Y2, Y2
+	VPXOR       Y2, Y1, Y1
+	VPADDQ      Y1, Y1, Y10
+	VPSRLQ      $0x3f, Y1, Y1
+	VPXOR       Y10, Y1, Y1
+	BYTE        $0xc4
+	BYTE        $0xe3
+	BYTE        $0xfd
+	BYTE        $0x00
+	BYTE        $0xdb
+	BYTE        $0x39
+	BYTE        $0xc4
+	BYTE        $0xe3
+	BYTE        $0xfd
+	BYTE        $0x00
+	BYTE        $0xd2
+	BYTE        $0x4e
+	BYTE        $0xc4
+	BYTE        $0xe3
+	BYTE        $0xfd
+	BYTE        $0x00
+	BYTE        $0xc9
+	BYTE        $0x93
+	BYTE        $0xc5
+	BYTE        $0x7a
+	BYTE        $0x7e
+	BYTE        $0x66
+	BYTE        $0x48
+	BYTE        $0xc5
+	BYTE        $0x7a
+	BYTE        $0x7e
+	BYTE        $0x5e
+	BYTE        $0x10
+	BYTE        $0xc4
+	BYTE        $0x63
+	BYTE        $0x99
+	BYTE        $0x22
+	BYTE        $0x66
+	BYTE        $0x28
+	BYTE        $0x01
+	BYTE        $0xc4
+	BYTE        $0x63
+	BYTE        $0xa1
+	BYTE        $0x22
+	BYTE        $0x5e
+	BYTE        $0x50
+	BYTE        $0x01
+	VINSERTI128 $0x01, X11, Y12, Y12
+	BYTE        $0xc5
+	BYTE        $0x7a
+	BYTE        $0x7e
+	BYTE        $0x2e
+	BYTE        $0xc5
+	BYTE        $0x7a
+	BYTE        $0x7e
+	BYTE        $0x5e
+	BYTE        $0x20
+	BYTE        $0xc4
+	BYTE        $0x63
+	BYTE        $0x91
+	BYTE        $0x22
+	BYTE        $0x6e
+	BYTE        $0x38
+	BYTE        $0x01
+	BYTE        $0xc4
+	BYTE        $0x63
+	BYTE        $0xa1
+	BYTE        $0x22
+	BYTE        $0x5e
+	BYTE        $0x78
+	BYTE        $0x01
+	VINSERTI128 $0x01, X11, Y13, Y13
+	BYTE        $0xc5
+	BYTE        $0x7a
+	BYTE        $0x7e
+	BYTE        $0x76
+	BYTE        $0x70
+	BYTE        $0xc5
+	BYTE        $0x7a
+	BYTE        $0x7e
+	BYTE        $0x5e
+	BYTE        $0x30
+	BYTE        $0xc4
+	BYTE        $0x63
+	BYTE        $0x89
+	BYTE        $0x22
+	BYTE        $0x76
+	BYTE        $0x58
+	BYTE        $0x01
+	BYTE        $0xc4
+	BYTE        $0x63
+	BYTE        $0xa1
+	BYTE        $0x22
+	BYTE        $0x5e
+	BYTE        $0x18
+	BYTE        $0x01
+	VINSERTI128 $0x01, X11, Y14, Y14
+	BYTE        $0xc5
+	BYTE        $0x7a
+	BYTE        $0x7e
+	BYTE        $0x7e
+	BYTE        $0x08
+	BYTE        $0xc5
+	BYTE        $0x7a
+	BYTE        $0x7e
+	BYTE        $0x5e
+	BYTE        $0x40
+	BYTE        $0xc4
+	BYTE        $0x63
+	BYTE        $0x81
+	BYTE        $0x22
+	BYTE        $0x7e
+	BYTE        $0x60
+	BYTE        $0x01
+	BYTE        $0xc4
+	BYTE        $0x63
+	BYTE        $0xa1
+	BYTE        $0x22
+	BYTE        $0x5e
+	BYTE        $0x68
+	BYTE        $0x01
+	VINSERTI128 $0x01, X11, Y15, Y15
+	VPADDQ      Y12, Y0, Y0
+	VPADDQ      Y1, Y0, Y0
+	VPXOR       Y0, Y3, Y3
+	VPSHUFD     $-79, Y3, Y3
+	VPADDQ      Y3, Y2, Y2
+	VPXOR       Y2, Y1, Y1
+	VPSHUFB     Y4, Y1, Y1
+	VPADDQ      Y13, Y0, Y0
+	VPADDQ      Y1, Y0, Y0
+	VPXOR       Y0, Y3, Y3
+	VPSHUFB     Y5, Y3, Y3
+	VPADDQ      Y3, Y2, Y2
+	VPXOR       Y2, Y1, Y1
+	VPADDQ      Y1, Y1, Y10
+	VPSRLQ      $0x3f, Y1, Y1
+	VPXOR       Y10, Y1, Y1
+	BYTE        $0xc4
+	BYTE        $0xe3
+	BYTE        $0xfd
+	BYTE        $0x00
+	BYTE        $0xc9
+	BYTE        $0x39
+	BYTE        $0xc4
+	BYTE        $0xe3
+	BYTE        $0xfd
+	BYTE        $0x00
+	BYTE        $0xd2
+	BYTE        $0x4e
+	BYTE        $0xc4
+	BYTE        $0xe3
+	BYTE        $0xfd
+	BYTE        $0x00
+	BYTE        $0xdb
+	BYTE        $0x93
+	VPADDQ      Y14, Y0, Y0
+	VPADDQ      Y1, Y0, Y0
+	VPXOR       Y0, Y3, Y3
+	VPSHUFD     $-79, Y3, Y3
+	VPADDQ      Y3, Y2, Y2
+	VPXOR       Y2, Y1, Y1
+	VPSHUFB     Y4, Y1, Y1
+	VPADDQ      Y15, Y0, Y0
+	VPADDQ      Y1, Y0, Y0
+	VPXOR       Y0, Y3, Y3
+	VPSHUFB     Y5, Y3, Y3
+	VPADDQ      Y3, Y2, Y2
+	VPXOR       Y2, Y1, Y1
+	VPADDQ      Y1, Y1, Y10
+	VPSRLQ      $0x3f, Y1, Y1
+	VPXOR       Y10, Y1, Y1
+	BYTE        $0xc4
+	BYTE        $0xe3
+	BYTE        $0xfd
+	BYTE        $0x00
+	BYTE        $0xdb
+	BYTE        $0x39
+	BYTE        $0xc4
+	BYTE        $0xe3
+	BYTE        $0xfd
+	BYTE        $0x00
+	BYTE        $0xd2
+	BYTE        $0x4e
+	BYTE        $0xc4
+	BYTE        $0xe3
+	BYTE        $0xfd
+	BYTE        $0x00
+	BYTE        $0xc9
+	BYTE        $0x93
+	BYTE        $0xc5
+	BYTE        $0x7a
+	BYTE        $0x7e
+	BYTE        $0x66
+	BYTE        $0x10
+	BYTE        $0xc5
+	BYTE        $0x7a
+	BYTE        $0x7e
+	BYTE        $0x1e
+	BYTE        $0xc4
+	BYTE        $0x63
+	BYTE        $0x99
+	BYTE        $0x22
+	BYTE        $0x66
+	BYTE        $0x30
+	BYTE        $0x01
+	BYTE        $0xc4
+	BYTE        $0x63
+	BYTE        $0xa1
+	BYTE        $0x22
+	BYTE        $0x5e
+	BYTE        $0x40
+	BYTE        $0x01
+	VINSERTI128 $0x01, X11, Y12, Y12
+	BYTE        $0xc5
+	BYTE        $0x7a
+	BYTE        $0x7e
+	BYTE        $0x6e
+	BYTE        $0x60
+	BYTE        $0xc5
+	BYTE        $0x7a
+	BYTE        $0x7e
+	BYTE        $0x5e
+	BYTE        $0x58
+	BYTE        $0xc4
+	BYTE        $0x63
+	BYTE        $0x91
+	BYTE        $0x22
+	BYTE        $0x6e
+	BYTE        $0x50
+	BYTE        $0x01
+	BYTE        $0xc4
+	BYTE        $0x63
+	BYTE        $0xa1
+	BYTE        $0x22
+	BYTE        $0x5e
+	BYTE        $0x18
+	BYTE        $0x01
+	VINSERTI128 $0x01, X11, Y13, Y13
+	BYTE        $0xc5
+	BYTE        $0x7a
+	BYTE        $0x7e
+	BYTE        $0x76
+	BYTE        $0x20
+	BYTE        $0xc5
+	BYTE        $0x7a
+	BYTE        $0x7e
+	BYTE        $0x5e
+	BYTE        $0x78
+	BYTE        $0xc4
+	BYTE        $0x63
+	BYTE        $0x89
+	BYTE        $0x22
+	BYTE        $0x76
+	BYTE        $0x38
+	BYTE        $0x01
+	BYTE        $0xc4
+	BYTE        $0x63
+	BYTE        $0xa1
+	BYTE        $0x22
+	BYTE        $0x5e
+	BYTE        $0x08
+	BYTE        $0x01
+	VINSERTI128 $0x01, X11, Y14, Y14
+	BYTE        $0xc5
+	BYTE        $0x7a
+	BYTE        $0x7e
+	BYTE        $0x7e
+	BYTE        $0x68
+	BYTE        $0xc5
+	BYTE        $0x7a
+	BYTE        $0x7e
+	BYTE        $0x5e
+	BYTE        $0x70
+	BYTE        $0xc4
+	BYTE        $0x63
+	BYTE        $0x81
+	BYTE        $0x22
+	BYTE        $0x7e
+	BYTE        $0x28
+	BYTE        $0x01
+	BYTE        $0xc4
+	BYTE        $0x63
+	BYTE        $0xa1
+	BYTE        $0x22
+	BYTE        $0x5e
+	BYTE        $0x48
+	BYTE        $0x01
+	VINSERTI128 $0x01, X11, Y15, Y15
+	VPADDQ      Y12, Y0, Y0
+	VPADDQ      Y1, Y0, Y0
+	VPXOR       Y0, Y3, Y3
+	VPSHUFD     $-79, Y3, Y3
+	VPADDQ      Y3, Y2, Y2
+	VPXOR       Y2, Y1, Y1
+	VPSHUFB     Y4, Y1, Y1
+	VPADDQ      Y13, Y0, Y0
+	VPADDQ      Y1, Y0, Y0
+	VPXOR       Y0, Y3, Y3
+	VPSHUFB     Y5, Y3, Y3
+	VPADDQ      Y3, Y2, Y2
+	VPXOR       Y2, Y1, Y1
+	VPADDQ      Y1, Y1, Y10
+	VPSRLQ      $0x3f, Y1, Y1
+	VPXOR       Y10, Y1, Y1
+	BYTE        $0xc4
+	BYTE        $0xe3
+	BYTE        $0xfd
+	BYTE        $0x00
+	BYTE        $0xc9
+	BYTE        $0x39
+	BYTE        $0xc4
+	BYTE        $0xe3
+	BYTE        $0xfd
+	BYTE        $0x00
+	BYTE        $0xd2
+	BYTE        $0x4e
+	BYTE        $0xc4
+	BYTE        $0xe3
+	BYTE        $0xfd
+	BYTE        $0x00
+	BYTE        $0xdb
+	BYTE        $0x93
+	VPADDQ      Y14, Y0, Y0
+	VPADDQ      Y1, Y0, Y0
+	VPXOR       Y0, Y3, Y3
+	VPSHUFD     $-79, Y3, Y3
+	VPADDQ      Y3, Y2, Y2
+	VPXOR       Y2, Y1, Y1
+	VPSHUFB     Y4, Y1, Y1
+	VPADDQ      Y15, Y0, Y0
+	VPADDQ      Y1, Y0, Y0
+	VPXOR       Y0, Y3, Y3
+	VPSHUFB     Y5, Y3, Y3
+	VPADDQ      Y3, Y2, Y2
+	VPXOR       Y2, Y1, Y1
+	VPADDQ      Y1, Y1, Y10
+	VPSRLQ      $0x3f, Y1, Y1
+	VPXOR       Y10, Y1, Y1
+	BYTE        $0xc4
+	BYTE        $0xe3
+	BYTE        $0xfd
+	BYTE        $0x00
+	BYTE        $0xdb
+	BYTE        $0x39
+	BYTE        $0xc4
+	BYTE        $0xe3
+	BYTE        $0xfd
+	BYTE        $0x00
+	BYTE        $0xd2
+	BYTE        $0x4e
+	BYTE        $0xc4
+	BYTE        $0xe3
+	BYTE        $0xfd
+	BYTE        $0x00
+	BYTE        $0xc9
+	BYTE        $0x93
+	BYTE        $0xc5
+	BYTE        $0x7a
+	BYTE        $0x7e
+	BYTE        $0x66
+	BYTE        $0x60
+	BYTE        $0xc5
+	BYTE        $0x7a
+	BYTE        $0x7e
+	BYTE        $0x5e
+	BYTE        $0x70
+	BYTE        $0xc4
+	BYTE        $0x63
+	BYTE        $0x99
+	BYTE        $0x22
+	BYTE        $0x66
+	BYTE        $0x08
+	BYTE        $0x01
+	BYTE        $0xc4
+	BYTE        $0x63
+	BYTE        $0xa1
+	BYTE        $0x22
+	BYTE        $0x5e
+	BYTE        $0x20
+	BYTE        $0x01
+	VINSERTI128 $0x01, X11, Y12, Y12
+	BYTE        $0xc5
+	BYTE        $0x7a
+	BYTE        $0x7e
+	BYTE        $0x6e
+	BYTE        $0x28
+	BYTE        $0xc5
+	BYTE        $0x7a
+	BYTE        $0x7e
+	BYTE        $0x5e
+	BYTE        $0x68
+	BYTE        $0xc4
+	BYTE        $0x63
+	BYTE        $0x91
+	BYTE        $0x22
+	BYTE        $0x6e
+	BYTE        $0x78
+	BYTE        $0x01
+	BYTE        $0xc4
+	BYTE        $0x63
+	BYTE        $0xa1
+	BYTE        $0x22
+	BYTE        $0x5e
+	BYTE        $0x50
+	BYTE        $0x01
+	VINSERTI128 $0x01, X11, Y13, Y13
+	BYTE        $0xc5
+	BYTE        $0x7a
+	BYTE        $0x7e
+	BYTE        $0x36
+	VPSHUFD     $0x4e, 64(SI), X11
+	BYTE        $0xc4
+	BYTE        $0x63
+	BYTE        $0x89
+	BYTE        $0x22
+	BYTE        $0x76
+	BYTE        $0x30
+	BYTE        $0x01
+	VINSERTI128 $0x01, X11, Y14, Y14
+	BYTE        $0xc5
+	BYTE        $0x7a
+	BYTE        $0x7e
+	BYTE        $0x7e
+	BYTE        $0x38
+	BYTE        $0xc5
+	BYTE        $0x7a
+	BYTE        $0x7e
+	BYTE        $0x5e
+	BYTE        $0x10
+	BYTE        $0xc4
+	BYTE        $0x63
+	BYTE        $0x81
+	BYTE        $0x22
+	BYTE        $0x7e
+	BYTE        $0x18
+	BYTE        $0x01
+	BYTE        $0xc4
+	BYTE        $0x63
+	BYTE        $0xa1
+	BYTE        $0x22
+	BYTE        $0x5e
+	BYTE        $0x58
+	BYTE        $0x01
+	VINSERTI128 $0x01, X11, Y15, Y15
+	VPADDQ      Y12, Y0, Y0
+	VPADDQ      Y1, Y0, Y0
+	VPXOR       Y0, Y3, Y3
+	VPSHUFD     $-79, Y3, Y3
+	VPADDQ      Y3, Y2, Y2
+	VPXOR       Y2, Y1, Y1
+	VPSHUFB     Y4, Y1, Y1
+	VPADDQ      Y13, Y0, Y0
+	VPADDQ      Y1, Y0, Y0
+	VPXOR       Y0, Y3, Y3
+	VPSHUFB     Y5, Y3, Y3
+	VPADDQ      Y3, Y2, Y2
+	VPXOR       Y2, Y1, Y1
+	VPADDQ      Y1, Y1, Y10
+	VPSRLQ      $0x3f, Y1, Y1
+	VPXOR       Y10, Y1, Y1
+	BYTE        $0xc4
+	BYTE        $0xe3
+	BYTE        $0xfd
+	BYTE        $0x00
+	BYTE        $0xc9
+	BYTE        $0x39
+	BYTE        $0xc4
+	BYTE        $0xe3
+	BYTE        $0xfd
+	BYTE        $0x00
+	BYTE        $0xd2
+	BYTE        $0x4e
+	BYTE        $0xc4
+	BYTE        $0xe3
+	BYTE        $0xfd
+	BYTE        $0x00
+	BYTE        $0xdb
+	BYTE        $0x93
+	VPADDQ      Y14, Y0, Y0
+	VPADDQ      Y1, Y0, Y0
+	VPXOR       Y0, Y3, Y3
+	VPSHUFD     $-79, Y3, Y3
+	VPADDQ      Y3, Y2, Y2
+	VPXOR       Y2, Y1, Y1
+	VPSHUFB     Y4, Y1, Y1
+	VPADDQ      Y15, Y0, Y0
+	VPADDQ      Y1, Y0, Y0
+	VPXOR       Y0, Y3, Y3
+	VPSHUFB     Y5, Y3, Y3
+	VPADDQ      Y3, Y2, Y2
+	VPXOR       Y2, Y1, Y1
+	VPADDQ      Y1, Y1, Y10
+	VPSRLQ      $0x3f, Y1, Y1
+	VPXOR       Y10, Y1, Y1
+	BYTE        $0xc4
+	BYTE        $0xe3
+	BYTE        $0xfd
+	BYTE        $0x00
+	BYTE        $0xdb
+	BYTE        $0x39
+	BYTE        $0xc4
+	BYTE        $0xe3
+	BYTE        $0xfd
+	BYTE        $0x00
+	BYTE        $0xd2
+	BYTE        $0x4e
+	BYTE        $0xc4
+	BYTE        $0xe3
+	BYTE        $0xfd
+	BYTE        $0x00
+	BYTE        $0xc9
+	BYTE        $0x93
+	BYTE        $0xc5
+	BYTE        $0x7a
+	BYTE        $0x7e
+	BYTE        $0x66
+	BYTE        $0x68
+	BYTE        $0xc5
+	BYTE        $0x7a
+	BYTE        $0x7e
+	BYTE        $0x5e
+	BYTE        $0x60
+	BYTE        $0xc4
+	BYTE        $0x63
+	BYTE        $0x99
+	BYTE        $0x22
+	BYTE        $0x66
+	BYTE        $0x38
+	BYTE        $0x01
+	BYTE        $0xc4
+	BYTE        $0x63
+	BYTE        $0xa1
+	BYTE        $0x22
+	BYTE        $0x5e
+	BYTE        $0x18
+	BYTE        $0x01
+	VINSERTI128 $0x01, X11, Y12, Y12
+	BYTE        $0xc5
+	BYTE        $0x7a
+	BYTE        $0x7e
+	BYTE        $0x6e
+	BYTE        $0x58
+	BYTE        $0xc5
+	BYTE        $0x7a
+	BYTE        $0x7e
+	BYTE        $0x5e
+	BYTE        $0x08
+	BYTE        $0xc4
+	BYTE        $0x63
+	BYTE        $0x91
+	BYTE        $0x22
+	BYTE        $0x6e
+	BYTE        $0x70
+	BYTE        $0x01
+	BYTE        $0xc4
+	BYTE        $0x63
+	BYTE        $0xa1
+	BYTE        $0x22
+	BYTE        $0x5e
+	BYTE        $0x48
+	BYTE        $0x01
+	VINSERTI128 $0x01, X11, Y13, Y13
+	BYTE        $0xc5
+	BYTE        $0x7a
+	BYTE        $0x7e
+	BYTE        $0x76
+	BYTE        $0x28
+	BYTE        $0xc5
+	BYTE        $0x7a
+	BYTE        $0x7e
+	BYTE        $0x5e
+	BYTE        $0x40
+	BYTE        $0xc4
+	BYTE        $0x63
+	BYTE        $0x89
+	BYTE        $0x22
+	BYTE        $0x76
+	BYTE        $0x78
+	BYTE        $0x01
+	BYTE        $0xc4
+	BYTE        $0x63
+	BYTE        $0xa1
+	BYTE        $0x22
+	BYTE        $0x5e
+	BYTE        $0x10
+	BYTE        $0x01
+	VINSERTI128 $0x01, X11, Y14, Y14
+	BYTE        $0xc5
+	BYTE        $0x7a
+	BYTE        $0x7e
+	BYTE        $0x3e
+	BYTE        $0xc5
+	BYTE        $0x7a
+	BYTE        $0x7e
+	BYTE        $0x5e
+	BYTE        $0x30
+	BYTE        $0xc4
+	BYTE        $0x63
+	BYTE        $0x81
+	BYTE        $0x22
+	BYTE        $0x7e
+	BYTE        $0x20
+	BYTE        $0x01
+	BYTE        $0xc4
+	BYTE        $0x63
+	BYTE        $0xa1
+	BYTE        $0x22
+	BYTE        $0x5e
+	BYTE        $0x50
+	BYTE        $0x01
+	VINSERTI128 $0x01, X11, Y15, Y15
+	VPADDQ      Y12, Y0, Y0
+	VPADDQ      Y1, Y0, Y0
+	VPXOR       Y0, Y3, Y3
+	VPSHUFD     $-79, Y3, Y3
+	VPADDQ      Y3, Y2, Y2
+	VPXOR       Y2, Y1, Y1
+	VPSHUFB     Y4, Y1, Y1
+	VPADDQ      Y13, Y0, Y0
+	VPADDQ      Y1, Y0, Y0
+	VPXOR       Y0, Y3, Y3
+	VPSHUFB     Y5, Y3, Y3
+	VPADDQ      Y3, Y2, Y2
+	VPXOR       Y2, Y1, Y1
+	VPADDQ      Y1, Y1, Y10
+	VPSRLQ      $0x3f, Y1, Y1
+	VPXOR       Y10, Y1, Y1
+	BYTE        $0xc4
+	BYTE        $0xe3
+	BYTE        $0xfd
+	BYTE        $0x00
+	BYTE        $0xc9
+	BYTE        $0x39
+	BYTE        $0xc4
+	BYTE        $0xe3
+	BYTE        $0xfd
+	BYTE        $0x00
+	BYTE        $0xd2
+	BYTE        $0x4e
+	BYTE        $0xc4
+	BYTE        $0xe3
+	BYTE        $0xfd
+	BYTE        $0x00
+	BYTE        $0xdb
+	BYTE        $0x93
+	VPADDQ      Y14, Y0, Y0
+	VPADDQ      Y1, Y0, Y0
+	VPXOR       Y0, Y3, Y3
+	VPSHUFD     $-79, Y3, Y3
+	VPADDQ      Y3, Y2, Y2
+	VPXOR       Y2, Y1, Y1
+	VPSHUFB     Y4, Y1, Y1
+	VPADDQ      Y15, Y0, Y0
+	VPADDQ      Y1, Y0, Y0
+	VPXOR       Y0, Y3, Y3
+	VPSHUFB     Y5, Y3, Y3
+	VPADDQ      Y3, Y2, Y2
+	VPXOR       Y2, Y1, Y1
+	VPADDQ      Y1, Y1, Y10
+	VPSRLQ      $0x3f, Y1, Y1
+	VPXOR       Y10, Y1, Y1
+	BYTE        $0xc4
+	BYTE        $0xe3
+	BYTE        $0xfd
+	BYTE        $0x00
+	BYTE        $0xdb
+	BYTE        $0x39
+	BYTE        $0xc4
+	BYTE        $0xe3
+	BYTE        $0xfd
+	BYTE        $0x00
+	BYTE        $0xd2
+	BYTE        $0x4e
+	BYTE        $0xc4
+	BYTE        $0xe3
+	BYTE        $0xfd
+	BYTE        $0x00
+	BYTE        $0xc9
+	BYTE        $0x93
+	BYTE        $0xc5
+	BYTE        $0x7a
+	BYTE        $0x7e
+	BYTE        $0x66
+	BYTE        $0x30
+	BYTE        $0xc5
+	BYTE        $0x7a
+	BYTE        $0x7e
+	BYTE        $0x5e
+	BYTE        $0x58
+	BYTE        $0xc4
+	BYTE        $0x63
+	BYTE        $0x99
+	BYTE        $0x22
+	BYTE        $0x66
+	BYTE        $0x70
+	BYTE        $0x01
+	BYTE        $0xc4
+	BYTE        $0x63
+	BYTE        $0xa1
+	BYTE        $0x22
+	BYTE        $0x1e
+	BYTE        $0x01
+	VINSERTI128 $0x01, X11, Y12, Y12
+	BYTE        $0xc5
+	BYTE        $0x7a
+	BYTE        $0x7e
+	BYTE        $0x6e
+	BYTE        $0x78
+	BYTE        $0xc5
+	BYTE        $0x7a
+	BYTE        $0x7e
+	BYTE        $0x5e
+	BYTE        $0x18
+	BYTE        $0xc4
+	BYTE        $0x63
+	BYTE        $0x91
+	BYTE        $0x22
+	BYTE        $0x6e
+	BYTE        $0x48
+	BYTE        $0x01
+	BYTE        $0xc4
+	BYTE        $0x63
+	BYTE        $0xa1
+	BYTE        $0x22
+	BYTE        $0x5e
+	BYTE        $0x40
+	BYTE        $0x01
+	VINSERTI128 $0x01, X11, Y13, Y13
+	BYTE        $0xc5
+	BYTE        $0x7a
+	BYTE        $0x7e
+	BYTE        $0x5e
+	BYTE        $0x08
+	VMOVDQU     96(SI), X14
+	BYTE        $0xc4
+	BYTE        $0x63
+	BYTE        $0xa1
+	BYTE        $0x22
+	BYTE        $0x5e
+	BYTE        $0x50
+	BYTE        $0x01
+	VINSERTI128 $0x01, X11, Y14, Y14
+	BYTE        $0xc5
+	BYTE        $0x7a
+	BYTE        $0x7e
+	BYTE        $0x7e
+	BYTE        $0x10
+	VMOVDQU     32(SI), X11
+	BYTE        $0xc4
+	BYTE        $0x63
+	BYTE        $0x81
+	BYTE        $0x22
+	BYTE        $0x7e
+	BYTE        $0x38
+	BYTE        $0x01
+	VINSERTI128 $0x01, X11, Y15, Y15
+	VPADDQ      Y12, Y0, Y0
+	VPADDQ      Y1, Y0, Y0
+	VPXOR       Y0, Y3, Y3
+	VPSHUFD     $-79, Y3, Y3
+	VPADDQ      Y3, Y2, Y2
+	VPXOR       Y2, Y1, Y1
+	VPSHUFB     Y4, Y1, Y1
+	VPADDQ      Y13, Y0, Y0
+	VPADDQ      Y1, Y0, Y0
+	VPXOR       Y0, Y3, Y3
+	VPSHUFB     Y5, Y3, Y3
+	VPADDQ      Y3, Y2, Y2
+	VPXOR       Y2, Y1, Y1
+	VPADDQ      Y1, Y1, Y10
+	VPSRLQ      $0x3f, Y1, Y1
+	VPXOR       Y10, Y1, Y1
+	BYTE        $0xc4
+	BYTE        $0xe3
+	BYTE        $0xfd
+	BYTE        $0x00
+	BYTE        $0xc9
+	BYTE        $0x39
+	BYTE        $0xc4
+	BYTE        $0xe3
+	BYTE        $0xfd
+	BYTE        $0x00
+	BYTE        $0xd2
+	BYTE        $0x4e
+	BYTE        $0xc4
+	BYTE        $0xe3
+	BYTE        $0xfd
+	BYTE        $0x00
+	BYTE        $0xdb
+	BYTE        $0x93
+	VPADDQ      Y14, Y0, Y0
+	VPADDQ      Y1, Y0, Y0
+	VPXOR       Y0, Y3, Y3
+	VPSHUFD     $-79, Y3, Y3
+	VPADDQ      Y3, Y2, Y2
+	VPXOR       Y2, Y1, Y1
+	VPSHUFB     Y4, Y1, Y1
+	VPADDQ      Y15, Y0, Y0
+	VPADDQ      Y1, Y0, Y0
+	VPXOR       Y0, Y3, Y3
+	VPSHUFB     Y5, Y3, Y3
+	VPADDQ      Y3, Y2, Y2
+	VPXOR       Y2, Y1, Y1
+	VPADDQ      Y1, Y1, Y10
+	VPSRLQ      $0x3f, Y1, Y1
+	VPXOR       Y10, Y1, Y1
+	BYTE        $0xc4
+	BYTE        $0xe3
+	BYTE        $0xfd
+	BYTE        $0x00
+	BYTE        $0xdb
+	BYTE        $0x39
+	BYTE        $0xc4
+	BYTE        $0xe3
+	BYTE        $0xfd
+	BYTE        $0x00
+	BYTE        $0xd2
+	BYTE        $0x4e
+	BYTE        $0xc4
+	BYTE        $0xe3
+	BYTE        $0xfd
+	BYTE        $0x00
+	BYTE        $0xc9
+	BYTE        $0x93
+	BYTE        $0xc5
+	BYTE        $0x7a
+	BYTE        $0x7e
+	BYTE        $0x66
+	BYTE        $0x50
+	BYTE        $0xc5
+	BYTE        $0x7a
+	BYTE        $0x7e
+	BYTE        $0x5e
+	BYTE        $0x38
+	BYTE        $0xc4
+	BYTE        $0x63
+	BYTE        $0x99
+	BYTE        $0x22
+	BYTE        $0x66
+	BYTE        $0x40
+	BYTE        $0x01
+	BYTE        $0xc4
+	BYTE        $0x63
+	BYTE        $0xa1
+	BYTE        $0x22
+	BYTE        $0x5e
+	BYTE        $0x08
+	BYTE        $0x01
+	VINSERTI128 $0x01, X11, Y12, Y12
+	BYTE        $0xc5
+	BYTE        $0x7a
+	BYTE        $0x7e
+	BYTE        $0x6e
+	BYTE        $0x10
+	VPSHUFD     $0x4e, 40(SI), X11
+	BYTE        $0xc4
+	BYTE        $0x63
+	BYTE        $0x91
+	BYTE        $0x22
+	BYTE        $0x6e
+	BYTE        $0x20
+	BYTE        $0x01
+	VINSERTI128 $0x01, X11, Y13, Y13
+	BYTE        $0xc5
+	BYTE        $0x7a
+	BYTE        $0x7e
+	BYTE        $0x76
+	BYTE        $0x78
+	BYTE        $0xc5
+	BYTE        $0x7a
+	BYTE        $0x7e
+	BYTE        $0x5e
+	BYTE        $0x18
+	BYTE        $0xc4
+	BYTE        $0x63
+	BYTE        $0x89
+	BYTE        $0x22
+	BYTE        $0x76
+	BYTE        $0x48
+	BYTE        $0x01
+	BYTE        $0xc4
+	BYTE        $0x63
+	BYTE        $0xa1
+	BYTE        $0x22
+	BYTE        $0x5e
+	BYTE        $0x68
+	BYTE        $0x01
+	VINSERTI128 $0x01, X11, Y14, Y14
+	BYTE        $0xc5
+	BYTE        $0x7a
+	BYTE        $0x7e
+	BYTE        $0x7e
+	BYTE        $0x58
+	BYTE        $0xc5
+	BYTE        $0x7a
+	BYTE        $0x7e
+	BYTE        $0x5e
+	BYTE        $0x60
+	BYTE        $0xc4
+	BYTE        $0x63
+	BYTE        $0x81
+	BYTE        $0x22
+	BYTE        $0x7e
+	BYTE        $0x70
+	BYTE        $0x01
+	BYTE        $0xc4
+	BYTE        $0x63
+	BYTE        $0xa1
+	BYTE        $0x22
+	BYTE        $0x1e
+	BYTE        $0x01
+	VINSERTI128 $0x01, X11, Y15, Y15
+	VPADDQ      Y12, Y0, Y0
+	VPADDQ      Y1, Y0, Y0
+	VPXOR       Y0, Y3, Y3
+	VPSHUFD     $-79, Y3, Y3
+	VPADDQ      Y3, Y2, Y2
+	VPXOR       Y2, Y1, Y1
+	VPSHUFB     Y4, Y1, Y1
+	VPADDQ      Y13, Y0, Y0
+	VPADDQ      Y1, Y0, Y0
+	VPXOR       Y0, Y3, Y3
+	VPSHUFB     Y5, Y3, Y3
+	VPADDQ      Y3, Y2, Y2
+	VPXOR       Y2, Y1, Y1
+	VPADDQ      Y1, Y1, Y10
+	VPSRLQ      $0x3f, Y1, Y1
+	VPXOR       Y10, Y1, Y1
+	BYTE        $0xc4
+	BYTE        $0xe3
+	BYTE        $0xfd
+	BYTE        $0x00
+	BYTE        $0xc9
+	BYTE        $0x39
+	BYTE        $0xc4
+	BYTE        $0xe3
+	BYTE        $0xfd
+	BYTE        $0x00
+	BYTE        $0xd2
+	BYTE        $0x4e
+	BYTE        $0xc4
+	BYTE        $0xe3
+	BYTE        $0xfd
+	BYTE        $0x00
+	BYTE        $0xdb
+	BYTE        $0x93
+	VPADDQ      Y14, Y0, Y0
+	VPADDQ      Y1, Y0, Y0
+	VPXOR       Y0, Y3, Y3
+	VPSHUFD     $-79, Y3, Y3
+	VPADDQ      Y3, Y2, Y2
+	VPXOR       Y2, Y1, Y1
+	VPSHUFB     Y4, Y1, Y1
+	VPADDQ      Y15, Y0, Y0
+	VPADDQ      Y1, Y0, Y0
+	VPXOR       Y0, Y3, Y3
+	VPSHUFB     Y5, Y3, Y3
+	VPADDQ      Y3, Y2, Y2
+	VPXOR       Y2, Y1, Y1
+	VPADDQ      Y1, Y1, Y10
+	VPSRLQ      $0x3f, Y1, Y1
+	VPXOR       Y10, Y1, Y1
+	BYTE        $0xc4
+	BYTE        $0xe3
+	BYTE        $0xfd
+	BYTE        $0x00
+	BYTE        $0xdb
+	BYTE        $0x39
+	BYTE        $0xc4
+	BYTE        $0xe3
+	BYTE        $0xfd
+	BYTE        $0x00
+	BYTE        $0xd2
+	BYTE        $0x4e
+	BYTE        $0xc4
+	BYTE        $0xe3
+	BYTE        $0xfd
+	BYTE        $0x00
+	BYTE        $0xc9
+	BYTE        $0x93
+	VPADDQ      32(DX), Y0, Y0
+	VPADDQ      Y1, Y0, Y0
+	VPXOR       Y0, Y3, Y3
+	VPSHUFD     $-79, Y3, Y3
+	VPADDQ      Y3, Y2, Y2
+	VPXOR       Y2, Y1, Y1
+	VPSHUFB     Y4, Y1, Y1
+	VPADDQ      64(DX), Y0, Y0
+	VPADDQ      Y1, Y0, Y0
+	VPXOR       Y0, Y3, Y3
+	VPSHUFB     Y5, Y3, Y3
+	VPADDQ      Y3, Y2, Y2
+	VPXOR       Y2, Y1, Y1
+	VPADDQ      Y1, Y1, Y10
+	VPSRLQ      $0x3f, Y1, Y1
+	VPXOR       Y10, Y1, Y1
+	BYTE        $0xc4
+	BYTE        $0xe3
+	BYTE        $0xfd
+	BYTE        $0x00
+	BYTE        $0xc9
+	BYTE        $0x39
+	BYTE        $0xc4
+	BYTE        $0xe3
+	BYTE        $0xfd
+	BYTE        $0x00
+	BYTE        $0xd2
+	BYTE        $0x4e
+	BYTE        $0xc4
+	BYTE        $0xe3
+	BYTE        $0xfd
+	BYTE        $0x00
+	BYTE        $0xdb
+	BYTE        $0x93
+	VPADDQ      96(DX), Y0, Y0
+	VPADDQ      Y1, Y0, Y0
+	VPXOR       Y0, Y3, Y3
+	VPSHUFD     $-79, Y3, Y3
+	VPADDQ      Y3, Y2, Y2
+	VPXOR       Y2, Y1, Y1
+	VPSHUFB     Y4, Y1, Y1
+	VPADDQ      128(DX), Y0, Y0
+	VPADDQ      Y1, Y0, Y0
+	VPXOR       Y0, Y3, Y3
+	VPSHUFB     Y5, Y3, Y3
+	VPADDQ      Y3, Y2, Y2
+	VPXOR       Y2, Y1, Y1
+	VPADDQ      Y1, Y1, Y10
+	VPSRLQ      $0x3f, Y1, Y1
+	VPXOR       Y10, Y1, Y1
+	BYTE        $0xc4
+	BYTE        $0xe3
+	BYTE        $0xfd
+	BYTE        $0x00
+	BYTE        $0xdb
+	BYTE        $0x39
+	BYTE        $0xc4
+	BYTE        $0xe3
+	BYTE        $0xfd
+	BYTE        $0x00
+	BYTE        $0xd2
+	BYTE        $0x4e
+	BYTE        $0xc4
+	BYTE        $0xe3
+	BYTE        $0xfd
+	BYTE        $0x00
+	BYTE        $0xc9
+	BYTE        $0x93
+	VPADDQ      160(DX), Y0, Y0
+	VPADDQ      Y1, Y0, Y0
+	VPXOR       Y0, Y3, Y3
+	VPSHUFD     $-79, Y3, Y3
+	VPADDQ      Y3, Y2, Y2
+	VPXOR       Y2, Y1, Y1
+	VPSHUFB     Y4, Y1, Y1
+	VPADDQ      192(DX), Y0, Y0
+	VPADDQ      Y1, Y0, Y0
+	VPXOR       Y0, Y3, Y3
+	VPSHUFB     Y5, Y3, Y3
+	VPADDQ      Y3, Y2, Y2
+	VPXOR       Y2, Y1, Y1
+	VPADDQ      Y1, Y1, Y10
+	VPSRLQ      $0x3f, Y1, Y1
+	VPXOR       Y10, Y1, Y1
+	BYTE        $0xc4
+	BYTE        $0xe3
+	BYTE        $0xfd
+	BYTE        $0x00
+	BYTE        $0xc9
+	BYTE        $0x39
+	BYTE        $0xc4
+	BYTE        $0xe3
+	BYTE        $0xfd
+	BYTE        $0x00
+	BYTE        $0xd2
+	BYTE        $0x4e
+	BYTE        $0xc4
+	BYTE        $0xe3
+	BYTE        $0xfd
+	BYTE        $0x00
+	BYTE        $0xdb
+	BYTE        $0x93
+	VPADDQ      224(DX), Y0, Y0
+	VPADDQ      Y1, Y0, Y0
+	VPXOR       Y0, Y3, Y3
+	VPSHUFD     $-79, Y3, Y3
+	VPADDQ      Y3, Y2, Y2
+	VPXOR       Y2, Y1, Y1
+	VPSHUFB     Y4, Y1, Y1
+	VPADDQ      256(DX), Y0, Y0
+	VPADDQ      Y1, Y0, Y0
+	VPXOR       Y0, Y3, Y3
+	VPSHUFB     Y5, Y3, Y3
+	VPADDQ      Y3, Y2, Y2
+	VPXOR       Y2, Y1, Y1
+	VPADDQ      Y1, Y1, Y10
+	VPSRLQ      $0x3f, Y1, Y1
+	VPXOR       Y10, Y1, Y1
+	BYTE        $0xc4
+	BYTE        $0xe3
+	BYTE        $0xfd
+	BYTE        $0x00
+	BYTE        $0xdb
+	BYTE        $0x39
+	BYTE        $0xc4
+	BYTE        $0xe3
+	BYTE        $0xfd
+	BYTE        $0x00
+	BYTE        $0xd2
+	BYTE        $0x4e
+	BYTE        $0xc4
+	BYTE        $0xe3
+	BYTE        $0xfd
+	BYTE        $0x00
+	BYTE        $0xc9
+	BYTE        $0x93
+	VPXOR       Y0, Y8, Y8
+	VPXOR       Y1, Y9, Y9
+	VPXOR       Y2, Y8, Y8
+	VPXOR       Y3, Y9, Y9
+	LEAQ        128(SI), SI
+	SUBQ        $0x80, DI
+	JNE         loop
+	MOVQ        R8, (BX)
+	MOVQ        R9, 8(BX)
+	VMOVDQU     Y8, (AX)
+	VMOVDQU     Y9, 32(AX)
 	VZEROUPPER
-
 	RET
 
-#define VPUNPCKLQDQ_X2_X2_X15 BYTE $0xC5; BYTE $0x69; BYTE $0x6C; BYTE $0xFA
-#define VPUNPCKLQDQ_X3_X3_X15 BYTE $0xC5; BYTE $0x61; BYTE $0x6C; BYTE $0xFB
-#define VPUNPCKLQDQ_X7_X7_X15 BYTE $0xC5; BYTE $0x41; BYTE $0x6C; BYTE $0xFF
-#define VPUNPCKLQDQ_X13_X13_X15 BYTE $0xC4; BYTE $0x41; BYTE $0x11; BYTE $0x6C; BYTE $0xFD
-#define VPUNPCKLQDQ_X14_X14_X15 BYTE $0xC4; BYTE $0x41; BYTE $0x09; BYTE $0x6C; BYTE $0xFE
-
-#define VPUNPCKHQDQ_X15_X2_X2 BYTE $0xC4; BYTE $0xC1; BYTE $0x69; BYTE $0x6D; BYTE $0xD7
-#define VPUNPCKHQDQ_X15_X3_X3 BYTE $0xC4; BYTE $0xC1; BYTE $0x61; BYTE $0x6D; BYTE $0xDF
-#define VPUNPCKHQDQ_X15_X6_X6 BYTE $0xC4; BYTE $0xC1; BYTE $0x49; BYTE $0x6D; BYTE $0xF7
-#define VPUNPCKHQDQ_X15_X7_X7 BYTE $0xC4; BYTE $0xC1; BYTE $0x41; BYTE $0x6D; BYTE $0xFF
-#define VPUNPCKHQDQ_X15_X3_X2 BYTE $0xC4; BYTE $0xC1; BYTE $0x61; BYTE $0x6D; BYTE $0xD7
-#define VPUNPCKHQDQ_X15_X7_X6 BYTE $0xC4; BYTE $0xC1; BYTE $0x41; BYTE $0x6D; BYTE $0xF7
-#define VPUNPCKHQDQ_X15_X13_X3 BYTE $0xC4; BYTE $0xC1; BYTE $0x11; BYTE $0x6D; BYTE $0xDF
-#define VPUNPCKHQDQ_X15_X13_X7 BYTE $0xC4; BYTE $0xC1; BYTE $0x11; BYTE $0x6D; BYTE $0xFF
-
-#define SHUFFLE_AVX() \
-	VMOVDQA X6, X13;         \
-	VMOVDQA X2, X14;         \
-	VMOVDQA X4, X6;          \
-	VPUNPCKLQDQ_X13_X13_X15; \
-	VMOVDQA X5, X4;          \
-	VMOVDQA X6, X5;          \
-	VPUNPCKHQDQ_X15_X7_X6;   \
-	VPUNPCKLQDQ_X7_X7_X15;   \
-	VPUNPCKHQDQ_X15_X13_X7;  \
-	VPUNPCKLQDQ_X3_X3_X15;   \
-	VPUNPCKHQDQ_X15_X2_X2;   \
-	VPUNPCKLQDQ_X14_X14_X15; \
-	VPUNPCKHQDQ_X15_X3_X3;   \
-
-#define SHUFFLE_AVX_INV() \
-	VMOVDQA X2, X13;         \
-	VMOVDQA X4, X14;         \
-	VPUNPCKLQDQ_X2_X2_X15;   \
-	VMOVDQA X5, X4;          \
-	VPUNPCKHQDQ_X15_X3_X2;   \
-	VMOVDQA X14, X5;         \
-	VPUNPCKLQDQ_X3_X3_X15;   \
-	VMOVDQA X6, X14;         \
-	VPUNPCKHQDQ_X15_X13_X3;  \
-	VPUNPCKLQDQ_X7_X7_X15;   \
-	VPUNPCKHQDQ_X15_X6_X6;   \
-	VPUNPCKLQDQ_X14_X14_X15; \
-	VPUNPCKHQDQ_X15_X7_X7;   \
-
-#define HALF_ROUND_AVX(v0, v1, v2, v3, v4, v5, v6, v7, m0, m1, m2, m3, t0, c40, c48) \
-	VPADDQ  m0, v0, v0;   \
-	VPADDQ  v2, v0, v0;   \
-	VPADDQ  m1, v1, v1;   \
-	VPADDQ  v3, v1, v1;   \
-	VPXOR   v0, v6, v6;   \
-	VPXOR   v1, v7, v7;   \
-	VPSHUFD $-79, v6, v6; \
-	VPSHUFD $-79, v7, v7; \
-	VPADDQ  v6, v4, v4;   \
-	VPADDQ  v7, v5, v5;   \
-	VPXOR   v4, v2, v2;   \
-	VPXOR   v5, v3, v3;   \
-	VPSHUFB c40, v2, v2;  \
-	VPSHUFB c40, v3, v3;  \
-	VPADDQ  m2, v0, v0;   \
-	VPADDQ  v2, v0, v0;   \
-	VPADDQ  m3, v1, v1;   \
-	VPADDQ  v3, v1, v1;   \
-	VPXOR   v0, v6, v6;   \
-	VPXOR   v1, v7, v7;   \
-	VPSHUFB c48, v6, v6;  \
-	VPSHUFB c48, v7, v7;  \
-	VPADDQ  v6, v4, v4;   \
-	VPADDQ  v7, v5, v5;   \
-	VPXOR   v4, v2, v2;   \
-	VPXOR   v5, v3, v3;   \
-	VPADDQ  v2, v2, t0;   \
-	VPSRLQ  $63, v2, v2;  \
-	VPXOR   t0, v2, v2;   \
-	VPADDQ  v3, v3, t0;   \
-	VPSRLQ  $63, v3, v3;  \
-	VPXOR   t0, v3, v3
-
-// load msg: X12 = (i0, i1), X13 = (i2, i3), X14 = (i4, i5), X15 = (i6, i7)
-// i0, i1, i2, i3, i4, i5, i6, i7 must not be 0
-#define LOAD_MSG_AVX(i0, i1, i2, i3, i4, i5, i6, i7) \
-	VMOVQ_SI_X12(i0*8);     \
-	VMOVQ_SI_X13(i2*8);     \
-	VMOVQ_SI_X14(i4*8);     \
-	VMOVQ_SI_X15(i6*8);     \
-	VPINSRQ_1_SI_X12(i1*8); \
-	VPINSRQ_1_SI_X13(i3*8); \
-	VPINSRQ_1_SI_X14(i5*8); \
-	VPINSRQ_1_SI_X15(i7*8)
-
-// load msg: X12 = (0, 2), X13 = (4, 6), X14 = (1, 3), X15 = (5, 7)
-#define LOAD_MSG_AVX_0_2_4_6_1_3_5_7() \
-	VMOVQ_SI_X12_0;        \
-	VMOVQ_SI_X13(4*8);     \
-	VMOVQ_SI_X14(1*8);     \
-	VMOVQ_SI_X15(5*8);     \
-	VPINSRQ_1_SI_X12(2*8); \
-	VPINSRQ_1_SI_X13(6*8); \
-	VPINSRQ_1_SI_X14(3*8); \
-	VPINSRQ_1_SI_X15(7*8)
-
-// load msg: X12 = (1, 0), X13 = (11, 5), X14 = (12, 2), X15 = (7, 3)
-#define LOAD_MSG_AVX_1_0_11_5_12_2_7_3() \
-	VPSHUFD $0x4E, 0*8(SI), X12; \
-	VMOVQ_SI_X13(11*8);          \
-	VMOVQ_SI_X14(12*8);          \
-	VMOVQ_SI_X15(7*8);           \
-	VPINSRQ_1_SI_X13(5*8);       \
-	VPINSRQ_1_SI_X14(2*8);       \
-	VPINSRQ_1_SI_X15(3*8)
-
-// load msg: X12 = (11, 12), X13 = (5, 15), X14 = (8, 0), X15 = (2, 13)
-#define LOAD_MSG_AVX_11_12_5_15_8_0_2_13() \
-	VMOVDQU 11*8(SI), X12;  \
-	VMOVQ_SI_X13(5*8);      \
-	VMOVQ_SI_X14(8*8);      \
-	VMOVQ_SI_X15(2*8);      \
-	VPINSRQ_1_SI_X13(15*8); \
-	VPINSRQ_1_SI_X14_0;     \
-	VPINSRQ_1_SI_X15(13*8)
-
-// load msg: X12 = (2, 5), X13 = (4, 15), X14 = (6, 10), X15 = (0, 8)
-#define LOAD_MSG_AVX_2_5_4_15_6_10_0_8() \
-	VMOVQ_SI_X12(2*8);      \
-	VMOVQ_SI_X13(4*8);      \
-	VMOVQ_SI_X14(6*8);      \
-	VMOVQ_SI_X15_0;         \
-	VPINSRQ_1_SI_X12(5*8);  \
-	VPINSRQ_1_SI_X13(15*8); \
-	VPINSRQ_1_SI_X14(10*8); \
-	VPINSRQ_1_SI_X15(8*8)
+DATA ·AVX2_c40<>+0(SB)/8, $0x0201000706050403
+DATA ·AVX2_c40<>+8(SB)/8, $0x0a09080f0e0d0c0b
+DATA ·AVX2_c40<>+16(SB)/8, $0x0201000706050403
+DATA ·AVX2_c40<>+24(SB)/8, $0x0a09080f0e0d0c0b
+GLOBL ·AVX2_c40<>(SB), RODATA|NOPTR, $32
 
-// load msg: X12 = (9, 5), X13 = (2, 10), X14 = (0, 7), X15 = (4, 15)
-#define LOAD_MSG_AVX_9_5_2_10_0_7_4_15() \
-	VMOVQ_SI_X12(9*8);      \
-	VMOVQ_SI_X13(2*8);      \
-	VMOVQ_SI_X14_0;         \
-	VMOVQ_SI_X15(4*8);      \
-	VPINSRQ_1_SI_X12(5*8);  \
-	VPINSRQ_1_SI_X13(10*8); \
-	VPINSRQ_1_SI_X14(7*8);  \
-	VPINSRQ_1_SI_X15(15*8)
+DATA ·AVX2_c48<>+0(SB)/8, $0x0100070605040302
+DATA ·AVX2_c48<>+8(SB)/8, $0x09080f0e0d0c0b0a
+DATA ·AVX2_c48<>+16(SB)/8, $0x0100070605040302
+DATA ·AVX2_c48<>+24(SB)/8, $0x09080f0e0d0c0b0a
+GLOBL ·AVX2_c48<>(SB), RODATA|NOPTR, $32
 
-// load msg: X12 = (2, 6), X13 = (0, 8), X14 = (12, 10), X15 = (11, 3)
-#define LOAD_MSG_AVX_2_6_0_8_12_10_11_3() \
-	VMOVQ_SI_X12(2*8);      \
-	VMOVQ_SI_X13_0;         \
-	VMOVQ_SI_X14(12*8);     \
-	VMOVQ_SI_X15(11*8);     \
-	VPINSRQ_1_SI_X12(6*8);  \
-	VPINSRQ_1_SI_X13(8*8);  \
-	VPINSRQ_1_SI_X14(10*8); \
-	VPINSRQ_1_SI_X15(3*8)
+DATA ·AVX2_iv0<>+0(SB)/8, $0x6a09e667f3bcc908
+DATA ·AVX2_iv0<>+8(SB)/8, $0xbb67ae8584caa73b
+DATA ·AVX2_iv0<>+16(SB)/8, $0x3c6ef372fe94f82b
+DATA ·AVX2_iv0<>+24(SB)/8, $0xa54ff53a5f1d36f1
+GLOBL ·AVX2_iv0<>(SB), RODATA|NOPTR, $32
 
-// load msg: X12 = (0, 6), X13 = (9, 8), X14 = (7, 3), X15 = (2, 11)
-#define LOAD_MSG_AVX_0_6_9_8_7_3_2_11() \
-	MOVQ    0*8(SI), X12;        \
-	VPSHUFD $0x4E, 8*8(SI), X13; \
-	MOVQ    7*8(SI), X14;        \
-	MOVQ    2*8(SI), X15;        \
-	VPINSRQ_1_SI_X12(6*8);       \
-	VPINSRQ_1_SI_X14(3*8);       \
-	VPINSRQ_1_SI_X15(11*8)
-
-// load msg: X12 = (6, 14), X13 = (11, 0), X14 = (15, 9), X15 = (3, 8)
-#define LOAD_MSG_AVX_6_14_11_0_15_9_3_8() \
-	MOVQ 6*8(SI), X12;      \
-	MOVQ 11*8(SI), X13;     \
-	MOVQ 15*8(SI), X14;     \
-	MOVQ 3*8(SI), X15;      \
-	VPINSRQ_1_SI_X12(14*8); \
-	VPINSRQ_1_SI_X13_0;     \
-	VPINSRQ_1_SI_X14(9*8);  \
-	VPINSRQ_1_SI_X15(8*8)
-
-// load msg: X12 = (5, 15), X13 = (8, 2), X14 = (0, 4), X15 = (6, 10)
-#define LOAD_MSG_AVX_5_15_8_2_0_4_6_10() \
-	MOVQ 5*8(SI), X12;      \
-	MOVQ 8*8(SI), X13;      \
-	MOVQ 0*8(SI), X14;      \
-	MOVQ 6*8(SI), X15;      \
-	VPINSRQ_1_SI_X12(15*8); \
-	VPINSRQ_1_SI_X13(2*8);  \
-	VPINSRQ_1_SI_X14(4*8);  \
-	VPINSRQ_1_SI_X15(10*8)
-
-// load msg: X12 = (12, 13), X13 = (1, 10), X14 = (2, 7), X15 = (4, 5)
-#define LOAD_MSG_AVX_12_13_1_10_2_7_4_5() \
-	VMOVDQU 12*8(SI), X12;  \
-	MOVQ    1*8(SI), X13;   \
-	MOVQ    2*8(SI), X14;   \
-	VPINSRQ_1_SI_X13(10*8); \
-	VPINSRQ_1_SI_X14(7*8);  \
-	VMOVDQU 4*8(SI), X15
-
-// load msg: X12 = (15, 9), X13 = (3, 13), X14 = (11, 14), X15 = (12, 0)
-#define LOAD_MSG_AVX_15_9_3_13_11_14_12_0() \
-	MOVQ 15*8(SI), X12;     \
-	MOVQ 3*8(SI), X13;      \
-	MOVQ 11*8(SI), X14;     \
-	MOVQ 12*8(SI), X15;     \
-	VPINSRQ_1_SI_X12(9*8);  \
-	VPINSRQ_1_SI_X13(13*8); \
-	VPINSRQ_1_SI_X14(14*8); \
-	VPINSRQ_1_SI_X15_0
+DATA ·AVX2_iv1<>+0(SB)/8, $0x510e527fade682d1
+DATA ·AVX2_iv1<>+8(SB)/8, $0x9b05688c2b3e6c1f
+DATA ·AVX2_iv1<>+16(SB)/8, $0x1f83d9abfb41bd6b
+DATA ·AVX2_iv1<>+24(SB)/8, $0x5be0cd19137e2179
+GLOBL ·AVX2_iv1<>(SB), RODATA|NOPTR, $32
 
 // func hashBlocksAVX(h *[8]uint64, c *[2]uint64, flag uint64, blocks []byte)
-TEXT ·hashBlocksAVX(SB), 4, $288-48 // frame size = 272 + 16 byte alignment
-	MOVQ h+0(FP), AX
-	MOVQ c+8(FP), BX
-	MOVQ flag+16(FP), CX
-	MOVQ blocks_base+24(FP), SI
-	MOVQ blocks_len+32(FP), DI
-
-	MOVQ SP, R10
-	ADDQ $15, R10
-	ANDQ $~15, R10
-
-	VMOVDQU ·AVX_c40<>(SB), X0
-	VMOVDQU ·AVX_c48<>(SB), X1
+// Requires: AVX, SSE2
+TEXT ·hashBlocksAVX(SB), NOSPLIT, $288-48
+	MOVQ    h+0(FP), AX
+	MOVQ    c+8(FP), BX
+	MOVQ    flag+16(FP), CX
+	MOVQ    blocks_base+24(FP), SI
+	MOVQ    blocks_len+32(FP), DI
+	MOVQ    SP, R10
+	ADDQ    $0x0f, R10
+	ANDQ    $-16, R10
+	VMOVDQU ·AVX_c40<>+0(SB), X0
+	VMOVDQU ·AVX_c48<>+0(SB), X1
 	VMOVDQA X0, X8
 	VMOVDQA X1, X9
-
-	VMOVDQU ·AVX_iv3<>(SB), X0
-	VMOVDQA X0, 0(R10)
-	XORQ    CX, 0(R10)          // 0(R10) = ·AVX_iv3 ^ (CX || 0)
-
-	VMOVDQU 0(AX), X10
+	VMOVDQU ·AVX_iv3<>+0(SB), X0
+	VMOVDQA X0, (R10)
+	XORQ    CX, (R10)
+	VMOVDQU (AX), X10
 	VMOVDQU 16(AX), X11
 	VMOVDQU 32(AX), X2
 	VMOVDQU 48(AX), X3
-
-	MOVQ 0(BX), R8
-	MOVQ 8(BX), R9
+	MOVQ    (BX), R8
+	MOVQ    8(BX), R9
 
 loop:
-	ADDQ $128, R8
-	CMPQ R8, $128
+	ADDQ $0x80, R8
+	CMPQ R8, $0x80
 	JGE  noinc
 	INCQ R9
 
 noinc:
-	VMOVQ_R8_X15
-	VPINSRQ_1_R9_X15
-
+	BYTE    $0xc4
+	BYTE    $0x41
+	BYTE    $0xf9
+	BYTE    $0x6e
+	BYTE    $0xf8
+	BYTE    $0xc4
+	BYTE    $0x43
+	BYTE    $0x81
+	BYTE    $0x22
+	BYTE    $0xf9
+	BYTE    $0x01
 	VMOVDQA X10, X0
 	VMOVDQA X11, X1
-	VMOVDQU ·AVX_iv0<>(SB), X4
-	VMOVDQU ·AVX_iv1<>(SB), X5
-	VMOVDQU ·AVX_iv2<>(SB), X6
-
+	VMOVDQU ·AVX_iv0<>+0(SB), X4
+	VMOVDQU ·AVX_iv1<>+0(SB), X5
+	VMOVDQU ·AVX_iv2<>+0(SB), X6
 	VPXOR   X15, X6, X6
-	VMOVDQA 0(R10), X7
-
-	LOAD_MSG_AVX_0_2_4_6_1_3_5_7()
+	VMOVDQA (R10), X7
+	BYTE    $0xc5
+	BYTE    $0x7a
+	BYTE    $0x7e
+	BYTE    $0x26
+	BYTE    $0xc5
+	BYTE    $0x7a
+	BYTE    $0x7e
+	BYTE    $0x6e
+	BYTE    $0x20
+	BYTE    $0xc5
+	BYTE    $0x7a
+	BYTE    $0x7e
+	BYTE    $0x76
+	BYTE    $0x08
+	BYTE    $0xc5
+	BYTE    $0x7a
+	BYTE    $0x7e
+	BYTE    $0x7e
+	BYTE    $0x28
+	BYTE    $0xc4
+	BYTE    $0x63
+	BYTE    $0x99
+	BYTE    $0x22
+	BYTE    $0x66
+	BYTE    $0x10
+	BYTE    $0x01
+	BYTE    $0xc4
+	BYTE    $0x63
+	BYTE    $0x91
+	BYTE    $0x22
+	BYTE    $0x6e
+	BYTE    $0x30
+	BYTE    $0x01
+	BYTE    $0xc4
+	BYTE    $0x63
+	BYTE    $0x89
+	BYTE    $0x22
+	BYTE    $0x76
+	BYTE    $0x18
+	BYTE    $0x01
+	BYTE    $0xc4
+	BYTE    $0x63
+	BYTE    $0x81
+	BYTE    $0x22
+	BYTE    $0x7e
+	BYTE    $0x38
+	BYTE    $0x01
 	VMOVDQA X12, 16(R10)
 	VMOVDQA X13, 32(R10)
 	VMOVDQA X14, 48(R10)
 	VMOVDQA X15, 64(R10)
-	HALF_ROUND_AVX(X0, X1, X2, X3, X4, X5, X6, X7, X12, X13, X14, X15, X15, X8, X9)
-	SHUFFLE_AVX()
-	LOAD_MSG_AVX(8, 10, 12, 14, 9, 11, 13, 15)
+	VPADDQ  X12, X0, X0
+	VPADDQ  X2, X0, X0
+	VPADDQ  X13, X1, X1
+	VPADDQ  X3, X1, X1
+	VPXOR   X0, X6, X6
+	VPXOR   X1, X7, X7
+	VPSHUFD $-79, X6, X6
+	VPSHUFD $-79, X7, X7
+	VPADDQ  X6, X4, X4
+	VPADDQ  X7, X5, X5
+	VPXOR   X4, X2, X2
+	VPXOR   X5, X3, X3
+	VPSHUFB X8, X2, X2
+	VPSHUFB X8, X3, X3
+	VPADDQ  X14, X0, X0
+	VPADDQ  X2, X0, X0
+	VPADDQ  X15, X1, X1
+	VPADDQ  X3, X1, X1
+	VPXOR   X0, X6, X6
+	VPXOR   X1, X7, X7
+	VPSHUFB X9, X6, X6
+	VPSHUFB X9, X7, X7
+	VPADDQ  X6, X4, X4
+	VPADDQ  X7, X5, X5
+	VPXOR   X4, X2, X2
+	VPXOR   X5, X3, X3
+	VPADDQ  X2, X2, X15
+	VPSRLQ  $0x3f, X2, X2
+	VPXOR   X15, X2, X2
+	VPADDQ  X3, X3, X15
+	VPSRLQ  $0x3f, X3, X3
+	VPXOR   X15, X3, X3
+	VMOVDQA X6, X13
+	VMOVDQA X2, X14
+	VMOVDQA X4, X6
+	BYTE    $0xc4
+	BYTE    $0x41
+	BYTE    $0x11
+	BYTE    $0x6c
+	BYTE    $0xfd
+	VMOVDQA X5, X4
+	VMOVDQA X6, X5
+	BYTE    $0xc4
+	BYTE    $0xc1
+	BYTE    $0x41
+	BYTE    $0x6d
+	BYTE    $0xf7
+	BYTE    $0xc5
+	BYTE    $0x41
+	BYTE    $0x6c
+	BYTE    $0xff
+	BYTE    $0xc4
+	BYTE    $0xc1
+	BYTE    $0x11
+	BYTE    $0x6d
+	BYTE    $0xff
+	BYTE    $0xc5
+	BYTE    $0x61
+	BYTE    $0x6c
+	BYTE    $0xfb
+	BYTE    $0xc4
+	BYTE    $0xc1
+	BYTE    $0x69
+	BYTE    $0x6d
+	BYTE    $0xd7
+	BYTE    $0xc4
+	BYTE    $0x41
+	BYTE    $0x09
+	BYTE    $0x6c
+	BYTE    $0xfe
+	BYTE    $0xc4
+	BYTE    $0xc1
+	BYTE    $0x61
+	BYTE    $0x6d
+	BYTE    $0xdf
+	BYTE    $0xc5
+	BYTE    $0x7a
+	BYTE    $0x7e
+	BYTE    $0x66
+	BYTE    $0x40
+	BYTE    $0xc5
+	BYTE    $0x7a
+	BYTE    $0x7e
+	BYTE    $0x6e
+	BYTE    $0x60
+	BYTE    $0xc5
+	BYTE    $0x7a
+	BYTE    $0x7e
+	BYTE    $0x76
+	BYTE    $0x48
+	BYTE    $0xc5
+	BYTE    $0x7a
+	BYTE    $0x7e
+	BYTE    $0x7e
+	BYTE    $0x68
+	BYTE    $0xc4
+	BYTE    $0x63
+	BYTE    $0x99
+	BYTE    $0x22
+	BYTE    $0x66
+	BYTE    $0x50
+	BYTE    $0x01
+	BYTE    $0xc4
+	BYTE    $0x63
+	BYTE    $0x91
+	BYTE    $0x22
+	BYTE    $0x6e
+	BYTE    $0x70
+	BYTE    $0x01
+	BYTE    $0xc4
+	BYTE    $0x63
+	BYTE    $0x89
+	BYTE    $0x22
+	BYTE    $0x76
+	BYTE    $0x58
+	BYTE    $0x01
+	BYTE    $0xc4
+	BYTE    $0x63
+	BYTE    $0x81
+	BYTE    $0x22
+	BYTE    $0x7e
+	BYTE    $0x78
+	BYTE    $0x01
 	VMOVDQA X12, 80(R10)
 	VMOVDQA X13, 96(R10)
 	VMOVDQA X14, 112(R10)
 	VMOVDQA X15, 128(R10)
-	HALF_ROUND_AVX(X0, X1, X2, X3, X4, X5, X6, X7, X12, X13, X14, X15, X15, X8, X9)
-	SHUFFLE_AVX_INV()
-
-	LOAD_MSG_AVX(14, 4, 9, 13, 10, 8, 15, 6)
+	VPADDQ  X12, X0, X0
+	VPADDQ  X2, X0, X0
+	VPADDQ  X13, X1, X1
+	VPADDQ  X3, X1, X1
+	VPXOR   X0, X6, X6
+	VPXOR   X1, X7, X7
+	VPSHUFD $-79, X6, X6
+	VPSHUFD $-79, X7, X7
+	VPADDQ  X6, X4, X4
+	VPADDQ  X7, X5, X5
+	VPXOR   X4, X2, X2
+	VPXOR   X5, X3, X3
+	VPSHUFB X8, X2, X2
+	VPSHUFB X8, X3, X3
+	VPADDQ  X14, X0, X0
+	VPADDQ  X2, X0, X0
+	VPADDQ  X15, X1, X1
+	VPADDQ  X3, X1, X1
+	VPXOR   X0, X6, X6
+	VPXOR   X1, X7, X7
+	VPSHUFB X9, X6, X6
+	VPSHUFB X9, X7, X7
+	VPADDQ  X6, X4, X4
+	VPADDQ  X7, X5, X5
+	VPXOR   X4, X2, X2
+	VPXOR   X5, X3, X3
+	VPADDQ  X2, X2, X15
+	VPSRLQ  $0x3f, X2, X2
+	VPXOR   X15, X2, X2
+	VPADDQ  X3, X3, X15
+	VPSRLQ  $0x3f, X3, X3
+	VPXOR   X15, X3, X3
+	VMOVDQA X2, X13
+	VMOVDQA X4, X14
+	BYTE    $0xc5
+	BYTE    $0x69
+	BYTE    $0x6c
+	BYTE    $0xfa
+	VMOVDQA X5, X4
+	BYTE    $0xc4
+	BYTE    $0xc1
+	BYTE    $0x61
+	BYTE    $0x6d
+	BYTE    $0xd7
+	VMOVDQA X14, X5
+	BYTE    $0xc5
+	BYTE    $0x61
+	BYTE    $0x6c
+	BYTE    $0xfb
+	VMOVDQA X6, X14
+	BYTE    $0xc4
+	BYTE    $0xc1
+	BYTE    $0x11
+	BYTE    $0x6d
+	BYTE    $0xdf
+	BYTE    $0xc5
+	BYTE    $0x41
+	BYTE    $0x6c
+	BYTE    $0xff
+	BYTE    $0xc4
+	BYTE    $0xc1
+	BYTE    $0x49
+	BYTE    $0x6d
+	BYTE    $0xf7
+	BYTE    $0xc4
+	BYTE    $0x41
+	BYTE    $0x09
+	BYTE    $0x6c
+	BYTE    $0xfe
+	BYTE    $0xc4
+	BYTE    $0xc1
+	BYTE    $0x41
+	BYTE    $0x6d
+	BYTE    $0xff
+	BYTE    $0xc5
+	BYTE    $0x7a
+	BYTE    $0x7e
+	BYTE    $0x66
+	BYTE    $0x70
+	BYTE    $0xc5
+	BYTE    $0x7a
+	BYTE    $0x7e
+	BYTE    $0x6e
+	BYTE    $0x48
+	BYTE    $0xc5
+	BYTE    $0x7a
+	BYTE    $0x7e
+	BYTE    $0x76
+	BYTE    $0x50
+	BYTE    $0xc5
+	BYTE    $0x7a
+	BYTE    $0x7e
+	BYTE    $0x7e
+	BYTE    $0x78
+	BYTE    $0xc4
+	BYTE    $0x63
+	BYTE    $0x99
+	BYTE    $0x22
+	BYTE    $0x66
+	BYTE    $0x20
+	BYTE    $0x01
+	BYTE    $0xc4
+	BYTE    $0x63
+	BYTE    $0x91
+	BYTE    $0x22
+	BYTE    $0x6e
+	BYTE    $0x68
+	BYTE    $0x01
+	BYTE    $0xc4
+	BYTE    $0x63
+	BYTE    $0x89
+	BYTE    $0x22
+	BYTE    $0x76
+	BYTE    $0x40
+	BYTE    $0x01
+	BYTE    $0xc4
+	BYTE    $0x63
+	BYTE    $0x81
+	BYTE    $0x22
+	BYTE    $0x7e
+	BYTE    $0x30
+	BYTE    $0x01
 	VMOVDQA X12, 144(R10)
 	VMOVDQA X13, 160(R10)
 	VMOVDQA X14, 176(R10)
 	VMOVDQA X15, 192(R10)
-	HALF_ROUND_AVX(X0, X1, X2, X3, X4, X5, X6, X7, X12, X13, X14, X15, X15, X8, X9)
-	SHUFFLE_AVX()
-	LOAD_MSG_AVX_1_0_11_5_12_2_7_3()
+	VPADDQ  X12, X0, X0
+	VPADDQ  X2, X0, X0
+	VPADDQ  X13, X1, X1
+	VPADDQ  X3, X1, X1
+	VPXOR   X0, X6, X6
+	VPXOR   X1, X7, X7
+	VPSHUFD $-79, X6, X6
+	VPSHUFD $-79, X7, X7
+	VPADDQ  X6, X4, X4
+	VPADDQ  X7, X5, X5
+	VPXOR   X4, X2, X2
+	VPXOR   X5, X3, X3
+	VPSHUFB X8, X2, X2
+	VPSHUFB X8, X3, X3
+	VPADDQ  X14, X0, X0
+	VPADDQ  X2, X0, X0
+	VPADDQ  X15, X1, X1
+	VPADDQ  X3, X1, X1
+	VPXOR   X0, X6, X6
+	VPXOR   X1, X7, X7
+	VPSHUFB X9, X6, X6
+	VPSHUFB X9, X7, X7
+	VPADDQ  X6, X4, X4
+	VPADDQ  X7, X5, X5
+	VPXOR   X4, X2, X2
+	VPXOR   X5, X3, X3
+	VPADDQ  X2, X2, X15
+	VPSRLQ  $0x3f, X2, X2
+	VPXOR   X15, X2, X2
+	VPADDQ  X3, X3, X15
+	VPSRLQ  $0x3f, X3, X3
+	VPXOR   X15, X3, X3
+	VMOVDQA X6, X13
+	VMOVDQA X2, X14
+	VMOVDQA X4, X6
+	BYTE    $0xc4
+	BYTE    $0x41
+	BYTE    $0x11
+	BYTE    $0x6c
+	BYTE    $0xfd
+	VMOVDQA X5, X4
+	VMOVDQA X6, X5
+	BYTE    $0xc4
+	BYTE    $0xc1
+	BYTE    $0x41
+	BYTE    $0x6d
+	BYTE    $0xf7
+	BYTE    $0xc5
+	BYTE    $0x41
+	BYTE    $0x6c
+	BYTE    $0xff
+	BYTE    $0xc4
+	BYTE    $0xc1
+	BYTE    $0x11
+	BYTE    $0x6d
+	BYTE    $0xff
+	BYTE    $0xc5
+	BYTE    $0x61
+	BYTE    $0x6c
+	BYTE    $0xfb
+	BYTE    $0xc4
+	BYTE    $0xc1
+	BYTE    $0x69
+	BYTE    $0x6d
+	BYTE    $0xd7
+	BYTE    $0xc4
+	BYTE    $0x41
+	BYTE    $0x09
+	BYTE    $0x6c
+	BYTE    $0xfe
+	BYTE    $0xc4
+	BYTE    $0xc1
+	BYTE    $0x61
+	BYTE    $0x6d
+	BYTE    $0xdf
+	VPSHUFD $0x4e, (SI), X12
+	BYTE    $0xc5
+	BYTE    $0x7a
+	BYTE    $0x7e
+	BYTE    $0x6e
+	BYTE    $0x58
+	BYTE    $0xc5
+	BYTE    $0x7a
+	BYTE    $0x7e
+	BYTE    $0x76
+	BYTE    $0x60
+	BYTE    $0xc5
+	BYTE    $0x7a
+	BYTE    $0x7e
+	BYTE    $0x7e
+	BYTE    $0x38
+	BYTE    $0xc4
+	BYTE    $0x63
+	BYTE    $0x91
+	BYTE    $0x22
+	BYTE    $0x6e
+	BYTE    $0x28
+	BYTE    $0x01
+	BYTE    $0xc4
+	BYTE    $0x63
+	BYTE    $0x89
+	BYTE    $0x22
+	BYTE    $0x76
+	BYTE    $0x10
+	BYTE    $0x01
+	BYTE    $0xc4
+	BYTE    $0x63
+	BYTE    $0x81
+	BYTE    $0x22
+	BYTE    $0x7e
+	BYTE    $0x18
+	BYTE    $0x01
 	VMOVDQA X12, 208(R10)
 	VMOVDQA X13, 224(R10)
 	VMOVDQA X14, 240(R10)
 	VMOVDQA X15, 256(R10)
-	HALF_ROUND_AVX(X0, X1, X2, X3, X4, X5, X6, X7, X12, X13, X14, X15, X15, X8, X9)
-	SHUFFLE_AVX_INV()
-
-	LOAD_MSG_AVX_11_12_5_15_8_0_2_13()
-	HALF_ROUND_AVX(X0, X1, X2, X3, X4, X5, X6, X7, X12, X13, X14, X15, X15, X8, X9)
-	SHUFFLE_AVX()
-	LOAD_MSG_AVX(10, 3, 7, 9, 14, 6, 1, 4)
-	HALF_ROUND_AVX(X0, X1, X2, X3, X4, X5, X6, X7, X12, X13, X14, X15, X15, X8, X9)
-	SHUFFLE_AVX_INV()
-
-	LOAD_MSG_AVX(7, 3, 13, 11, 9, 1, 12, 14)
-	HALF_ROUND_AVX(X0, X1, X2, X3, X4, X5, X6, X7, X12, X13, X14, X15, X15, X8, X9)
-	SHUFFLE_AVX()
-	LOAD_MSG_AVX_2_5_4_15_6_10_0_8()
-	HALF_ROUND_AVX(X0, X1, X2, X3, X4, X5, X6, X7, X12, X13, X14, X15, X15, X8, X9)
-	SHUFFLE_AVX_INV()
-
-	LOAD_MSG_AVX_9_5_2_10_0_7_4_15()
-	HALF_ROUND_AVX(X0, X1, X2, X3, X4, X5, X6, X7, X12, X13, X14, X15, X15, X8, X9)
-	SHUFFLE_AVX()
-	LOAD_MSG_AVX(14, 11, 6, 3, 1, 12, 8, 13)
-	HALF_ROUND_AVX(X0, X1, X2, X3, X4, X5, X6, X7, X12, X13, X14, X15, X15, X8, X9)
-	SHUFFLE_AVX_INV()
-
-	LOAD_MSG_AVX_2_6_0_8_12_10_11_3()
-	HALF_ROUND_AVX(X0, X1, X2, X3, X4, X5, X6, X7, X12, X13, X14, X15, X15, X8, X9)
-	SHUFFLE_AVX()
-	LOAD_MSG_AVX(4, 7, 15, 1, 13, 5, 14, 9)
-	HALF_ROUND_AVX(X0, X1, X2, X3, X4, X5, X6, X7, X12, X13, X14, X15, X15, X8, X9)
-	SHUFFLE_AVX_INV()
-
-	LOAD_MSG_AVX(12, 1, 14, 4, 5, 15, 13, 10)
-	HALF_ROUND_AVX(X0, X1, X2, X3, X4, X5, X6, X7, X12, X13, X14, X15, X15, X8, X9)
-	SHUFFLE_AVX()
-	LOAD_MSG_AVX_0_6_9_8_7_3_2_11()
-	HALF_ROUND_AVX(X0, X1, X2, X3, X4, X5, X6, X7, X12, X13, X14, X15, X15, X8, X9)
-	SHUFFLE_AVX_INV()
-
-	LOAD_MSG_AVX(13, 7, 12, 3, 11, 14, 1, 9)
-	HALF_ROUND_AVX(X0, X1, X2, X3, X4, X5, X6, X7, X12, X13, X14, X15, X15, X8, X9)
-	SHUFFLE_AVX()
-	LOAD_MSG_AVX_5_15_8_2_0_4_6_10()
-	HALF_ROUND_AVX(X0, X1, X2, X3, X4, X5, X6, X7, X12, X13, X14, X15, X15, X8, X9)
-	SHUFFLE_AVX_INV()
-
-	LOAD_MSG_AVX_6_14_11_0_15_9_3_8()
-	HALF_ROUND_AVX(X0, X1, X2, X3, X4, X5, X6, X7, X12, X13, X14, X15, X15, X8, X9)
-	SHUFFLE_AVX()
-	LOAD_MSG_AVX_12_13_1_10_2_7_4_5()
-	HALF_ROUND_AVX(X0, X1, X2, X3, X4, X5, X6, X7, X12, X13, X14, X15, X15, X8, X9)
-	SHUFFLE_AVX_INV()
-
-	LOAD_MSG_AVX(10, 8, 7, 1, 2, 4, 6, 5)
-	HALF_ROUND_AVX(X0, X1, X2, X3, X4, X5, X6, X7, X12, X13, X14, X15, X15, X8, X9)
-	SHUFFLE_AVX()
-	LOAD_MSG_AVX_15_9_3_13_11_14_12_0()
-	HALF_ROUND_AVX(X0, X1, X2, X3, X4, X5, X6, X7, X12, X13, X14, X15, X15, X8, X9)
-	SHUFFLE_AVX_INV()
-
-	HALF_ROUND_AVX(X0, X1, X2, X3, X4, X5, X6, X7, 16(R10), 32(R10), 48(R10), 64(R10), X15, X8, X9)
-	SHUFFLE_AVX()
-	HALF_ROUND_AVX(X0, X1, X2, X3, X4, X5, X6, X7, 80(R10), 96(R10), 112(R10), 128(R10), X15, X8, X9)
-	SHUFFLE_AVX_INV()
-
-	HALF_ROUND_AVX(X0, X1, X2, X3, X4, X5, X6, X7, 144(R10), 160(R10), 176(R10), 192(R10), X15, X8, X9)
-	SHUFFLE_AVX()
-	HALF_ROUND_AVX(X0, X1, X2, X3, X4, X5, X6, X7, 208(R10), 224(R10), 240(R10), 256(R10), X15, X8, X9)
-	SHUFFLE_AVX_INV()
-
+	VPADDQ  X12, X0, X0
+	VPADDQ  X2, X0, X0
+	VPADDQ  X13, X1, X1
+	VPADDQ  X3, X1, X1
+	VPXOR   X0, X6, X6
+	VPXOR   X1, X7, X7
+	VPSHUFD $-79, X6, X6
+	VPSHUFD $-79, X7, X7
+	VPADDQ  X6, X4, X4
+	VPADDQ  X7, X5, X5
+	VPXOR   X4, X2, X2
+	VPXOR   X5, X3, X3
+	VPSHUFB X8, X2, X2
+	VPSHUFB X8, X3, X3
+	VPADDQ  X14, X0, X0
+	VPADDQ  X2, X0, X0
+	VPADDQ  X15, X1, X1
+	VPADDQ  X3, X1, X1
+	VPXOR   X0, X6, X6
+	VPXOR   X1, X7, X7
+	VPSHUFB X9, X6, X6
+	VPSHUFB X9, X7, X7
+	VPADDQ  X6, X4, X4
+	VPADDQ  X7, X5, X5
+	VPXOR   X4, X2, X2
+	VPXOR   X5, X3, X3
+	VPADDQ  X2, X2, X15
+	VPSRLQ  $0x3f, X2, X2
+	VPXOR   X15, X2, X2
+	VPADDQ  X3, X3, X15
+	VPSRLQ  $0x3f, X3, X3
+	VPXOR   X15, X3, X3
+	VMOVDQA X2, X13
+	VMOVDQA X4, X14
+	BYTE    $0xc5
+	BYTE    $0x69
+	BYTE    $0x6c
+	BYTE    $0xfa
+	VMOVDQA X5, X4
+	BYTE    $0xc4
+	BYTE    $0xc1
+	BYTE    $0x61
+	BYTE    $0x6d
+	BYTE    $0xd7
+	VMOVDQA X14, X5
+	BYTE    $0xc5
+	BYTE    $0x61
+	BYTE    $0x6c
+	BYTE    $0xfb
+	VMOVDQA X6, X14
+	BYTE    $0xc4
+	BYTE    $0xc1
+	BYTE    $0x11
+	BYTE    $0x6d
+	BYTE    $0xdf
+	BYTE    $0xc5
+	BYTE    $0x41
+	BYTE    $0x6c
+	BYTE    $0xff
+	BYTE    $0xc4
+	BYTE    $0xc1
+	BYTE    $0x49
+	BYTE    $0x6d
+	BYTE    $0xf7
+	BYTE    $0xc4
+	BYTE    $0x41
+	BYTE    $0x09
+	BYTE    $0x6c
+	BYTE    $0xfe
+	BYTE    $0xc4
+	BYTE    $0xc1
+	BYTE    $0x41
+	BYTE    $0x6d
+	BYTE    $0xff
+	VMOVDQU 88(SI), X12
+	BYTE    $0xc5
+	BYTE    $0x7a
+	BYTE    $0x7e
+	BYTE    $0x6e
+	BYTE    $0x28
+	BYTE    $0xc5
+	BYTE    $0x7a
+	BYTE    $0x7e
+	BYTE    $0x76
+	BYTE    $0x40
+	BYTE    $0xc5
+	BYTE    $0x7a
+	BYTE    $0x7e
+	BYTE    $0x7e
+	BYTE    $0x10
+	BYTE    $0xc4
+	BYTE    $0x63
+	BYTE    $0x91
+	BYTE    $0x22
+	BYTE    $0x6e
+	BYTE    $0x78
+	BYTE    $0x01
+	BYTE    $0xc4
+	BYTE    $0x63
+	BYTE    $0x89
+	BYTE    $0x22
+	BYTE    $0x36
+	BYTE    $0x01
+	BYTE    $0xc4
+	BYTE    $0x63
+	BYTE    $0x81
+	BYTE    $0x22
+	BYTE    $0x7e
+	BYTE    $0x68
+	BYTE    $0x01
+	VPADDQ  X12, X0, X0
+	VPADDQ  X2, X0, X0
+	VPADDQ  X13, X1, X1
+	VPADDQ  X3, X1, X1
+	VPXOR   X0, X6, X6
+	VPXOR   X1, X7, X7
+	VPSHUFD $-79, X6, X6
+	VPSHUFD $-79, X7, X7
+	VPADDQ  X6, X4, X4
+	VPADDQ  X7, X5, X5
+	VPXOR   X4, X2, X2
+	VPXOR   X5, X3, X3
+	VPSHUFB X8, X2, X2
+	VPSHUFB X8, X3, X3
+	VPADDQ  X14, X0, X0
+	VPADDQ  X2, X0, X0
+	VPADDQ  X15, X1, X1
+	VPADDQ  X3, X1, X1
+	VPXOR   X0, X6, X6
+	VPXOR   X1, X7, X7
+	VPSHUFB X9, X6, X6
+	VPSHUFB X9, X7, X7
+	VPADDQ  X6, X4, X4
+	VPADDQ  X7, X5, X5
+	VPXOR   X4, X2, X2
+	VPXOR   X5, X3, X3
+	VPADDQ  X2, X2, X15
+	VPSRLQ  $0x3f, X2, X2
+	VPXOR   X15, X2, X2
+	VPADDQ  X3, X3, X15
+	VPSRLQ  $0x3f, X3, X3
+	VPXOR   X15, X3, X3
+	VMOVDQA X6, X13
+	VMOVDQA X2, X14
+	VMOVDQA X4, X6
+	BYTE    $0xc4
+	BYTE    $0x41
+	BYTE    $0x11
+	BYTE    $0x6c
+	BYTE    $0xfd
+	VMOVDQA X5, X4
+	VMOVDQA X6, X5
+	BYTE    $0xc4
+	BYTE    $0xc1
+	BYTE    $0x41
+	BYTE    $0x6d
+	BYTE    $0xf7
+	BYTE    $0xc5
+	BYTE    $0x41
+	BYTE    $0x6c
+	BYTE    $0xff
+	BYTE    $0xc4
+	BYTE    $0xc1
+	BYTE    $0x11
+	BYTE    $0x6d
+	BYTE    $0xff
+	BYTE    $0xc5
+	BYTE    $0x61
+	BYTE    $0x6c
+	BYTE    $0xfb
+	BYTE    $0xc4
+	BYTE    $0xc1
+	BYTE    $0x69
+	BYTE    $0x6d
+	BYTE    $0xd7
+	BYTE    $0xc4
+	BYTE    $0x41
+	BYTE    $0x09
+	BYTE    $0x6c
+	BYTE    $0xfe
+	BYTE    $0xc4
+	BYTE    $0xc1
+	BYTE    $0x61
+	BYTE    $0x6d
+	BYTE    $0xdf
+	BYTE    $0xc5
+	BYTE    $0x7a
+	BYTE    $0x7e
+	BYTE    $0x66
+	BYTE    $0x50
+	BYTE    $0xc5
+	BYTE    $0x7a
+	BYTE    $0x7e
+	BYTE    $0x6e
+	BYTE    $0x38
+	BYTE    $0xc5
+	BYTE    $0x7a
+	BYTE    $0x7e
+	BYTE    $0x76
+	BYTE    $0x70
+	BYTE    $0xc5
+	BYTE    $0x7a
+	BYTE    $0x7e
+	BYTE    $0x7e
+	BYTE    $0x08
+	BYTE    $0xc4
+	BYTE    $0x63
+	BYTE    $0x99
+	BYTE    $0x22
+	BYTE    $0x66
+	BYTE    $0x18
+	BYTE    $0x01
+	BYTE    $0xc4
+	BYTE    $0x63
+	BYTE    $0x91
+	BYTE    $0x22
+	BYTE    $0x6e
+	BYTE    $0x48
+	BYTE    $0x01
+	BYTE    $0xc4
+	BYTE    $0x63
+	BYTE    $0x89
+	BYTE    $0x22
+	BYTE    $0x76
+	BYTE    $0x30
+	BYTE    $0x01
+	BYTE    $0xc4
+	BYTE    $0x63
+	BYTE    $0x81
+	BYTE    $0x22
+	BYTE    $0x7e
+	BYTE    $0x20
+	BYTE    $0x01
+	VPADDQ  X12, X0, X0
+	VPADDQ  X2, X0, X0
+	VPADDQ  X13, X1, X1
+	VPADDQ  X3, X1, X1
+	VPXOR   X0, X6, X6
+	VPXOR   X1, X7, X7
+	VPSHUFD $-79, X6, X6
+	VPSHUFD $-79, X7, X7
+	VPADDQ  X6, X4, X4
+	VPADDQ  X7, X5, X5
+	VPXOR   X4, X2, X2
+	VPXOR   X5, X3, X3
+	VPSHUFB X8, X2, X2
+	VPSHUFB X8, X3, X3
+	VPADDQ  X14, X0, X0
+	VPADDQ  X2, X0, X0
+	VPADDQ  X15, X1, X1
+	VPADDQ  X3, X1, X1
+	VPXOR   X0, X6, X6
+	VPXOR   X1, X7, X7
+	VPSHUFB X9, X6, X6
+	VPSHUFB X9, X7, X7
+	VPADDQ  X6, X4, X4
+	VPADDQ  X7, X5, X5
+	VPXOR   X4, X2, X2
+	VPXOR   X5, X3, X3
+	VPADDQ  X2, X2, X15
+	VPSRLQ  $0x3f, X2, X2
+	VPXOR   X15, X2, X2
+	VPADDQ  X3, X3, X15
+	VPSRLQ  $0x3f, X3, X3
+	VPXOR   X15, X3, X3
+	VMOVDQA X2, X13
+	VMOVDQA X4, X14
+	BYTE    $0xc5
+	BYTE    $0x69
+	BYTE    $0x6c
+	BYTE    $0xfa
+	VMOVDQA X5, X4
+	BYTE    $0xc4
+	BYTE    $0xc1
+	BYTE    $0x61
+	BYTE    $0x6d
+	BYTE    $0xd7
+	VMOVDQA X14, X5
+	BYTE    $0xc5
+	BYTE    $0x61
+	BYTE    $0x6c
+	BYTE    $0xfb
+	VMOVDQA X6, X14
+	BYTE    $0xc4
+	BYTE    $0xc1
+	BYTE    $0x11
+	BYTE    $0x6d
+	BYTE    $0xdf
+	BYTE    $0xc5
+	BYTE    $0x41
+	BYTE    $0x6c
+	BYTE    $0xff
+	BYTE    $0xc4
+	BYTE    $0xc1
+	BYTE    $0x49
+	BYTE    $0x6d
+	BYTE    $0xf7
+	BYTE    $0xc4
+	BYTE    $0x41
+	BYTE    $0x09
+	BYTE    $0x6c
+	BYTE    $0xfe
+	BYTE    $0xc4
+	BYTE    $0xc1
+	BYTE    $0x41
+	BYTE    $0x6d
+	BYTE    $0xff
+	BYTE    $0xc5
+	BYTE    $0x7a
+	BYTE    $0x7e
+	BYTE    $0x66
+	BYTE    $0x38
+	BYTE    $0xc5
+	BYTE    $0x7a
+	BYTE    $0x7e
+	BYTE    $0x6e
+	BYTE    $0x68
+	BYTE    $0xc5
+	BYTE    $0x7a
+	BYTE    $0x7e
+	BYTE    $0x76
+	BYTE    $0x48
+	BYTE    $0xc5
+	BYTE    $0x7a
+	BYTE    $0x7e
+	BYTE    $0x7e
+	BYTE    $0x60
+	BYTE    $0xc4
+	BYTE    $0x63
+	BYTE    $0x99
+	BYTE    $0x22
+	BYTE    $0x66
+	BYTE    $0x18
+	BYTE    $0x01
+	BYTE    $0xc4
+	BYTE    $0x63
+	BYTE    $0x91
+	BYTE    $0x22
+	BYTE    $0x6e
+	BYTE    $0x58
+	BYTE    $0x01
+	BYTE    $0xc4
+	BYTE    $0x63
+	BYTE    $0x89
+	BYTE    $0x22
+	BYTE    $0x76
+	BYTE    $0x08
+	BYTE    $0x01
+	BYTE    $0xc4
+	BYTE    $0x63
+	BYTE    $0x81
+	BYTE    $0x22
+	BYTE    $0x7e
+	BYTE    $0x70
+	BYTE    $0x01
+	VPADDQ  X12, X0, X0
+	VPADDQ  X2, X0, X0
+	VPADDQ  X13, X1, X1
+	VPADDQ  X3, X1, X1
+	VPXOR   X0, X6, X6
+	VPXOR   X1, X7, X7
+	VPSHUFD $-79, X6, X6
+	VPSHUFD $-79, X7, X7
+	VPADDQ  X6, X4, X4
+	VPADDQ  X7, X5, X5
+	VPXOR   X4, X2, X2
+	VPXOR   X5, X3, X3
+	VPSHUFB X8, X2, X2
+	VPSHUFB X8, X3, X3
+	VPADDQ  X14, X0, X0
+	VPADDQ  X2, X0, X0
+	VPADDQ  X15, X1, X1
+	VPADDQ  X3, X1, X1
+	VPXOR   X0, X6, X6
+	VPXOR   X1, X7, X7
+	VPSHUFB X9, X6, X6
+	VPSHUFB X9, X7, X7
+	VPADDQ  X6, X4, X4
+	VPADDQ  X7, X5, X5
+	VPXOR   X4, X2, X2
+	VPXOR   X5, X3, X3
+	VPADDQ  X2, X2, X15
+	VPSRLQ  $0x3f, X2, X2
+	VPXOR   X15, X2, X2
+	VPADDQ  X3, X3, X15
+	VPSRLQ  $0x3f, X3, X3
+	VPXOR   X15, X3, X3
+	VMOVDQA X6, X13
+	VMOVDQA X2, X14
+	VMOVDQA X4, X6
+	BYTE    $0xc4
+	BYTE    $0x41
+	BYTE    $0x11
+	BYTE    $0x6c
+	BYTE    $0xfd
+	VMOVDQA X5, X4
+	VMOVDQA X6, X5
+	BYTE    $0xc4
+	BYTE    $0xc1
+	BYTE    $0x41
+	BYTE    $0x6d
+	BYTE    $0xf7
+	BYTE    $0xc5
+	BYTE    $0x41
+	BYTE    $0x6c
+	BYTE    $0xff
+	BYTE    $0xc4
+	BYTE    $0xc1
+	BYTE    $0x11
+	BYTE    $0x6d
+	BYTE    $0xff
+	BYTE    $0xc5
+	BYTE    $0x61
+	BYTE    $0x6c
+	BYTE    $0xfb
+	BYTE    $0xc4
+	BYTE    $0xc1
+	BYTE    $0x69
+	BYTE    $0x6d
+	BYTE    $0xd7
+	BYTE    $0xc4
+	BYTE    $0x41
+	BYTE    $0x09
+	BYTE    $0x6c
+	BYTE    $0xfe
+	BYTE    $0xc4
+	BYTE    $0xc1
+	BYTE    $0x61
+	BYTE    $0x6d
+	BYTE    $0xdf
+	BYTE    $0xc5
+	BYTE    $0x7a
+	BYTE    $0x7e
+	BYTE    $0x66
+	BYTE    $0x10
+	BYTE    $0xc5
+	BYTE    $0x7a
+	BYTE    $0x7e
+	BYTE    $0x6e
+	BYTE    $0x20
+	BYTE    $0xc5
+	BYTE    $0x7a
+	BYTE    $0x7e
+	BYTE    $0x76
+	BYTE    $0x30
+	BYTE    $0xc5
+	BYTE    $0x7a
+	BYTE    $0x7e
+	BYTE    $0x3e
+	BYTE    $0xc4
+	BYTE    $0x63
+	BYTE    $0x99
+	BYTE    $0x22
+	BYTE    $0x66
+	BYTE    $0x28
+	BYTE    $0x01
+	BYTE    $0xc4
+	BYTE    $0x63
+	BYTE    $0x91
+	BYTE    $0x22
+	BYTE    $0x6e
+	BYTE    $0x78
+	BYTE    $0x01
+	BYTE    $0xc4
+	BYTE    $0x63
+	BYTE    $0x89
+	BYTE    $0x22
+	BYTE    $0x76
+	BYTE    $0x50
+	BYTE    $0x01
+	BYTE    $0xc4
+	BYTE    $0x63
+	BYTE    $0x81
+	BYTE    $0x22
+	BYTE    $0x7e
+	BYTE    $0x40
+	BYTE    $0x01
+	VPADDQ  X12, X0, X0
+	VPADDQ  X2, X0, X0
+	VPADDQ  X13, X1, X1
+	VPADDQ  X3, X1, X1
+	VPXOR   X0, X6, X6
+	VPXOR   X1, X7, X7
+	VPSHUFD $-79, X6, X6
+	VPSHUFD $-79, X7, X7
+	VPADDQ  X6, X4, X4
+	VPADDQ  X7, X5, X5
+	VPXOR   X4, X2, X2
+	VPXOR   X5, X3, X3
+	VPSHUFB X8, X2, X2
+	VPSHUFB X8, X3, X3
+	VPADDQ  X14, X0, X0
+	VPADDQ  X2, X0, X0
+	VPADDQ  X15, X1, X1
+	VPADDQ  X3, X1, X1
+	VPXOR   X0, X6, X6
+	VPXOR   X1, X7, X7
+	VPSHUFB X9, X6, X6
+	VPSHUFB X9, X7, X7
+	VPADDQ  X6, X4, X4
+	VPADDQ  X7, X5, X5
+	VPXOR   X4, X2, X2
+	VPXOR   X5, X3, X3
+	VPADDQ  X2, X2, X15
+	VPSRLQ  $0x3f, X2, X2
+	VPXOR   X15, X2, X2
+	VPADDQ  X3, X3, X15
+	VPSRLQ  $0x3f, X3, X3
+	VPXOR   X15, X3, X3
+	VMOVDQA X2, X13
+	VMOVDQA X4, X14
+	BYTE    $0xc5
+	BYTE    $0x69
+	BYTE    $0x6c
+	BYTE    $0xfa
+	VMOVDQA X5, X4
+	BYTE    $0xc4
+	BYTE    $0xc1
+	BYTE    $0x61
+	BYTE    $0x6d
+	BYTE    $0xd7
+	VMOVDQA X14, X5
+	BYTE    $0xc5
+	BYTE    $0x61
+	BYTE    $0x6c
+	BYTE    $0xfb
+	VMOVDQA X6, X14
+	BYTE    $0xc4
+	BYTE    $0xc1
+	BYTE    $0x11
+	BYTE    $0x6d
+	BYTE    $0xdf
+	BYTE    $0xc5
+	BYTE    $0x41
+	BYTE    $0x6c
+	BYTE    $0xff
+	BYTE    $0xc4
+	BYTE    $0xc1
+	BYTE    $0x49
+	BYTE    $0x6d
+	BYTE    $0xf7
+	BYTE    $0xc4
+	BYTE    $0x41
+	BYTE    $0x09
+	BYTE    $0x6c
+	BYTE    $0xfe
+	BYTE    $0xc4
+	BYTE    $0xc1
+	BYTE    $0x41
+	BYTE    $0x6d
+	BYTE    $0xff
+	BYTE    $0xc5
+	BYTE    $0x7a
+	BYTE    $0x7e
+	BYTE    $0x66
+	BYTE    $0x48
+	BYTE    $0xc5
+	BYTE    $0x7a
+	BYTE    $0x7e
+	BYTE    $0x6e
+	BYTE    $0x10
+	BYTE    $0xc5
+	BYTE    $0x7a
+	BYTE    $0x7e
+	BYTE    $0x36
+	BYTE    $0xc5
+	BYTE    $0x7a
+	BYTE    $0x7e
+	BYTE    $0x7e
+	BYTE    $0x20
+	BYTE    $0xc4
+	BYTE    $0x63
+	BYTE    $0x99
+	BYTE    $0x22
+	BYTE    $0x66
+	BYTE    $0x28
+	BYTE    $0x01
+	BYTE    $0xc4
+	BYTE    $0x63
+	BYTE    $0x91
+	BYTE    $0x22
+	BYTE    $0x6e
+	BYTE    $0x50
+	BYTE    $0x01
+	BYTE    $0xc4
+	BYTE    $0x63
+	BYTE    $0x89
+	BYTE    $0x22
+	BYTE    $0x76
+	BYTE    $0x38
+	BYTE    $0x01
+	BYTE    $0xc4
+	BYTE    $0x63
+	BYTE    $0x81
+	BYTE    $0x22
+	BYTE    $0x7e
+	BYTE    $0x78
+	BYTE    $0x01
+	VPADDQ  X12, X0, X0
+	VPADDQ  X2, X0, X0
+	VPADDQ  X13, X1, X1
+	VPADDQ  X3, X1, X1
+	VPXOR   X0, X6, X6
+	VPXOR   X1, X7, X7
+	VPSHUFD $-79, X6, X6
+	VPSHUFD $-79, X7, X7
+	VPADDQ  X6, X4, X4
+	VPADDQ  X7, X5, X5
+	VPXOR   X4, X2, X2
+	VPXOR   X5, X3, X3
+	VPSHUFB X8, X2, X2
+	VPSHUFB X8, X3, X3
+	VPADDQ  X14, X0, X0
+	VPADDQ  X2, X0, X0
+	VPADDQ  X15, X1, X1
+	VPADDQ  X3, X1, X1
+	VPXOR   X0, X6, X6
+	VPXOR   X1, X7, X7
+	VPSHUFB X9, X6, X6
+	VPSHUFB X9, X7, X7
+	VPADDQ  X6, X4, X4
+	VPADDQ  X7, X5, X5
+	VPXOR   X4, X2, X2
+	VPXOR   X5, X3, X3
+	VPADDQ  X2, X2, X15
+	VPSRLQ  $0x3f, X2, X2
+	VPXOR   X15, X2, X2
+	VPADDQ  X3, X3, X15
+	VPSRLQ  $0x3f, X3, X3
+	VPXOR   X15, X3, X3
+	VMOVDQA X6, X13
+	VMOVDQA X2, X14
+	VMOVDQA X4, X6
+	BYTE    $0xc4
+	BYTE    $0x41
+	BYTE    $0x11
+	BYTE    $0x6c
+	BYTE    $0xfd
+	VMOVDQA X5, X4
+	VMOVDQA X6, X5
+	BYTE    $0xc4
+	BYTE    $0xc1
+	BYTE    $0x41
+	BYTE    $0x6d
+	BYTE    $0xf7
+	BYTE    $0xc5
+	BYTE    $0x41
+	BYTE    $0x6c
+	BYTE    $0xff
+	BYTE    $0xc4
+	BYTE    $0xc1
+	BYTE    $0x11
+	BYTE    $0x6d
+	BYTE    $0xff
+	BYTE    $0xc5
+	BYTE    $0x61
+	BYTE    $0x6c
+	BYTE    $0xfb
+	BYTE    $0xc4
+	BYTE    $0xc1
+	BYTE    $0x69
+	BYTE    $0x6d
+	BYTE    $0xd7
+	BYTE    $0xc4
+	BYTE    $0x41
+	BYTE    $0x09
+	BYTE    $0x6c
+	BYTE    $0xfe
+	BYTE    $0xc4
+	BYTE    $0xc1
+	BYTE    $0x61
+	BYTE    $0x6d
+	BYTE    $0xdf
+	BYTE    $0xc5
+	BYTE    $0x7a
+	BYTE    $0x7e
+	BYTE    $0x66
+	BYTE    $0x70
+	BYTE    $0xc5
+	BYTE    $0x7a
+	BYTE    $0x7e
+	BYTE    $0x6e
+	BYTE    $0x30
+	BYTE    $0xc5
+	BYTE    $0x7a
+	BYTE    $0x7e
+	BYTE    $0x76
+	BYTE    $0x08
+	BYTE    $0xc5
+	BYTE    $0x7a
+	BYTE    $0x7e
+	BYTE    $0x7e
+	BYTE    $0x40
+	BYTE    $0xc4
+	BYTE    $0x63
+	BYTE    $0x99
+	BYTE    $0x22
+	BYTE    $0x66
+	BYTE    $0x58
+	BYTE    $0x01
+	BYTE    $0xc4
+	BYTE    $0x63
+	BYTE    $0x91
+	BYTE    $0x22
+	BYTE    $0x6e
+	BYTE    $0x18
+	BYTE    $0x01
+	BYTE    $0xc4
+	BYTE    $0x63
+	BYTE    $0x89
+	BYTE    $0x22
+	BYTE    $0x76
+	BYTE    $0x60
+	BYTE    $0x01
+	BYTE    $0xc4
+	BYTE    $0x63
+	BYTE    $0x81
+	BYTE    $0x22
+	BYTE    $0x7e
+	BYTE    $0x68
+	BYTE    $0x01
+	VPADDQ  X12, X0, X0
+	VPADDQ  X2, X0, X0
+	VPADDQ  X13, X1, X1
+	VPADDQ  X3, X1, X1
+	VPXOR   X0, X6, X6
+	VPXOR   X1, X7, X7
+	VPSHUFD $-79, X6, X6
+	VPSHUFD $-79, X7, X7
+	VPADDQ  X6, X4, X4
+	VPADDQ  X7, X5, X5
+	VPXOR   X4, X2, X2
+	VPXOR   X5, X3, X3
+	VPSHUFB X8, X2, X2
+	VPSHUFB X8, X3, X3
+	VPADDQ  X14, X0, X0
+	VPADDQ  X2, X0, X0
+	VPADDQ  X15, X1, X1
+	VPADDQ  X3, X1, X1
+	VPXOR   X0, X6, X6
+	VPXOR   X1, X7, X7
+	VPSHUFB X9, X6, X6
+	VPSHUFB X9, X7, X7
+	VPADDQ  X6, X4, X4
+	VPADDQ  X7, X5, X5
+	VPXOR   X4, X2, X2
+	VPXOR   X5, X3, X3
+	VPADDQ  X2, X2, X15
+	VPSRLQ  $0x3f, X2, X2
+	VPXOR   X15, X2, X2
+	VPADDQ  X3, X3, X15
+	VPSRLQ  $0x3f, X3, X3
+	VPXOR   X15, X3, X3
+	VMOVDQA X2, X13
+	VMOVDQA X4, X14
+	BYTE    $0xc5
+	BYTE    $0x69
+	BYTE    $0x6c
+	BYTE    $0xfa
+	VMOVDQA X5, X4
+	BYTE    $0xc4
+	BYTE    $0xc1
+	BYTE    $0x61
+	BYTE    $0x6d
+	BYTE    $0xd7
+	VMOVDQA X14, X5
+	BYTE    $0xc5
+	BYTE    $0x61
+	BYTE    $0x6c
+	BYTE    $0xfb
+	VMOVDQA X6, X14
+	BYTE    $0xc4
+	BYTE    $0xc1
+	BYTE    $0x11
+	BYTE    $0x6d
+	BYTE    $0xdf
+	BYTE    $0xc5
+	BYTE    $0x41
+	BYTE    $0x6c
+	BYTE    $0xff
+	BYTE    $0xc4
+	BYTE    $0xc1
+	BYTE    $0x49
+	BYTE    $0x6d
+	BYTE    $0xf7
+	BYTE    $0xc4
+	BYTE    $0x41
+	BYTE    $0x09
+	BYTE    $0x6c
+	BYTE    $0xfe
+	BYTE    $0xc4
+	BYTE    $0xc1
+	BYTE    $0x41
+	BYTE    $0x6d
+	BYTE    $0xff
+	BYTE    $0xc5
+	BYTE    $0x7a
+	BYTE    $0x7e
+	BYTE    $0x66
+	BYTE    $0x10
+	BYTE    $0xc5
+	BYTE    $0x7a
+	BYTE    $0x7e
+	BYTE    $0x2e
+	BYTE    $0xc5
+	BYTE    $0x7a
+	BYTE    $0x7e
+	BYTE    $0x76
+	BYTE    $0x60
+	BYTE    $0xc5
+	BYTE    $0x7a
+	BYTE    $0x7e
+	BYTE    $0x7e
+	BYTE    $0x58
+	BYTE    $0xc4
+	BYTE    $0x63
+	BYTE    $0x99
+	BYTE    $0x22
+	BYTE    $0x66
+	BYTE    $0x30
+	BYTE    $0x01
+	BYTE    $0xc4
+	BYTE    $0x63
+	BYTE    $0x91
+	BYTE    $0x22
+	BYTE    $0x6e
+	BYTE    $0x40
+	BYTE    $0x01
+	BYTE    $0xc4
+	BYTE    $0x63
+	BYTE    $0x89
+	BYTE    $0x22
+	BYTE    $0x76
+	BYTE    $0x50
+	BYTE    $0x01
+	BYTE    $0xc4
+	BYTE    $0x63
+	BYTE    $0x81
+	BYTE    $0x22
+	BYTE    $0x7e
+	BYTE    $0x18
+	BYTE    $0x01
+	VPADDQ  X12, X0, X0
+	VPADDQ  X2, X0, X0
+	VPADDQ  X13, X1, X1
+	VPADDQ  X3, X1, X1
+	VPXOR   X0, X6, X6
+	VPXOR   X1, X7, X7
+	VPSHUFD $-79, X6, X6
+	VPSHUFD $-79, X7, X7
+	VPADDQ  X6, X4, X4
+	VPADDQ  X7, X5, X5
+	VPXOR   X4, X2, X2
+	VPXOR   X5, X3, X3
+	VPSHUFB X8, X2, X2
+	VPSHUFB X8, X3, X3
+	VPADDQ  X14, X0, X0
+	VPADDQ  X2, X0, X0
+	VPADDQ  X15, X1, X1
+	VPADDQ  X3, X1, X1
+	VPXOR   X0, X6, X6
+	VPXOR   X1, X7, X7
+	VPSHUFB X9, X6, X6
+	VPSHUFB X9, X7, X7
+	VPADDQ  X6, X4, X4
+	VPADDQ  X7, X5, X5
+	VPXOR   X4, X2, X2
+	VPXOR   X5, X3, X3
+	VPADDQ  X2, X2, X15
+	VPSRLQ  $0x3f, X2, X2
+	VPXOR   X15, X2, X2
+	VPADDQ  X3, X3, X15
+	VPSRLQ  $0x3f, X3, X3
+	VPXOR   X15, X3, X3
+	VMOVDQA X6, X13
+	VMOVDQA X2, X14
+	VMOVDQA X4, X6
+	BYTE    $0xc4
+	BYTE    $0x41
+	BYTE    $0x11
+	BYTE    $0x6c
+	BYTE    $0xfd
+	VMOVDQA X5, X4
+	VMOVDQA X6, X5
+	BYTE    $0xc4
+	BYTE    $0xc1
+	BYTE    $0x41
+	BYTE    $0x6d
+	BYTE    $0xf7
+	BYTE    $0xc5
+	BYTE    $0x41
+	BYTE    $0x6c
+	BYTE    $0xff
+	BYTE    $0xc4
+	BYTE    $0xc1
+	BYTE    $0x11
+	BYTE    $0x6d
+	BYTE    $0xff
+	BYTE    $0xc5
+	BYTE    $0x61
+	BYTE    $0x6c
+	BYTE    $0xfb
+	BYTE    $0xc4
+	BYTE    $0xc1
+	BYTE    $0x69
+	BYTE    $0x6d
+	BYTE    $0xd7
+	BYTE    $0xc4
+	BYTE    $0x41
+	BYTE    $0x09
+	BYTE    $0x6c
+	BYTE    $0xfe
+	BYTE    $0xc4
+	BYTE    $0xc1
+	BYTE    $0x61
+	BYTE    $0x6d
+	BYTE    $0xdf
+	BYTE    $0xc5
+	BYTE    $0x7a
+	BYTE    $0x7e
+	BYTE    $0x66
+	BYTE    $0x20
+	BYTE    $0xc5
+	BYTE    $0x7a
+	BYTE    $0x7e
+	BYTE    $0x6e
+	BYTE    $0x78
+	BYTE    $0xc5
+	BYTE    $0x7a
+	BYTE    $0x7e
+	BYTE    $0x76
+	BYTE    $0x68
+	BYTE    $0xc5
+	BYTE    $0x7a
+	BYTE    $0x7e
+	BYTE    $0x7e
+	BYTE    $0x70
+	BYTE    $0xc4
+	BYTE    $0x63
+	BYTE    $0x99
+	BYTE    $0x22
+	BYTE    $0x66
+	BYTE    $0x38
+	BYTE    $0x01
+	BYTE    $0xc4
+	BYTE    $0x63
+	BYTE    $0x91
+	BYTE    $0x22
+	BYTE    $0x6e
+	BYTE    $0x08
+	BYTE    $0x01
+	BYTE    $0xc4
+	BYTE    $0x63
+	BYTE    $0x89
+	BYTE    $0x22
+	BYTE    $0x76
+	BYTE    $0x28
+	BYTE    $0x01
+	BYTE    $0xc4
+	BYTE    $0x63
+	BYTE    $0x81
+	BYTE    $0x22
+	BYTE    $0x7e
+	BYTE    $0x48
+	BYTE    $0x01
+	VPADDQ  X12, X0, X0
+	VPADDQ  X2, X0, X0
+	VPADDQ  X13, X1, X1
+	VPADDQ  X3, X1, X1
+	VPXOR   X0, X6, X6
+	VPXOR   X1, X7, X7
+	VPSHUFD $-79, X6, X6
+	VPSHUFD $-79, X7, X7
+	VPADDQ  X6, X4, X4
+	VPADDQ  X7, X5, X5
+	VPXOR   X4, X2, X2
+	VPXOR   X5, X3, X3
+	VPSHUFB X8, X2, X2
+	VPSHUFB X8, X3, X3
+	VPADDQ  X14, X0, X0
+	VPADDQ  X2, X0, X0
+	VPADDQ  X15, X1, X1
+	VPADDQ  X3, X1, X1
+	VPXOR   X0, X6, X6
+	VPXOR   X1, X7, X7
+	VPSHUFB X9, X6, X6
+	VPSHUFB X9, X7, X7
+	VPADDQ  X6, X4, X4
+	VPADDQ  X7, X5, X5
+	VPXOR   X4, X2, X2
+	VPXOR   X5, X3, X3
+	VPADDQ  X2, X2, X15
+	VPSRLQ  $0x3f, X2, X2
+	VPXOR   X15, X2, X2
+	VPADDQ  X3, X3, X15
+	VPSRLQ  $0x3f, X3, X3
+	VPXOR   X15, X3, X3
+	VMOVDQA X2, X13
+	VMOVDQA X4, X14
+	BYTE    $0xc5
+	BYTE    $0x69
+	BYTE    $0x6c
+	BYTE    $0xfa
+	VMOVDQA X5, X4
+	BYTE    $0xc4
+	BYTE    $0xc1
+	BYTE    $0x61
+	BYTE    $0x6d
+	BYTE    $0xd7
+	VMOVDQA X14, X5
+	BYTE    $0xc5
+	BYTE    $0x61
+	BYTE    $0x6c
+	BYTE    $0xfb
+	VMOVDQA X6, X14
+	BYTE    $0xc4
+	BYTE    $0xc1
+	BYTE    $0x11
+	BYTE    $0x6d
+	BYTE    $0xdf
+	BYTE    $0xc5
+	BYTE    $0x41
+	BYTE    $0x6c
+	BYTE    $0xff
+	BYTE    $0xc4
+	BYTE    $0xc1
+	BYTE    $0x49
+	BYTE    $0x6d
+	BYTE    $0xf7
+	BYTE    $0xc4
+	BYTE    $0x41
+	BYTE    $0x09
+	BYTE    $0x6c
+	BYTE    $0xfe
+	BYTE    $0xc4
+	BYTE    $0xc1
+	BYTE    $0x41
+	BYTE    $0x6d
+	BYTE    $0xff
+	BYTE    $0xc5
+	BYTE    $0x7a
+	BYTE    $0x7e
+	BYTE    $0x66
+	BYTE    $0x60
+	BYTE    $0xc5
+	BYTE    $0x7a
+	BYTE    $0x7e
+	BYTE    $0x6e
+	BYTE    $0x70
+	BYTE    $0xc5
+	BYTE    $0x7a
+	BYTE    $0x7e
+	BYTE    $0x76
+	BYTE    $0x28
+	BYTE    $0xc5
+	BYTE    $0x7a
+	BYTE    $0x7e
+	BYTE    $0x7e
+	BYTE    $0x68
+	BYTE    $0xc4
+	BYTE    $0x63
+	BYTE    $0x99
+	BYTE    $0x22
+	BYTE    $0x66
+	BYTE    $0x08
+	BYTE    $0x01
+	BYTE    $0xc4
+	BYTE    $0x63
+	BYTE    $0x91
+	BYTE    $0x22
+	BYTE    $0x6e
+	BYTE    $0x20
+	BYTE    $0x01
+	BYTE    $0xc4
+	BYTE    $0x63
+	BYTE    $0x89
+	BYTE    $0x22
+	BYTE    $0x76
+	BYTE    $0x78
+	BYTE    $0x01
+	BYTE    $0xc4
+	BYTE    $0x63
+	BYTE    $0x81
+	BYTE    $0x22
+	BYTE    $0x7e
+	BYTE    $0x50
+	BYTE    $0x01
+	VPADDQ  X12, X0, X0
+	VPADDQ  X2, X0, X0
+	VPADDQ  X13, X1, X1
+	VPADDQ  X3, X1, X1
+	VPXOR   X0, X6, X6
+	VPXOR   X1, X7, X7
+	VPSHUFD $-79, X6, X6
+	VPSHUFD $-79, X7, X7
+	VPADDQ  X6, X4, X4
+	VPADDQ  X7, X5, X5
+	VPXOR   X4, X2, X2
+	VPXOR   X5, X3, X3
+	VPSHUFB X8, X2, X2
+	VPSHUFB X8, X3, X3
+	VPADDQ  X14, X0, X0
+	VPADDQ  X2, X0, X0
+	VPADDQ  X15, X1, X1
+	VPADDQ  X3, X1, X1
+	VPXOR   X0, X6, X6
+	VPXOR   X1, X7, X7
+	VPSHUFB X9, X6, X6
+	VPSHUFB X9, X7, X7
+	VPADDQ  X6, X4, X4
+	VPADDQ  X7, X5, X5
+	VPXOR   X4, X2, X2
+	VPXOR   X5, X3, X3
+	VPADDQ  X2, X2, X15
+	VPSRLQ  $0x3f, X2, X2
+	VPXOR   X15, X2, X2
+	VPADDQ  X3, X3, X15
+	VPSRLQ  $0x3f, X3, X3
+	VPXOR   X15, X3, X3
+	VMOVDQA X6, X13
+	VMOVDQA X2, X14
+	VMOVDQA X4, X6
+	BYTE    $0xc4
+	BYTE    $0x41
+	BYTE    $0x11
+	BYTE    $0x6c
+	BYTE    $0xfd
+	VMOVDQA X5, X4
+	VMOVDQA X6, X5
+	BYTE    $0xc4
+	BYTE    $0xc1
+	BYTE    $0x41
+	BYTE    $0x6d
+	BYTE    $0xf7
+	BYTE    $0xc5
+	BYTE    $0x41
+	BYTE    $0x6c
+	BYTE    $0xff
+	BYTE    $0xc4
+	BYTE    $0xc1
+	BYTE    $0x11
+	BYTE    $0x6d
+	BYTE    $0xff
+	BYTE    $0xc5
+	BYTE    $0x61
+	BYTE    $0x6c
+	BYTE    $0xfb
+	BYTE    $0xc4
+	BYTE    $0xc1
+	BYTE    $0x69
+	BYTE    $0x6d
+	BYTE    $0xd7
+	BYTE    $0xc4
+	BYTE    $0x41
+	BYTE    $0x09
+	BYTE    $0x6c
+	BYTE    $0xfe
+	BYTE    $0xc4
+	BYTE    $0xc1
+	BYTE    $0x61
+	BYTE    $0x6d
+	BYTE    $0xdf
+	MOVQ    (SI), X12
+	VPSHUFD $0x4e, 64(SI), X13
+	MOVQ    56(SI), X14
+	MOVQ    16(SI), X15
+	BYTE    $0xc4
+	BYTE    $0x63
+	BYTE    $0x99
+	BYTE    $0x22
+	BYTE    $0x66
+	BYTE    $0x30
+	BYTE    $0x01
+	BYTE    $0xc4
+	BYTE    $0x63
+	BYTE    $0x89
+	BYTE    $0x22
+	BYTE    $0x76
+	BYTE    $0x18
+	BYTE    $0x01
+	BYTE    $0xc4
+	BYTE    $0x63
+	BYTE    $0x81
+	BYTE    $0x22
+	BYTE    $0x7e
+	BYTE    $0x58
+	BYTE    $0x01
+	VPADDQ  X12, X0, X0
+	VPADDQ  X2, X0, X0
+	VPADDQ  X13, X1, X1
+	VPADDQ  X3, X1, X1
+	VPXOR   X0, X6, X6
+	VPXOR   X1, X7, X7
+	VPSHUFD $-79, X6, X6
+	VPSHUFD $-79, X7, X7
+	VPADDQ  X6, X4, X4
+	VPADDQ  X7, X5, X5
+	VPXOR   X4, X2, X2
+	VPXOR   X5, X3, X3
+	VPSHUFB X8, X2, X2
+	VPSHUFB X8, X3, X3
+	VPADDQ  X14, X0, X0
+	VPADDQ  X2, X0, X0
+	VPADDQ  X15, X1, X1
+	VPADDQ  X3, X1, X1
+	VPXOR   X0, X6, X6
+	VPXOR   X1, X7, X7
+	VPSHUFB X9, X6, X6
+	VPSHUFB X9, X7, X7
+	VPADDQ  X6, X4, X4
+	VPADDQ  X7, X5, X5
+	VPXOR   X4, X2, X2
+	VPXOR   X5, X3, X3
+	VPADDQ  X2, X2, X15
+	VPSRLQ  $0x3f, X2, X2
+	VPXOR   X15, X2, X2
+	VPADDQ  X3, X3, X15
+	VPSRLQ  $0x3f, X3, X3
+	VPXOR   X15, X3, X3
+	VMOVDQA X2, X13
+	VMOVDQA X4, X14
+	BYTE    $0xc5
+	BYTE    $0x69
+	BYTE    $0x6c
+	BYTE    $0xfa
+	VMOVDQA X5, X4
+	BYTE    $0xc4
+	BYTE    $0xc1
+	BYTE    $0x61
+	BYTE    $0x6d
+	BYTE    $0xd7
+	VMOVDQA X14, X5
+	BYTE    $0xc5
+	BYTE    $0x61
+	BYTE    $0x6c
+	BYTE    $0xfb
+	VMOVDQA X6, X14
+	BYTE    $0xc4
+	BYTE    $0xc1
+	BYTE    $0x11
+	BYTE    $0x6d
+	BYTE    $0xdf
+	BYTE    $0xc5
+	BYTE    $0x41
+	BYTE    $0x6c
+	BYTE    $0xff
+	BYTE    $0xc4
+	BYTE    $0xc1
+	BYTE    $0x49
+	BYTE    $0x6d
+	BYTE    $0xf7
+	BYTE    $0xc4
+	BYTE    $0x41
+	BYTE    $0x09
+	BYTE    $0x6c
+	BYTE    $0xfe
+	BYTE    $0xc4
+	BYTE    $0xc1
+	BYTE    $0x41
+	BYTE    $0x6d
+	BYTE    $0xff
+	BYTE    $0xc5
+	BYTE    $0x7a
+	BYTE    $0x7e
+	BYTE    $0x66
+	BYTE    $0x68
+	BYTE    $0xc5
+	BYTE    $0x7a
+	BYTE    $0x7e
+	BYTE    $0x6e
+	BYTE    $0x60
+	BYTE    $0xc5
+	BYTE    $0x7a
+	BYTE    $0x7e
+	BYTE    $0x76
+	BYTE    $0x58
+	BYTE    $0xc5
+	BYTE    $0x7a
+	BYTE    $0x7e
+	BYTE    $0x7e
+	BYTE    $0x08
+	BYTE    $0xc4
+	BYTE    $0x63
+	BYTE    $0x99
+	BYTE    $0x22
+	BYTE    $0x66
+	BYTE    $0x38
+	BYTE    $0x01
+	BYTE    $0xc4
+	BYTE    $0x63
+	BYTE    $0x91
+	BYTE    $0x22
+	BYTE    $0x6e
+	BYTE    $0x18
+	BYTE    $0x01
+	BYTE    $0xc4
+	BYTE    $0x63
+	BYTE    $0x89
+	BYTE    $0x22
+	BYTE    $0x76
+	BYTE    $0x70
+	BYTE    $0x01
+	BYTE    $0xc4
+	BYTE    $0x63
+	BYTE    $0x81
+	BYTE    $0x22
+	BYTE    $0x7e
+	BYTE    $0x48
+	BYTE    $0x01
+	VPADDQ  X12, X0, X0
+	VPADDQ  X2, X0, X0
+	VPADDQ  X13, X1, X1
+	VPADDQ  X3, X1, X1
+	VPXOR   X0, X6, X6
+	VPXOR   X1, X7, X7
+	VPSHUFD $-79, X6, X6
+	VPSHUFD $-79, X7, X7
+	VPADDQ  X6, X4, X4
+	VPADDQ  X7, X5, X5
+	VPXOR   X4, X2, X2
+	VPXOR   X5, X3, X3
+	VPSHUFB X8, X2, X2
+	VPSHUFB X8, X3, X3
+	VPADDQ  X14, X0, X0
+	VPADDQ  X2, X0, X0
+	VPADDQ  X15, X1, X1
+	VPADDQ  X3, X1, X1
+	VPXOR   X0, X6, X6
+	VPXOR   X1, X7, X7
+	VPSHUFB X9, X6, X6
+	VPSHUFB X9, X7, X7
+	VPADDQ  X6, X4, X4
+	VPADDQ  X7, X5, X5
+	VPXOR   X4, X2, X2
+	VPXOR   X5, X3, X3
+	VPADDQ  X2, X2, X15
+	VPSRLQ  $0x3f, X2, X2
+	VPXOR   X15, X2, X2
+	VPADDQ  X3, X3, X15
+	VPSRLQ  $0x3f, X3, X3
+	VPXOR   X15, X3, X3
+	VMOVDQA X6, X13
+	VMOVDQA X2, X14
+	VMOVDQA X4, X6
+	BYTE    $0xc4
+	BYTE    $0x41
+	BYTE    $0x11
+	BYTE    $0x6c
+	BYTE    $0xfd
+	VMOVDQA X5, X4
+	VMOVDQA X6, X5
+	BYTE    $0xc4
+	BYTE    $0xc1
+	BYTE    $0x41
+	BYTE    $0x6d
+	BYTE    $0xf7
+	BYTE    $0xc5
+	BYTE    $0x41
+	BYTE    $0x6c
+	BYTE    $0xff
+	BYTE    $0xc4
+	BYTE    $0xc1
+	BYTE    $0x11
+	BYTE    $0x6d
+	BYTE    $0xff
+	BYTE    $0xc5
+	BYTE    $0x61
+	BYTE    $0x6c
+	BYTE    $0xfb
+	BYTE    $0xc4
+	BYTE    $0xc1
+	BYTE    $0x69
+	BYTE    $0x6d
+	BYTE    $0xd7
+	BYTE    $0xc4
+	BYTE    $0x41
+	BYTE    $0x09
+	BYTE    $0x6c
+	BYTE    $0xfe
+	BYTE    $0xc4
+	BYTE    $0xc1
+	BYTE    $0x61
+	BYTE    $0x6d
+	BYTE    $0xdf
+	MOVQ    40(SI), X12
+	MOVQ    64(SI), X13
+	MOVQ    (SI), X14
+	MOVQ    48(SI), X15
+	BYTE    $0xc4
+	BYTE    $0x63
+	BYTE    $0x99
+	BYTE    $0x22
+	BYTE    $0x66
+	BYTE    $0x78
+	BYTE    $0x01
+	BYTE    $0xc4
+	BYTE    $0x63
+	BYTE    $0x91
+	BYTE    $0x22
+	BYTE    $0x6e
+	BYTE    $0x10
+	BYTE    $0x01
+	BYTE    $0xc4
+	BYTE    $0x63
+	BYTE    $0x89
+	BYTE    $0x22
+	BYTE    $0x76
+	BYTE    $0x20
+	BYTE    $0x01
+	BYTE    $0xc4
+	BYTE    $0x63
+	BYTE    $0x81
+	BYTE    $0x22
+	BYTE    $0x7e
+	BYTE    $0x50
+	BYTE    $0x01
+	VPADDQ  X12, X0, X0
+	VPADDQ  X2, X0, X0
+	VPADDQ  X13, X1, X1
+	VPADDQ  X3, X1, X1
+	VPXOR   X0, X6, X6
+	VPXOR   X1, X7, X7
+	VPSHUFD $-79, X6, X6
+	VPSHUFD $-79, X7, X7
+	VPADDQ  X6, X4, X4
+	VPADDQ  X7, X5, X5
+	VPXOR   X4, X2, X2
+	VPXOR   X5, X3, X3
+	VPSHUFB X8, X2, X2
+	VPSHUFB X8, X3, X3
+	VPADDQ  X14, X0, X0
+	VPADDQ  X2, X0, X0
+	VPADDQ  X15, X1, X1
+	VPADDQ  X3, X1, X1
+	VPXOR   X0, X6, X6
+	VPXOR   X1, X7, X7
+	VPSHUFB X9, X6, X6
+	VPSHUFB X9, X7, X7
+	VPADDQ  X6, X4, X4
+	VPADDQ  X7, X5, X5
+	VPXOR   X4, X2, X2
+	VPXOR   X5, X3, X3
+	VPADDQ  X2, X2, X15
+	VPSRLQ  $0x3f, X2, X2
+	VPXOR   X15, X2, X2
+	VPADDQ  X3, X3, X15
+	VPSRLQ  $0x3f, X3, X3
+	VPXOR   X15, X3, X3
+	VMOVDQA X2, X13
+	VMOVDQA X4, X14
+	BYTE    $0xc5
+	BYTE    $0x69
+	BYTE    $0x6c
+	BYTE    $0xfa
+	VMOVDQA X5, X4
+	BYTE    $0xc4
+	BYTE    $0xc1
+	BYTE    $0x61
+	BYTE    $0x6d
+	BYTE    $0xd7
+	VMOVDQA X14, X5
+	BYTE    $0xc5
+	BYTE    $0x61
+	BYTE    $0x6c
+	BYTE    $0xfb
+	VMOVDQA X6, X14
+	BYTE    $0xc4
+	BYTE    $0xc1
+	BYTE    $0x11
+	BYTE    $0x6d
+	BYTE    $0xdf
+	BYTE    $0xc5
+	BYTE    $0x41
+	BYTE    $0x6c
+	BYTE    $0xff
+	BYTE    $0xc4
+	BYTE    $0xc1
+	BYTE    $0x49
+	BYTE    $0x6d
+	BYTE    $0xf7
+	BYTE    $0xc4
+	BYTE    $0x41
+	BYTE    $0x09
+	BYTE    $0x6c
+	BYTE    $0xfe
+	BYTE    $0xc4
+	BYTE    $0xc1
+	BYTE    $0x41
+	BYTE    $0x6d
+	BYTE    $0xff
+	MOVQ    48(SI), X12
+	MOVQ    88(SI), X13
+	MOVQ    120(SI), X14
+	MOVQ    24(SI), X15
+	BYTE    $0xc4
+	BYTE    $0x63
+	BYTE    $0x99
+	BYTE    $0x22
+	BYTE    $0x66
+	BYTE    $0x70
+	BYTE    $0x01
+	BYTE    $0xc4
+	BYTE    $0x63
+	BYTE    $0x91
+	BYTE    $0x22
+	BYTE    $0x2e
+	BYTE    $0x01
+	BYTE    $0xc4
+	BYTE    $0x63
+	BYTE    $0x89
+	BYTE    $0x22
+	BYTE    $0x76
+	BYTE    $0x48
+	BYTE    $0x01
+	BYTE    $0xc4
+	BYTE    $0x63
+	BYTE    $0x81
+	BYTE    $0x22
+	BYTE    $0x7e
+	BYTE    $0x40
+	BYTE    $0x01
+	VPADDQ  X12, X0, X0
+	VPADDQ  X2, X0, X0
+	VPADDQ  X13, X1, X1
+	VPADDQ  X3, X1, X1
+	VPXOR   X0, X6, X6
+	VPXOR   X1, X7, X7
+	VPSHUFD $-79, X6, X6
+	VPSHUFD $-79, X7, X7
+	VPADDQ  X6, X4, X4
+	VPADDQ  X7, X5, X5
+	VPXOR   X4, X2, X2
+	VPXOR   X5, X3, X3
+	VPSHUFB X8, X2, X2
+	VPSHUFB X8, X3, X3
+	VPADDQ  X14, X0, X0
+	VPADDQ  X2, X0, X0
+	VPADDQ  X15, X1, X1
+	VPADDQ  X3, X1, X1
+	VPXOR   X0, X6, X6
+	VPXOR   X1, X7, X7
+	VPSHUFB X9, X6, X6
+	VPSHUFB X9, X7, X7
+	VPADDQ  X6, X4, X4
+	VPADDQ  X7, X5, X5
+	VPXOR   X4, X2, X2
+	VPXOR   X5, X3, X3
+	VPADDQ  X2, X2, X15
+	VPSRLQ  $0x3f, X2, X2
+	VPXOR   X15, X2, X2
+	VPADDQ  X3, X3, X15
+	VPSRLQ  $0x3f, X3, X3
+	VPXOR   X15, X3, X3
+	VMOVDQA X6, X13
+	VMOVDQA X2, X14
+	VMOVDQA X4, X6
+	BYTE    $0xc4
+	BYTE    $0x41
+	BYTE    $0x11
+	BYTE    $0x6c
+	BYTE    $0xfd
+	VMOVDQA X5, X4
+	VMOVDQA X6, X5
+	BYTE    $0xc4
+	BYTE    $0xc1
+	BYTE    $0x41
+	BYTE    $0x6d
+	BYTE    $0xf7
+	BYTE    $0xc5
+	BYTE    $0x41
+	BYTE    $0x6c
+	BYTE    $0xff
+	BYTE    $0xc4
+	BYTE    $0xc1
+	BYTE    $0x11
+	BYTE    $0x6d
+	BYTE    $0xff
+	BYTE    $0xc5
+	BYTE    $0x61
+	BYTE    $0x6c
+	BYTE    $0xfb
+	BYTE    $0xc4
+	BYTE    $0xc1
+	BYTE    $0x69
+	BYTE    $0x6d
+	BYTE    $0xd7
+	BYTE    $0xc4
+	BYTE    $0x41
+	BYTE    $0x09
+	BYTE    $0x6c
+	BYTE    $0xfe
+	BYTE    $0xc4
+	BYTE    $0xc1
+	BYTE    $0x61
+	BYTE    $0x6d
+	BYTE    $0xdf
+	VMOVDQU 96(SI), X12
+	MOVQ    8(SI), X13
+	MOVQ    16(SI), X14
+	BYTE    $0xc4
+	BYTE    $0x63
+	BYTE    $0x91
+	BYTE    $0x22
+	BYTE    $0x6e
+	BYTE    $0x50
+	BYTE    $0x01
+	BYTE    $0xc4
+	BYTE    $0x63
+	BYTE    $0x89
+	BYTE    $0x22
+	BYTE    $0x76
+	BYTE    $0x38
+	BYTE    $0x01
+	VMOVDQU 32(SI), X15
+	VPADDQ  X12, X0, X0
+	VPADDQ  X2, X0, X0
+	VPADDQ  X13, X1, X1
+	VPADDQ  X3, X1, X1
+	VPXOR   X0, X6, X6
+	VPXOR   X1, X7, X7
+	VPSHUFD $-79, X6, X6
+	VPSHUFD $-79, X7, X7
+	VPADDQ  X6, X4, X4
+	VPADDQ  X7, X5, X5
+	VPXOR   X4, X2, X2
+	VPXOR   X5, X3, X3
+	VPSHUFB X8, X2, X2
+	VPSHUFB X8, X3, X3
+	VPADDQ  X14, X0, X0
+	VPADDQ  X2, X0, X0
+	VPADDQ  X15, X1, X1
+	VPADDQ  X3, X1, X1
+	VPXOR   X0, X6, X6
+	VPXOR   X1, X7, X7
+	VPSHUFB X9, X6, X6
+	VPSHUFB X9, X7, X7
+	VPADDQ  X6, X4, X4
+	VPADDQ  X7, X5, X5
+	VPXOR   X4, X2, X2
+	VPXOR   X5, X3, X3
+	VPADDQ  X2, X2, X15
+	VPSRLQ  $0x3f, X2, X2
+	VPXOR   X15, X2, X2
+	VPADDQ  X3, X3, X15
+	VPSRLQ  $0x3f, X3, X3
+	VPXOR   X15, X3, X3
+	VMOVDQA X2, X13
+	VMOVDQA X4, X14
+	BYTE    $0xc5
+	BYTE    $0x69
+	BYTE    $0x6c
+	BYTE    $0xfa
+	VMOVDQA X5, X4
+	BYTE    $0xc4
+	BYTE    $0xc1
+	BYTE    $0x61
+	BYTE    $0x6d
+	BYTE    $0xd7
+	VMOVDQA X14, X5
+	BYTE    $0xc5
+	BYTE    $0x61
+	BYTE    $0x6c
+	BYTE    $0xfb
+	VMOVDQA X6, X14
+	BYTE    $0xc4
+	BYTE    $0xc1
+	BYTE    $0x11
+	BYTE    $0x6d
+	BYTE    $0xdf
+	BYTE    $0xc5
+	BYTE    $0x41
+	BYTE    $0x6c
+	BYTE    $0xff
+	BYTE    $0xc4
+	BYTE    $0xc1
+	BYTE    $0x49
+	BYTE    $0x6d
+	BYTE    $0xf7
+	BYTE    $0xc4
+	BYTE    $0x41
+	BYTE    $0x09
+	BYTE    $0x6c
+	BYTE    $0xfe
+	BYTE    $0xc4
+	BYTE    $0xc1
+	BYTE    $0x41
+	BYTE    $0x6d
+	BYTE    $0xff
+	BYTE    $0xc5
+	BYTE    $0x7a
+	BYTE    $0x7e
+	BYTE    $0x66
+	BYTE    $0x50
+	BYTE    $0xc5
+	BYTE    $0x7a
+	BYTE    $0x7e
+	BYTE    $0x6e
+	BYTE    $0x38
+	BYTE    $0xc5
+	BYTE    $0x7a
+	BYTE    $0x7e
+	BYTE    $0x76
+	BYTE    $0x10
+	BYTE    $0xc5
+	BYTE    $0x7a
+	BYTE    $0x7e
+	BYTE    $0x7e
+	BYTE    $0x30
+	BYTE    $0xc4
+	BYTE    $0x63
+	BYTE    $0x99
+	BYTE    $0x22
+	BYTE    $0x66
+	BYTE    $0x40
+	BYTE    $0x01
+	BYTE    $0xc4
+	BYTE    $0x63
+	BYTE    $0x91
+	BYTE    $0x22
+	BYTE    $0x6e
+	BYTE    $0x08
+	BYTE    $0x01
+	BYTE    $0xc4
+	BYTE    $0x63
+	BYTE    $0x89
+	BYTE    $0x22
+	BYTE    $0x76
+	BYTE    $0x20
+	BYTE    $0x01
+	BYTE    $0xc4
+	BYTE    $0x63
+	BYTE    $0x81
+	BYTE    $0x22
+	BYTE    $0x7e
+	BYTE    $0x28
+	BYTE    $0x01
+	VPADDQ  X12, X0, X0
+	VPADDQ  X2, X0, X0
+	VPADDQ  X13, X1, X1
+	VPADDQ  X3, X1, X1
+	VPXOR   X0, X6, X6
+	VPXOR   X1, X7, X7
+	VPSHUFD $-79, X6, X6
+	VPSHUFD $-79, X7, X7
+	VPADDQ  X6, X4, X4
+	VPADDQ  X7, X5, X5
+	VPXOR   X4, X2, X2
+	VPXOR   X5, X3, X3
+	VPSHUFB X8, X2, X2
+	VPSHUFB X8, X3, X3
+	VPADDQ  X14, X0, X0
+	VPADDQ  X2, X0, X0
+	VPADDQ  X15, X1, X1
+	VPADDQ  X3, X1, X1
+	VPXOR   X0, X6, X6
+	VPXOR   X1, X7, X7
+	VPSHUFB X9, X6, X6
+	VPSHUFB X9, X7, X7
+	VPADDQ  X6, X4, X4
+	VPADDQ  X7, X5, X5
+	VPXOR   X4, X2, X2
+	VPXOR   X5, X3, X3
+	VPADDQ  X2, X2, X15
+	VPSRLQ  $0x3f, X2, X2
+	VPXOR   X15, X2, X2
+	VPADDQ  X3, X3, X15
+	VPSRLQ  $0x3f, X3, X3
+	VPXOR   X15, X3, X3
+	VMOVDQA X6, X13
+	VMOVDQA X2, X14
+	VMOVDQA X4, X6
+	BYTE    $0xc4
+	BYTE    $0x41
+	BYTE    $0x11
+	BYTE    $0x6c
+	BYTE    $0xfd
+	VMOVDQA X5, X4
+	VMOVDQA X6, X5
+	BYTE    $0xc4
+	BYTE    $0xc1
+	BYTE    $0x41
+	BYTE    $0x6d
+	BYTE    $0xf7
+	BYTE    $0xc5
+	BYTE    $0x41
+	BYTE    $0x6c
+	BYTE    $0xff
+	BYTE    $0xc4
+	BYTE    $0xc1
+	BYTE    $0x11
+	BYTE    $0x6d
+	BYTE    $0xff
+	BYTE    $0xc5
+	BYTE    $0x61
+	BYTE    $0x6c
+	BYTE    $0xfb
+	BYTE    $0xc4
+	BYTE    $0xc1
+	BYTE    $0x69
+	BYTE    $0x6d
+	BYTE    $0xd7
+	BYTE    $0xc4
+	BYTE    $0x41
+	BYTE    $0x09
+	BYTE    $0x6c
+	BYTE    $0xfe
+	BYTE    $0xc4
+	BYTE    $0xc1
+	BYTE    $0x61
+	BYTE    $0x6d
+	BYTE    $0xdf
+	MOVQ    120(SI), X12
+	MOVQ    24(SI), X13
+	MOVQ    88(SI), X14
+	MOVQ    96(SI), X15
+	BYTE    $0xc4
+	BYTE    $0x63
+	BYTE    $0x99
+	BYTE    $0x22
+	BYTE    $0x66
+	BYTE    $0x48
+	BYTE    $0x01
+	BYTE    $0xc4
+	BYTE    $0x63
+	BYTE    $0x91
+	BYTE    $0x22
+	BYTE    $0x6e
+	BYTE    $0x68
+	BYTE    $0x01
+	BYTE    $0xc4
+	BYTE    $0x63
+	BYTE    $0x89
+	BYTE    $0x22
+	BYTE    $0x76
+	BYTE    $0x70
+	BYTE    $0x01
+	BYTE    $0xc4
+	BYTE    $0x63
+	BYTE    $0x81
+	BYTE    $0x22
+	BYTE    $0x3e
+	BYTE    $0x01
+	VPADDQ  X12, X0, X0
+	VPADDQ  X2, X0, X0
+	VPADDQ  X13, X1, X1
+	VPADDQ  X3, X1, X1
+	VPXOR   X0, X6, X6
+	VPXOR   X1, X7, X7
+	VPSHUFD $-79, X6, X6
+	VPSHUFD $-79, X7, X7
+	VPADDQ  X6, X4, X4
+	VPADDQ  X7, X5, X5
+	VPXOR   X4, X2, X2
+	VPXOR   X5, X3, X3
+	VPSHUFB X8, X2, X2
+	VPSHUFB X8, X3, X3
+	VPADDQ  X14, X0, X0
+	VPADDQ  X2, X0, X0
+	VPADDQ  X15, X1, X1
+	VPADDQ  X3, X1, X1
+	VPXOR   X0, X6, X6
+	VPXOR   X1, X7, X7
+	VPSHUFB X9, X6, X6
+	VPSHUFB X9, X7, X7
+	VPADDQ  X6, X4, X4
+	VPADDQ  X7, X5, X5
+	VPXOR   X4, X2, X2
+	VPXOR   X5, X3, X3
+	VPADDQ  X2, X2, X15
+	VPSRLQ  $0x3f, X2, X2
+	VPXOR   X15, X2, X2
+	VPADDQ  X3, X3, X15
+	VPSRLQ  $0x3f, X3, X3
+	VPXOR   X15, X3, X3
+	VMOVDQA X2, X13
+	VMOVDQA X4, X14
+	BYTE    $0xc5
+	BYTE    $0x69
+	BYTE    $0x6c
+	BYTE    $0xfa
+	VMOVDQA X5, X4
+	BYTE    $0xc4
+	BYTE    $0xc1
+	BYTE    $0x61
+	BYTE    $0x6d
+	BYTE    $0xd7
+	VMOVDQA X14, X5
+	BYTE    $0xc5
+	BYTE    $0x61
+	BYTE    $0x6c
+	BYTE    $0xfb
+	VMOVDQA X6, X14
+	BYTE    $0xc4
+	BYTE    $0xc1
+	BYTE    $0x11
+	BYTE    $0x6d
+	BYTE    $0xdf
+	BYTE    $0xc5
+	BYTE    $0x41
+	BYTE    $0x6c
+	BYTE    $0xff
+	BYTE    $0xc4
+	BYTE    $0xc1
+	BYTE    $0x49
+	BYTE    $0x6d
+	BYTE    $0xf7
+	BYTE    $0xc4
+	BYTE    $0x41
+	BYTE    $0x09
+	BYTE    $0x6c
+	BYTE    $0xfe
+	BYTE    $0xc4
+	BYTE    $0xc1
+	BYTE    $0x41
+	BYTE    $0x6d
+	BYTE    $0xff
+	VPADDQ  16(R10), X0, X0
+	VPADDQ  X2, X0, X0
+	VPADDQ  32(R10), X1, X1
+	VPADDQ  X3, X1, X1
+	VPXOR   X0, X6, X6
+	VPXOR   X1, X7, X7
+	VPSHUFD $-79, X6, X6
+	VPSHUFD $-79, X7, X7
+	VPADDQ  X6, X4, X4
+	VPADDQ  X7, X5, X5
+	VPXOR   X4, X2, X2
+	VPXOR   X5, X3, X3
+	VPSHUFB X8, X2, X2
+	VPSHUFB X8, X3, X3
+	VPADDQ  48(R10), X0, X0
+	VPADDQ  X2, X0, X0
+	VPADDQ  64(R10), X1, X1
+	VPADDQ  X3, X1, X1
+	VPXOR   X0, X6, X6
+	VPXOR   X1, X7, X7
+	VPSHUFB X9, X6, X6
+	VPSHUFB X9, X7, X7
+	VPADDQ  X6, X4, X4
+	VPADDQ  X7, X5, X5
+	VPXOR   X4, X2, X2
+	VPXOR   X5, X3, X3
+	VPADDQ  X2, X2, X15
+	VPSRLQ  $0x3f, X2, X2
+	VPXOR   X15, X2, X2
+	VPADDQ  X3, X3, X15
+	VPSRLQ  $0x3f, X3, X3
+	VPXOR   X15, X3, X3
+	VMOVDQA X6, X13
+	VMOVDQA X2, X14
+	VMOVDQA X4, X6
+	BYTE    $0xc4
+	BYTE    $0x41
+	BYTE    $0x11
+	BYTE    $0x6c
+	BYTE    $0xfd
+	VMOVDQA X5, X4
+	VMOVDQA X6, X5
+	BYTE    $0xc4
+	BYTE    $0xc1
+	BYTE    $0x41
+	BYTE    $0x6d
+	BYTE    $0xf7
+	BYTE    $0xc5
+	BYTE    $0x41
+	BYTE    $0x6c
+	BYTE    $0xff
+	BYTE    $0xc4
+	BYTE    $0xc1
+	BYTE    $0x11
+	BYTE    $0x6d
+	BYTE    $0xff
+	BYTE    $0xc5
+	BYTE    $0x61
+	BYTE    $0x6c
+	BYTE    $0xfb
+	BYTE    $0xc4
+	BYTE    $0xc1
+	BYTE    $0x69
+	BYTE    $0x6d
+	BYTE    $0xd7
+	BYTE    $0xc4
+	BYTE    $0x41
+	BYTE    $0x09
+	BYTE    $0x6c
+	BYTE    $0xfe
+	BYTE    $0xc4
+	BYTE    $0xc1
+	BYTE    $0x61
+	BYTE    $0x6d
+	BYTE    $0xdf
+	VPADDQ  80(R10), X0, X0
+	VPADDQ  X2, X0, X0
+	VPADDQ  96(R10), X1, X1
+	VPADDQ  X3, X1, X1
+	VPXOR   X0, X6, X6
+	VPXOR   X1, X7, X7
+	VPSHUFD $-79, X6, X6
+	VPSHUFD $-79, X7, X7
+	VPADDQ  X6, X4, X4
+	VPADDQ  X7, X5, X5
+	VPXOR   X4, X2, X2
+	VPXOR   X5, X3, X3
+	VPSHUFB X8, X2, X2
+	VPSHUFB X8, X3, X3
+	VPADDQ  112(R10), X0, X0
+	VPADDQ  X2, X0, X0
+	VPADDQ  128(R10), X1, X1
+	VPADDQ  X3, X1, X1
+	VPXOR   X0, X6, X6
+	VPXOR   X1, X7, X7
+	VPSHUFB X9, X6, X6
+	VPSHUFB X9, X7, X7
+	VPADDQ  X6, X4, X4
+	VPADDQ  X7, X5, X5
+	VPXOR   X4, X2, X2
+	VPXOR   X5, X3, X3
+	VPADDQ  X2, X2, X15
+	VPSRLQ  $0x3f, X2, X2
+	VPXOR   X15, X2, X2
+	VPADDQ  X3, X3, X15
+	VPSRLQ  $0x3f, X3, X3
+	VPXOR   X15, X3, X3
+	VMOVDQA X2, X13
+	VMOVDQA X4, X14
+	BYTE    $0xc5
+	BYTE    $0x69
+	BYTE    $0x6c
+	BYTE    $0xfa
+	VMOVDQA X5, X4
+	BYTE    $0xc4
+	BYTE    $0xc1
+	BYTE    $0x61
+	BYTE    $0x6d
+	BYTE    $0xd7
+	VMOVDQA X14, X5
+	BYTE    $0xc5
+	BYTE    $0x61
+	BYTE    $0x6c
+	BYTE    $0xfb
+	VMOVDQA X6, X14
+	BYTE    $0xc4
+	BYTE    $0xc1
+	BYTE    $0x11
+	BYTE    $0x6d
+	BYTE    $0xdf
+	BYTE    $0xc5
+	BYTE    $0x41
+	BYTE    $0x6c
+	BYTE    $0xff
+	BYTE    $0xc4
+	BYTE    $0xc1
+	BYTE    $0x49
+	BYTE    $0x6d
+	BYTE    $0xf7
+	BYTE    $0xc4
+	BYTE    $0x41
+	BYTE    $0x09
+	BYTE    $0x6c
+	BYTE    $0xfe
+	BYTE    $0xc4
+	BYTE    $0xc1
+	BYTE    $0x41
+	BYTE    $0x6d
+	BYTE    $0xff
+	VPADDQ  144(R10), X0, X0
+	VPADDQ  X2, X0, X0
+	VPADDQ  160(R10), X1, X1
+	VPADDQ  X3, X1, X1
+	VPXOR   X0, X6, X6
+	VPXOR   X1, X7, X7
+	VPSHUFD $-79, X6, X6
+	VPSHUFD $-79, X7, X7
+	VPADDQ  X6, X4, X4
+	VPADDQ  X7, X5, X5
+	VPXOR   X4, X2, X2
+	VPXOR   X5, X3, X3
+	VPSHUFB X8, X2, X2
+	VPSHUFB X8, X3, X3
+	VPADDQ  176(R10), X0, X0
+	VPADDQ  X2, X0, X0
+	VPADDQ  192(R10), X1, X1
+	VPADDQ  X3, X1, X1
+	VPXOR   X0, X6, X6
+	VPXOR   X1, X7, X7
+	VPSHUFB X9, X6, X6
+	VPSHUFB X9, X7, X7
+	VPADDQ  X6, X4, X4
+	VPADDQ  X7, X5, X5
+	VPXOR   X4, X2, X2
+	VPXOR   X5, X3, X3
+	VPADDQ  X2, X2, X15
+	VPSRLQ  $0x3f, X2, X2
+	VPXOR   X15, X2, X2
+	VPADDQ  X3, X3, X15
+	VPSRLQ  $0x3f, X3, X3
+	VPXOR   X15, X3, X3
+	VMOVDQA X6, X13
+	VMOVDQA X2, X14
+	VMOVDQA X4, X6
+	BYTE    $0xc4
+	BYTE    $0x41
+	BYTE    $0x11
+	BYTE    $0x6c
+	BYTE    $0xfd
+	VMOVDQA X5, X4
+	VMOVDQA X6, X5
+	BYTE    $0xc4
+	BYTE    $0xc1
+	BYTE    $0x41
+	BYTE    $0x6d
+	BYTE    $0xf7
+	BYTE    $0xc5
+	BYTE    $0x41
+	BYTE    $0x6c
+	BYTE    $0xff
+	BYTE    $0xc4
+	BYTE    $0xc1
+	BYTE    $0x11
+	BYTE    $0x6d
+	BYTE    $0xff
+	BYTE    $0xc5
+	BYTE    $0x61
+	BYTE    $0x6c
+	BYTE    $0xfb
+	BYTE    $0xc4
+	BYTE    $0xc1
+	BYTE    $0x69
+	BYTE    $0x6d
+	BYTE    $0xd7
+	BYTE    $0xc4
+	BYTE    $0x41
+	BYTE    $0x09
+	BYTE    $0x6c
+	BYTE    $0xfe
+	BYTE    $0xc4
+	BYTE    $0xc1
+	BYTE    $0x61
+	BYTE    $0x6d
+	BYTE    $0xdf
+	VPADDQ  208(R10), X0, X0
+	VPADDQ  X2, X0, X0
+	VPADDQ  224(R10), X1, X1
+	VPADDQ  X3, X1, X1
+	VPXOR   X0, X6, X6
+	VPXOR   X1, X7, X7
+	VPSHUFD $-79, X6, X6
+	VPSHUFD $-79, X7, X7
+	VPADDQ  X6, X4, X4
+	VPADDQ  X7, X5, X5
+	VPXOR   X4, X2, X2
+	VPXOR   X5, X3, X3
+	VPSHUFB X8, X2, X2
+	VPSHUFB X8, X3, X3
+	VPADDQ  240(R10), X0, X0
+	VPADDQ  X2, X0, X0
+	VPADDQ  256(R10), X1, X1
+	VPADDQ  X3, X1, X1
+	VPXOR   X0, X6, X6
+	VPXOR   X1, X7, X7
+	VPSHUFB X9, X6, X6
+	VPSHUFB X9, X7, X7
+	VPADDQ  X6, X4, X4
+	VPADDQ  X7, X5, X5
+	VPXOR   X4, X2, X2
+	VPXOR   X5, X3, X3
+	VPADDQ  X2, X2, X15
+	VPSRLQ  $0x3f, X2, X2
+	VPXOR   X15, X2, X2
+	VPADDQ  X3, X3, X15
+	VPSRLQ  $0x3f, X3, X3
+	VPXOR   X15, X3, X3
+	VMOVDQA X2, X13
+	VMOVDQA X4, X14
+	BYTE    $0xc5
+	BYTE    $0x69
+	BYTE    $0x6c
+	BYTE    $0xfa
+	VMOVDQA X5, X4
+	BYTE    $0xc4
+	BYTE    $0xc1
+	BYTE    $0x61
+	BYTE    $0x6d
+	BYTE    $0xd7
+	VMOVDQA X14, X5
+	BYTE    $0xc5
+	BYTE    $0x61
+	BYTE    $0x6c
+	BYTE    $0xfb
+	VMOVDQA X6, X14
+	BYTE    $0xc4
+	BYTE    $0xc1
+	BYTE    $0x11
+	BYTE    $0x6d
+	BYTE    $0xdf
+	BYTE    $0xc5
+	BYTE    $0x41
+	BYTE    $0x6c
+	BYTE    $0xff
+	BYTE    $0xc4
+	BYTE    $0xc1
+	BYTE    $0x49
+	BYTE    $0x6d
+	BYTE    $0xf7
+	BYTE    $0xc4
+	BYTE    $0x41
+	BYTE    $0x09
+	BYTE    $0x6c
+	BYTE    $0xfe
+	BYTE    $0xc4
+	BYTE    $0xc1
+	BYTE    $0x41
+	BYTE    $0x6d
+	BYTE    $0xff
 	VMOVDQU 32(AX), X14
 	VMOVDQU 48(AX), X15
 	VPXOR   X0, X10, X10
@@ -729,16 +4524,36 @@ noinc:
 	VPXOR   X7, X15, X3
 	VMOVDQU X2, 32(AX)
 	VMOVDQU X3, 48(AX)
+	LEAQ    128(SI), SI
+	SUBQ    $0x80, DI
+	JNE     loop
+	VMOVDQU X10, (AX)
+	VMOVDQU X11, 16(AX)
+	MOVQ    R8, (BX)
+	MOVQ    R9, 8(BX)
+	VZEROUPPER
+	RET
 
-	LEAQ 128(SI), SI
-	SUBQ $128, DI
-	JNE  loop
+DATA ·AVX_c40<>+0(SB)/8, $0x0201000706050403
+DATA ·AVX_c40<>+8(SB)/8, $0x0a09080f0e0d0c0b
+GLOBL ·AVX_c40<>(SB), RODATA|NOPTR, $16
 
-	VMOVDQU X10, 0(AX)
-	VMOVDQU X11, 16(AX)
+DATA ·AVX_c48<>+0(SB)/8, $0x0100070605040302
+DATA ·AVX_c48<>+8(SB)/8, $0x09080f0e0d0c0b0a
+GLOBL ·AVX_c48<>(SB), RODATA|NOPTR, $16
 
-	MOVQ R8, 0(BX)
-	MOVQ R9, 8(BX)
-	VZEROUPPER
+DATA ·AVX_iv3<>+0(SB)/8, $0x1f83d9abfb41bd6b
+DATA ·AVX_iv3<>+8(SB)/8, $0x5be0cd19137e2179
+GLOBL ·AVX_iv3<>(SB), RODATA|NOPTR, $16
 
-	RET
+DATA ·AVX_iv0<>+0(SB)/8, $0x6a09e667f3bcc908
+DATA ·AVX_iv0<>+8(SB)/8, $0xbb67ae8584caa73b
+GLOBL ·AVX_iv0<>(SB), RODATA|NOPTR, $16
+
+DATA ·AVX_iv1<>+0(SB)/8, $0x3c6ef372fe94f82b
+DATA ·AVX_iv1<>+8(SB)/8, $0xa54ff53a5f1d36f1
+GLOBL ·AVX_iv1<>(SB), RODATA|NOPTR, $16
+
+DATA ·AVX_iv2<>+0(SB)/8, $0x510e527fade682d1
+DATA ·AVX_iv2<>+8(SB)/8, $0x9b05688c2b3e6c1f
+GLOBL ·AVX_iv2<>(SB), RODATA|NOPTR, $16
diff --git a/vendor/golang.org/x/crypto/blake2b/blake2b_amd64.s b/vendor/golang.org/x/crypto/blake2b/blake2b_amd64.s
index adfac00c15c9c..9a0ce2124462f 100644
--- a/vendor/golang.org/x/crypto/blake2b/blake2b_amd64.s
+++ b/vendor/golang.org/x/crypto/blake2b/blake2b_amd64.s
@@ -1,278 +1,1441 @@
-// Copyright 2016 The Go Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
+// Code generated by command: go run blake2b_amd64_asm.go -out ../../blake2b_amd64.s -pkg blake2b. DO NOT EDIT.
 
 //go:build amd64 && gc && !purego
 
 #include "textflag.h"
 
-DATA ·iv0<>+0x00(SB)/8, $0x6a09e667f3bcc908
-DATA ·iv0<>+0x08(SB)/8, $0xbb67ae8584caa73b
-GLOBL ·iv0<>(SB), (NOPTR+RODATA), $16
-
-DATA ·iv1<>+0x00(SB)/8, $0x3c6ef372fe94f82b
-DATA ·iv1<>+0x08(SB)/8, $0xa54ff53a5f1d36f1
-GLOBL ·iv1<>(SB), (NOPTR+RODATA), $16
-
-DATA ·iv2<>+0x00(SB)/8, $0x510e527fade682d1
-DATA ·iv2<>+0x08(SB)/8, $0x9b05688c2b3e6c1f
-GLOBL ·iv2<>(SB), (NOPTR+RODATA), $16
-
-DATA ·iv3<>+0x00(SB)/8, $0x1f83d9abfb41bd6b
-DATA ·iv3<>+0x08(SB)/8, $0x5be0cd19137e2179
-GLOBL ·iv3<>(SB), (NOPTR+RODATA), $16
-
-DATA ·c40<>+0x00(SB)/8, $0x0201000706050403
-DATA ·c40<>+0x08(SB)/8, $0x0a09080f0e0d0c0b
-GLOBL ·c40<>(SB), (NOPTR+RODATA), $16
-
-DATA ·c48<>+0x00(SB)/8, $0x0100070605040302
-DATA ·c48<>+0x08(SB)/8, $0x09080f0e0d0c0b0a
-GLOBL ·c48<>(SB), (NOPTR+RODATA), $16
-
-#define SHUFFLE(v2, v3, v4, v5, v6, v7, t1, t2) \
-	MOVO       v4, t1; \
-	MOVO       v5, v4; \
-	MOVO       t1, v5; \
-	MOVO       v6, t1; \
-	PUNPCKLQDQ v6, t2; \
-	PUNPCKHQDQ v7, v6; \
-	PUNPCKHQDQ t2, v6; \
-	PUNPCKLQDQ v7, t2; \
-	MOVO       t1, v7; \
-	MOVO       v2, t1; \
-	PUNPCKHQDQ t2, v7; \
-	PUNPCKLQDQ v3, t2; \
-	PUNPCKHQDQ t2, v2; \
-	PUNPCKLQDQ t1, t2; \
-	PUNPCKHQDQ t2, v3
-
-#define SHUFFLE_INV(v2, v3, v4, v5, v6, v7, t1, t2) \
-	MOVO       v4, t1; \
-	MOVO       v5, v4; \
-	MOVO       t1, v5; \
-	MOVO       v2, t1; \
-	PUNPCKLQDQ v2, t2; \
-	PUNPCKHQDQ v3, v2; \
-	PUNPCKHQDQ t2, v2; \
-	PUNPCKLQDQ v3, t2; \
-	MOVO       t1, v3; \
-	MOVO       v6, t1; \
-	PUNPCKHQDQ t2, v3; \
-	PUNPCKLQDQ v7, t2; \
-	PUNPCKHQDQ t2, v6; \
-	PUNPCKLQDQ t1, t2; \
-	PUNPCKHQDQ t2, v7
-
-#define HALF_ROUND(v0, v1, v2, v3, v4, v5, v6, v7, m0, m1, m2, m3, t0, c40, c48) \
-	PADDQ  m0, v0;        \
-	PADDQ  m1, v1;        \
-	PADDQ  v2, v0;        \
-	PADDQ  v3, v1;        \
-	PXOR   v0, v6;        \
-	PXOR   v1, v7;        \
-	PSHUFD $0xB1, v6, v6; \
-	PSHUFD $0xB1, v7, v7; \
-	PADDQ  v6, v4;        \
-	PADDQ  v7, v5;        \
-	PXOR   v4, v2;        \
-	PXOR   v5, v3;        \
-	PSHUFB c40, v2;       \
-	PSHUFB c40, v3;       \
-	PADDQ  m2, v0;        \
-	PADDQ  m3, v1;        \
-	PADDQ  v2, v0;        \
-	PADDQ  v3, v1;        \
-	PXOR   v0, v6;        \
-	PXOR   v1, v7;        \
-	PSHUFB c48, v6;       \
-	PSHUFB c48, v7;       \
-	PADDQ  v6, v4;        \
-	PADDQ  v7, v5;        \
-	PXOR   v4, v2;        \
-	PXOR   v5, v3;        \
-	MOVOU  v2, t0;        \
-	PADDQ  v2, t0;        \
-	PSRLQ  $63, v2;       \
-	PXOR   t0, v2;        \
-	MOVOU  v3, t0;        \
-	PADDQ  v3, t0;        \
-	PSRLQ  $63, v3;       \
-	PXOR   t0, v3
-
-#define LOAD_MSG(m0, m1, m2, m3, src, i0, i1, i2, i3, i4, i5, i6, i7) \
-	MOVQ   i0*8(src), m0;     \
-	PINSRQ $1, i1*8(src), m0; \
-	MOVQ   i2*8(src), m1;     \
-	PINSRQ $1, i3*8(src), m1; \
-	MOVQ   i4*8(src), m2;     \
-	PINSRQ $1, i5*8(src), m2; \
-	MOVQ   i6*8(src), m3;     \
-	PINSRQ $1, i7*8(src), m3
-
 // func hashBlocksSSE4(h *[8]uint64, c *[2]uint64, flag uint64, blocks []byte)
-TEXT ·hashBlocksSSE4(SB), 4, $288-48 // frame size = 272 + 16 byte alignment
-	MOVQ h+0(FP), AX
-	MOVQ c+8(FP), BX
-	MOVQ flag+16(FP), CX
-	MOVQ blocks_base+24(FP), SI
-	MOVQ blocks_len+32(FP), DI
-
-	MOVQ SP, R10
-	ADDQ $15, R10
-	ANDQ $~15, R10
-
-	MOVOU ·iv3<>(SB), X0
-	MOVO  X0, 0(R10)
-	XORQ  CX, 0(R10)     // 0(R10) = ·iv3 ^ (CX || 0)
-
-	MOVOU ·c40<>(SB), X13
-	MOVOU ·c48<>(SB), X14
-
-	MOVOU 0(AX), X12
+// Requires: SSE2, SSE4.1, SSSE3
+TEXT ·hashBlocksSSE4(SB), NOSPLIT, $288-48
+	MOVQ  h+0(FP), AX
+	MOVQ  c+8(FP), BX
+	MOVQ  flag+16(FP), CX
+	MOVQ  blocks_base+24(FP), SI
+	MOVQ  blocks_len+32(FP), DI
+	MOVQ  SP, R10
+	ADDQ  $0x0f, R10
+	ANDQ  $-16, R10
+	MOVOU ·iv3<>+0(SB), X0
+	MOVO  X0, (R10)
+	XORQ  CX, (R10)
+	MOVOU ·c40<>+0(SB), X13
+	MOVOU ·c48<>+0(SB), X14
+	MOVOU (AX), X12
 	MOVOU 16(AX), X15
-
-	MOVQ 0(BX), R8
-	MOVQ 8(BX), R9
+	MOVQ  (BX), R8
+	MOVQ  8(BX), R9
 
 loop:
-	ADDQ $128, R8
-	CMPQ R8, $128
+	ADDQ $0x80, R8
+	CMPQ R8, $0x80
 	JGE  noinc
 	INCQ R9
 
 noinc:
-	MOVQ R8, X8
-	PINSRQ $1, R9, X8
-
-	MOVO X12, X0
-	MOVO X15, X1
-	MOVOU 32(AX), X2
-	MOVOU 48(AX), X3
-	MOVOU ·iv0<>(SB), X4
-	MOVOU ·iv1<>(SB), X5
-	MOVOU ·iv2<>(SB), X6
-
-	PXOR X8, X6
-	MOVO 0(R10), X7
-
-	LOAD_MSG(X8, X9, X10, X11, SI, 0, 2, 4, 6, 1, 3, 5, 7)
-	MOVO X8, 16(R10)
-	MOVO X9, 32(R10)
-	MOVO X10, 48(R10)
-	MOVO X11, 64(R10)
-	HALF_ROUND(X0, X1, X2, X3, X4, X5, X6, X7, X8, X9, X10, X11, X11, X13, X14)
-	SHUFFLE(X2, X3, X4, X5, X6, X7, X8, X9)
-	LOAD_MSG(X8, X9, X10, X11, SI, 8, 10, 12, 14, 9, 11, 13, 15)
-	MOVO X8, 80(R10)
-	MOVO X9, 96(R10)
-	MOVO X10, 112(R10)
-	MOVO X11, 128(R10)
-	HALF_ROUND(X0, X1, X2, X3, X4, X5, X6, X7, X8, X9, X10, X11, X11, X13, X14)
-	SHUFFLE_INV(X2, X3, X4, X5, X6, X7, X8, X9)
-
-	LOAD_MSG(X8, X9, X10, X11, SI, 14, 4, 9, 13, 10, 8, 15, 6)
-	MOVO X8, 144(R10)
-	MOVO X9, 160(R10)
-	MOVO X10, 176(R10)
-	MOVO X11, 192(R10)
-	HALF_ROUND(X0, X1, X2, X3, X4, X5, X6, X7, X8, X9, X10, X11, X11, X13, X14)
-	SHUFFLE(X2, X3, X4, X5, X6, X7, X8, X9)
-	LOAD_MSG(X8, X9, X10, X11, SI, 1, 0, 11, 5, 12, 2, 7, 3)
-	MOVO X8, 208(R10)
-	MOVO X9, 224(R10)
-	MOVO X10, 240(R10)
-	MOVO X11, 256(R10)
-	HALF_ROUND(X0, X1, X2, X3, X4, X5, X6, X7, X8, X9, X10, X11, X11, X13, X14)
-	SHUFFLE_INV(X2, X3, X4, X5, X6, X7, X8, X9)
-
-	LOAD_MSG(X8, X9, X10, X11, SI, 11, 12, 5, 15, 8, 0, 2, 13)
-	HALF_ROUND(X0, X1, X2, X3, X4, X5, X6, X7, X8, X9, X10, X11, X11, X13, X14)
-	SHUFFLE(X2, X3, X4, X5, X6, X7, X8, X9)
-	LOAD_MSG(X8, X9, X10, X11, SI, 10, 3, 7, 9, 14, 6, 1, 4)
-	HALF_ROUND(X0, X1, X2, X3, X4, X5, X6, X7, X8, X9, X10, X11, X11, X13, X14)
-	SHUFFLE_INV(X2, X3, X4, X5, X6, X7, X8, X9)
-
-	LOAD_MSG(X8, X9, X10, X11, SI, 7, 3, 13, 11, 9, 1, 12, 14)
-	HALF_ROUND(X0, X1, X2, X3, X4, X5, X6, X7, X8, X9, X10, X11, X11, X13, X14)
-	SHUFFLE(X2, X3, X4, X5, X6, X7, X8, X9)
-	LOAD_MSG(X8, X9, X10, X11, SI, 2, 5, 4, 15, 6, 10, 0, 8)
-	HALF_ROUND(X0, X1, X2, X3, X4, X5, X6, X7, X8, X9, X10, X11, X11, X13, X14)
-	SHUFFLE_INV(X2, X3, X4, X5, X6, X7, X8, X9)
-
-	LOAD_MSG(X8, X9, X10, X11, SI, 9, 5, 2, 10, 0, 7, 4, 15)
-	HALF_ROUND(X0, X1, X2, X3, X4, X5, X6, X7, X8, X9, X10, X11, X11, X13, X14)
-	SHUFFLE(X2, X3, X4, X5, X6, X7, X8, X9)
-	LOAD_MSG(X8, X9, X10, X11, SI, 14, 11, 6, 3, 1, 12, 8, 13)
-	HALF_ROUND(X0, X1, X2, X3, X4, X5, X6, X7, X8, X9, X10, X11, X11, X13, X14)
-	SHUFFLE_INV(X2, X3, X4, X5, X6, X7, X8, X9)
-
-	LOAD_MSG(X8, X9, X10, X11, SI, 2, 6, 0, 8, 12, 10, 11, 3)
-	HALF_ROUND(X0, X1, X2, X3, X4, X5, X6, X7, X8, X9, X10, X11, X11, X13, X14)
-	SHUFFLE(X2, X3, X4, X5, X6, X7, X8, X9)
-	LOAD_MSG(X8, X9, X10, X11, SI, 4, 7, 15, 1, 13, 5, 14, 9)
-	HALF_ROUND(X0, X1, X2, X3, X4, X5, X6, X7, X8, X9, X10, X11, X11, X13, X14)
-	SHUFFLE_INV(X2, X3, X4, X5, X6, X7, X8, X9)
-
-	LOAD_MSG(X8, X9, X10, X11, SI, 12, 1, 14, 4, 5, 15, 13, 10)
-	HALF_ROUND(X0, X1, X2, X3, X4, X5, X6, X7, X8, X9, X10, X11, X11, X13, X14)
-	SHUFFLE(X2, X3, X4, X5, X6, X7, X8, X9)
-	LOAD_MSG(X8, X9, X10, X11, SI, 0, 6, 9, 8, 7, 3, 2, 11)
-	HALF_ROUND(X0, X1, X2, X3, X4, X5, X6, X7, X8, X9, X10, X11, X11, X13, X14)
-	SHUFFLE_INV(X2, X3, X4, X5, X6, X7, X8, X9)
-
-	LOAD_MSG(X8, X9, X10, X11, SI, 13, 7, 12, 3, 11, 14, 1, 9)
-	HALF_ROUND(X0, X1, X2, X3, X4, X5, X6, X7, X8, X9, X10, X11, X11, X13, X14)
-	SHUFFLE(X2, X3, X4, X5, X6, X7, X8, X9)
-	LOAD_MSG(X8, X9, X10, X11, SI, 5, 15, 8, 2, 0, 4, 6, 10)
-	HALF_ROUND(X0, X1, X2, X3, X4, X5, X6, X7, X8, X9, X10, X11, X11, X13, X14)
-	SHUFFLE_INV(X2, X3, X4, X5, X6, X7, X8, X9)
-
-	LOAD_MSG(X8, X9, X10, X11, SI, 6, 14, 11, 0, 15, 9, 3, 8)
-	HALF_ROUND(X0, X1, X2, X3, X4, X5, X6, X7, X8, X9, X10, X11, X11, X13, X14)
-	SHUFFLE(X2, X3, X4, X5, X6, X7, X8, X9)
-	LOAD_MSG(X8, X9, X10, X11, SI, 12, 13, 1, 10, 2, 7, 4, 5)
-	HALF_ROUND(X0, X1, X2, X3, X4, X5, X6, X7, X8, X9, X10, X11, X11, X13, X14)
-	SHUFFLE_INV(X2, X3, X4, X5, X6, X7, X8, X9)
-
-	LOAD_MSG(X8, X9, X10, X11, SI, 10, 8, 7, 1, 2, 4, 6, 5)
-	HALF_ROUND(X0, X1, X2, X3, X4, X5, X6, X7, X8, X9, X10, X11, X11, X13, X14)
-	SHUFFLE(X2, X3, X4, X5, X6, X7, X8, X9)
-	LOAD_MSG(X8, X9, X10, X11, SI, 15, 9, 3, 13, 11, 14, 12, 0)
-	HALF_ROUND(X0, X1, X2, X3, X4, X5, X6, X7, X8, X9, X10, X11, X11, X13, X14)
-	SHUFFLE_INV(X2, X3, X4, X5, X6, X7, X8, X9)
-
-	HALF_ROUND(X0, X1, X2, X3, X4, X5, X6, X7, 16(R10), 32(R10), 48(R10), 64(R10), X11, X13, X14)
-	SHUFFLE(X2, X3, X4, X5, X6, X7, X8, X9)
-	HALF_ROUND(X0, X1, X2, X3, X4, X5, X6, X7, 80(R10), 96(R10), 112(R10), 128(R10), X11, X13, X14)
-	SHUFFLE_INV(X2, X3, X4, X5, X6, X7, X8, X9)
+	MOVQ       R8, X8
+	PINSRQ     $0x01, R9, X8
+	MOVO       X12, X0
+	MOVO       X15, X1
+	MOVOU      32(AX), X2
+	MOVOU      48(AX), X3
+	MOVOU      ·iv0<>+0(SB), X4
+	MOVOU      ·iv1<>+0(SB), X5
+	MOVOU      ·iv2<>+0(SB), X6
+	PXOR       X8, X6
+	MOVO       (R10), X7
+	MOVQ       (SI), X8
+	PINSRQ     $0x01, 16(SI), X8
+	MOVQ       32(SI), X9
+	PINSRQ     $0x01, 48(SI), X9
+	MOVQ       8(SI), X10
+	PINSRQ     $0x01, 24(SI), X10
+	MOVQ       40(SI), X11
+	PINSRQ     $0x01, 56(SI), X11
+	MOVO       X8, 16(R10)
+	MOVO       X9, 32(R10)
+	MOVO       X10, 48(R10)
+	MOVO       X11, 64(R10)
+	PADDQ      X8, X0
+	PADDQ      X9, X1
+	PADDQ      X2, X0
+	PADDQ      X3, X1
+	PXOR       X0, X6
+	PXOR       X1, X7
+	PSHUFD     $0xb1, X6, X6
+	PSHUFD     $0xb1, X7, X7
+	PADDQ      X6, X4
+	PADDQ      X7, X5
+	PXOR       X4, X2
+	PXOR       X5, X3
+	PSHUFB     X13, X2
+	PSHUFB     X13, X3
+	PADDQ      X10, X0
+	PADDQ      X11, X1
+	PADDQ      X2, X0
+	PADDQ      X3, X1
+	PXOR       X0, X6
+	PXOR       X1, X7
+	PSHUFB     X14, X6
+	PSHUFB     X14, X7
+	PADDQ      X6, X4
+	PADDQ      X7, X5
+	PXOR       X4, X2
+	PXOR       X5, X3
+	MOVOU      X2, X11
+	PADDQ      X2, X11
+	PSRLQ      $0x3f, X2
+	PXOR       X11, X2
+	MOVOU      X3, X11
+	PADDQ      X3, X11
+	PSRLQ      $0x3f, X3
+	PXOR       X11, X3
+	MOVO       X4, X8
+	MOVO       X5, X4
+	MOVO       X8, X5
+	MOVO       X6, X8
+	PUNPCKLQDQ X6, X9
+	PUNPCKHQDQ X7, X6
+	PUNPCKHQDQ X9, X6
+	PUNPCKLQDQ X7, X9
+	MOVO       X8, X7
+	MOVO       X2, X8
+	PUNPCKHQDQ X9, X7
+	PUNPCKLQDQ X3, X9
+	PUNPCKHQDQ X9, X2
+	PUNPCKLQDQ X8, X9
+	PUNPCKHQDQ X9, X3
+	MOVQ       64(SI), X8
+	PINSRQ     $0x01, 80(SI), X8
+	MOVQ       96(SI), X9
+	PINSRQ     $0x01, 112(SI), X9
+	MOVQ       72(SI), X10
+	PINSRQ     $0x01, 88(SI), X10
+	MOVQ       104(SI), X11
+	PINSRQ     $0x01, 120(SI), X11
+	MOVO       X8, 80(R10)
+	MOVO       X9, 96(R10)
+	MOVO       X10, 112(R10)
+	MOVO       X11, 128(R10)
+	PADDQ      X8, X0
+	PADDQ      X9, X1
+	PADDQ      X2, X0
+	PADDQ      X3, X1
+	PXOR       X0, X6
+	PXOR       X1, X7
+	PSHUFD     $0xb1, X6, X6
+	PSHUFD     $0xb1, X7, X7
+	PADDQ      X6, X4
+	PADDQ      X7, X5
+	PXOR       X4, X2
+	PXOR       X5, X3
+	PSHUFB     X13, X2
+	PSHUFB     X13, X3
+	PADDQ      X10, X0
+	PADDQ      X11, X1
+	PADDQ      X2, X0
+	PADDQ      X3, X1
+	PXOR       X0, X6
+	PXOR       X1, X7
+	PSHUFB     X14, X6
+	PSHUFB     X14, X7
+	PADDQ      X6, X4
+	PADDQ      X7, X5
+	PXOR       X4, X2
+	PXOR       X5, X3
+	MOVOU      X2, X11
+	PADDQ      X2, X11
+	PSRLQ      $0x3f, X2
+	PXOR       X11, X2
+	MOVOU      X3, X11
+	PADDQ      X3, X11
+	PSRLQ      $0x3f, X3
+	PXOR       X11, X3
+	MOVO       X4, X8
+	MOVO       X5, X4
+	MOVO       X8, X5
+	MOVO       X2, X8
+	PUNPCKLQDQ X2, X9
+	PUNPCKHQDQ X3, X2
+	PUNPCKHQDQ X9, X2
+	PUNPCKLQDQ X3, X9
+	MOVO       X8, X3
+	MOVO       X6, X8
+	PUNPCKHQDQ X9, X3
+	PUNPCKLQDQ X7, X9
+	PUNPCKHQDQ X9, X6
+	PUNPCKLQDQ X8, X9
+	PUNPCKHQDQ X9, X7
+	MOVQ       112(SI), X8
+	PINSRQ     $0x01, 32(SI), X8
+	MOVQ       72(SI), X9
+	PINSRQ     $0x01, 104(SI), X9
+	MOVQ       80(SI), X10
+	PINSRQ     $0x01, 64(SI), X10
+	MOVQ       120(SI), X11
+	PINSRQ     $0x01, 48(SI), X11
+	MOVO       X8, 144(R10)
+	MOVO       X9, 160(R10)
+	MOVO       X10, 176(R10)
+	MOVO       X11, 192(R10)
+	PADDQ      X8, X0
+	PADDQ      X9, X1
+	PADDQ      X2, X0
+	PADDQ      X3, X1
+	PXOR       X0, X6
+	PXOR       X1, X7
+	PSHUFD     $0xb1, X6, X6
+	PSHUFD     $0xb1, X7, X7
+	PADDQ      X6, X4
+	PADDQ      X7, X5
+	PXOR       X4, X2
+	PXOR       X5, X3
+	PSHUFB     X13, X2
+	PSHUFB     X13, X3
+	PADDQ      X10, X0
+	PADDQ      X11, X1
+	PADDQ      X2, X0
+	PADDQ      X3, X1
+	PXOR       X0, X6
+	PXOR       X1, X7
+	PSHUFB     X14, X6
+	PSHUFB     X14, X7
+	PADDQ      X6, X4
+	PADDQ      X7, X5
+	PXOR       X4, X2
+	PXOR       X5, X3
+	MOVOU      X2, X11
+	PADDQ      X2, X11
+	PSRLQ      $0x3f, X2
+	PXOR       X11, X2
+	MOVOU      X3, X11
+	PADDQ      X3, X11
+	PSRLQ      $0x3f, X3
+	PXOR       X11, X3
+	MOVO       X4, X8
+	MOVO       X5, X4
+	MOVO       X8, X5
+	MOVO       X6, X8
+	PUNPCKLQDQ X6, X9
+	PUNPCKHQDQ X7, X6
+	PUNPCKHQDQ X9, X6
+	PUNPCKLQDQ X7, X9
+	MOVO       X8, X7
+	MOVO       X2, X8
+	PUNPCKHQDQ X9, X7
+	PUNPCKLQDQ X3, X9
+	PUNPCKHQDQ X9, X2
+	PUNPCKLQDQ X8, X9
+	PUNPCKHQDQ X9, X3
+	MOVQ       8(SI), X8
+	PINSRQ     $0x01, (SI), X8
+	MOVQ       88(SI), X9
+	PINSRQ     $0x01, 40(SI), X9
+	MOVQ       96(SI), X10
+	PINSRQ     $0x01, 16(SI), X10
+	MOVQ       56(SI), X11
+	PINSRQ     $0x01, 24(SI), X11
+	MOVO       X8, 208(R10)
+	MOVO       X9, 224(R10)
+	MOVO       X10, 240(R10)
+	MOVO       X11, 256(R10)
+	PADDQ      X8, X0
+	PADDQ      X9, X1
+	PADDQ      X2, X0
+	PADDQ      X3, X1
+	PXOR       X0, X6
+	PXOR       X1, X7
+	PSHUFD     $0xb1, X6, X6
+	PSHUFD     $0xb1, X7, X7
+	PADDQ      X6, X4
+	PADDQ      X7, X5
+	PXOR       X4, X2
+	PXOR       X5, X3
+	PSHUFB     X13, X2
+	PSHUFB     X13, X3
+	PADDQ      X10, X0
+	PADDQ      X11, X1
+	PADDQ      X2, X0
+	PADDQ      X3, X1
+	PXOR       X0, X6
+	PXOR       X1, X7
+	PSHUFB     X14, X6
+	PSHUFB     X14, X7
+	PADDQ      X6, X4
+	PADDQ      X7, X5
+	PXOR       X4, X2
+	PXOR       X5, X3
+	MOVOU      X2, X11
+	PADDQ      X2, X11
+	PSRLQ      $0x3f, X2
+	PXOR       X11, X2
+	MOVOU      X3, X11
+	PADDQ      X3, X11
+	PSRLQ      $0x3f, X3
+	PXOR       X11, X3
+	MOVO       X4, X8
+	MOVO       X5, X4
+	MOVO       X8, X5
+	MOVO       X2, X8
+	PUNPCKLQDQ X2, X9
+	PUNPCKHQDQ X3, X2
+	PUNPCKHQDQ X9, X2
+	PUNPCKLQDQ X3, X9
+	MOVO       X8, X3
+	MOVO       X6, X8
+	PUNPCKHQDQ X9, X3
+	PUNPCKLQDQ X7, X9
+	PUNPCKHQDQ X9, X6
+	PUNPCKLQDQ X8, X9
+	PUNPCKHQDQ X9, X7
+	MOVQ       88(SI), X8
+	PINSRQ     $0x01, 96(SI), X8
+	MOVQ       40(SI), X9
+	PINSRQ     $0x01, 120(SI), X9
+	MOVQ       64(SI), X10
+	PINSRQ     $0x01, (SI), X10
+	MOVQ       16(SI), X11
+	PINSRQ     $0x01, 104(SI), X11
+	PADDQ      X8, X0
+	PADDQ      X9, X1
+	PADDQ      X2, X0
+	PADDQ      X3, X1
+	PXOR       X0, X6
+	PXOR       X1, X7
+	PSHUFD     $0xb1, X6, X6
+	PSHUFD     $0xb1, X7, X7
+	PADDQ      X6, X4
+	PADDQ      X7, X5
+	PXOR       X4, X2
+	PXOR       X5, X3
+	PSHUFB     X13, X2
+	PSHUFB     X13, X3
+	PADDQ      X10, X0
+	PADDQ      X11, X1
+	PADDQ      X2, X0
+	PADDQ      X3, X1
+	PXOR       X0, X6
+	PXOR       X1, X7
+	PSHUFB     X14, X6
+	PSHUFB     X14, X7
+	PADDQ      X6, X4
+	PADDQ      X7, X5
+	PXOR       X4, X2
+	PXOR       X5, X3
+	MOVOU      X2, X11
+	PADDQ      X2, X11
+	PSRLQ      $0x3f, X2
+	PXOR       X11, X2
+	MOVOU      X3, X11
+	PADDQ      X3, X11
+	PSRLQ      $0x3f, X3
+	PXOR       X11, X3
+	MOVO       X4, X8
+	MOVO       X5, X4
+	MOVO       X8, X5
+	MOVO       X6, X8
+	PUNPCKLQDQ X6, X9
+	PUNPCKHQDQ X7, X6
+	PUNPCKHQDQ X9, X6
+	PUNPCKLQDQ X7, X9
+	MOVO       X8, X7
+	MOVO       X2, X8
+	PUNPCKHQDQ X9, X7
+	PUNPCKLQDQ X3, X9
+	PUNPCKHQDQ X9, X2
+	PUNPCKLQDQ X8, X9
+	PUNPCKHQDQ X9, X3
+	MOVQ       80(SI), X8
+	PINSRQ     $0x01, 24(SI), X8
+	MOVQ       56(SI), X9
+	PINSRQ     $0x01, 72(SI), X9
+	MOVQ       112(SI), X10
+	PINSRQ     $0x01, 48(SI), X10
+	MOVQ       8(SI), X11
+	PINSRQ     $0x01, 32(SI), X11
+	PADDQ      X8, X0
+	PADDQ      X9, X1
+	PADDQ      X2, X0
+	PADDQ      X3, X1
+	PXOR       X0, X6
+	PXOR       X1, X7
+	PSHUFD     $0xb1, X6, X6
+	PSHUFD     $0xb1, X7, X7
+	PADDQ      X6, X4
+	PADDQ      X7, X5
+	PXOR       X4, X2
+	PXOR       X5, X3
+	PSHUFB     X13, X2
+	PSHUFB     X13, X3
+	PADDQ      X10, X0
+	PADDQ      X11, X1
+	PADDQ      X2, X0
+	PADDQ      X3, X1
+	PXOR       X0, X6
+	PXOR       X1, X7
+	PSHUFB     X14, X6
+	PSHUFB     X14, X7
+	PADDQ      X6, X4
+	PADDQ      X7, X5
+	PXOR       X4, X2
+	PXOR       X5, X3
+	MOVOU      X2, X11
+	PADDQ      X2, X11
+	PSRLQ      $0x3f, X2
+	PXOR       X11, X2
+	MOVOU      X3, X11
+	PADDQ      X3, X11
+	PSRLQ      $0x3f, X3
+	PXOR       X11, X3
+	MOVO       X4, X8
+	MOVO       X5, X4
+	MOVO       X8, X5
+	MOVO       X2, X8
+	PUNPCKLQDQ X2, X9
+	PUNPCKHQDQ X3, X2
+	PUNPCKHQDQ X9, X2
+	PUNPCKLQDQ X3, X9
+	MOVO       X8, X3
+	MOVO       X6, X8
+	PUNPCKHQDQ X9, X3
+	PUNPCKLQDQ X7, X9
+	PUNPCKHQDQ X9, X6
+	PUNPCKLQDQ X8, X9
+	PUNPCKHQDQ X9, X7
+	MOVQ       56(SI), X8
+	PINSRQ     $0x01, 24(SI), X8
+	MOVQ       104(SI), X9
+	PINSRQ     $0x01, 88(SI), X9
+	MOVQ       72(SI), X10
+	PINSRQ     $0x01, 8(SI), X10
+	MOVQ       96(SI), X11
+	PINSRQ     $0x01, 112(SI), X11
+	PADDQ      X8, X0
+	PADDQ      X9, X1
+	PADDQ      X2, X0
+	PADDQ      X3, X1
+	PXOR       X0, X6
+	PXOR       X1, X7
+	PSHUFD     $0xb1, X6, X6
+	PSHUFD     $0xb1, X7, X7
+	PADDQ      X6, X4
+	PADDQ      X7, X5
+	PXOR       X4, X2
+	PXOR       X5, X3
+	PSHUFB     X13, X2
+	PSHUFB     X13, X3
+	PADDQ      X10, X0
+	PADDQ      X11, X1
+	PADDQ      X2, X0
+	PADDQ      X3, X1
+	PXOR       X0, X6
+	PXOR       X1, X7
+	PSHUFB     X14, X6
+	PSHUFB     X14, X7
+	PADDQ      X6, X4
+	PADDQ      X7, X5
+	PXOR       X4, X2
+	PXOR       X5, X3
+	MOVOU      X2, X11
+	PADDQ      X2, X11
+	PSRLQ      $0x3f, X2
+	PXOR       X11, X2
+	MOVOU      X3, X11
+	PADDQ      X3, X11
+	PSRLQ      $0x3f, X3
+	PXOR       X11, X3
+	MOVO       X4, X8
+	MOVO       X5, X4
+	MOVO       X8, X5
+	MOVO       X6, X8
+	PUNPCKLQDQ X6, X9
+	PUNPCKHQDQ X7, X6
+	PUNPCKHQDQ X9, X6
+	PUNPCKLQDQ X7, X9
+	MOVO       X8, X7
+	MOVO       X2, X8
+	PUNPCKHQDQ X9, X7
+	PUNPCKLQDQ X3, X9
+	PUNPCKHQDQ X9, X2
+	PUNPCKLQDQ X8, X9
+	PUNPCKHQDQ X9, X3
+	MOVQ       16(SI), X8
+	PINSRQ     $0x01, 40(SI), X8
+	MOVQ       32(SI), X9
+	PINSRQ     $0x01, 120(SI), X9
+	MOVQ       48(SI), X10
+	PINSRQ     $0x01, 80(SI), X10
+	MOVQ       (SI), X11
+	PINSRQ     $0x01, 64(SI), X11
+	PADDQ      X8, X0
+	PADDQ      X9, X1
+	PADDQ      X2, X0
+	PADDQ      X3, X1
+	PXOR       X0, X6
+	PXOR       X1, X7
+	PSHUFD     $0xb1, X6, X6
+	PSHUFD     $0xb1, X7, X7
+	PADDQ      X6, X4
+	PADDQ      X7, X5
+	PXOR       X4, X2
+	PXOR       X5, X3
+	PSHUFB     X13, X2
+	PSHUFB     X13, X3
+	PADDQ      X10, X0
+	PADDQ      X11, X1
+	PADDQ      X2, X0
+	PADDQ      X3, X1
+	PXOR       X0, X6
+	PXOR       X1, X7
+	PSHUFB     X14, X6
+	PSHUFB     X14, X7
+	PADDQ      X6, X4
+	PADDQ      X7, X5
+	PXOR       X4, X2
+	PXOR       X5, X3
+	MOVOU      X2, X11
+	PADDQ      X2, X11
+	PSRLQ      $0x3f, X2
+	PXOR       X11, X2
+	MOVOU      X3, X11
+	PADDQ      X3, X11
+	PSRLQ      $0x3f, X3
+	PXOR       X11, X3
+	MOVO       X4, X8
+	MOVO       X5, X4
+	MOVO       X8, X5
+	MOVO       X2, X8
+	PUNPCKLQDQ X2, X9
+	PUNPCKHQDQ X3, X2
+	PUNPCKHQDQ X9, X2
+	PUNPCKLQDQ X3, X9
+	MOVO       X8, X3
+	MOVO       X6, X8
+	PUNPCKHQDQ X9, X3
+	PUNPCKLQDQ X7, X9
+	PUNPCKHQDQ X9, X6
+	PUNPCKLQDQ X8, X9
+	PUNPCKHQDQ X9, X7
+	MOVQ       72(SI), X8
+	PINSRQ     $0x01, 40(SI), X8
+	MOVQ       16(SI), X9
+	PINSRQ     $0x01, 80(SI), X9
+	MOVQ       (SI), X10
+	PINSRQ     $0x01, 56(SI), X10
+	MOVQ       32(SI), X11
+	PINSRQ     $0x01, 120(SI), X11
+	PADDQ      X8, X0
+	PADDQ      X9, X1
+	PADDQ      X2, X0
+	PADDQ      X3, X1
+	PXOR       X0, X6
+	PXOR       X1, X7
+	PSHUFD     $0xb1, X6, X6
+	PSHUFD     $0xb1, X7, X7
+	PADDQ      X6, X4
+	PADDQ      X7, X5
+	PXOR       X4, X2
+	PXOR       X5, X3
+	PSHUFB     X13, X2
+	PSHUFB     X13, X3
+	PADDQ      X10, X0
+	PADDQ      X11, X1
+	PADDQ      X2, X0
+	PADDQ      X3, X1
+	PXOR       X0, X6
+	PXOR       X1, X7
+	PSHUFB     X14, X6
+	PSHUFB     X14, X7
+	PADDQ      X6, X4
+	PADDQ      X7, X5
+	PXOR       X4, X2
+	PXOR       X5, X3
+	MOVOU      X2, X11
+	PADDQ      X2, X11
+	PSRLQ      $0x3f, X2
+	PXOR       X11, X2
+	MOVOU      X3, X11
+	PADDQ      X3, X11
+	PSRLQ      $0x3f, X3
+	PXOR       X11, X3
+	MOVO       X4, X8
+	MOVO       X5, X4
+	MOVO       X8, X5
+	MOVO       X6, X8
+	PUNPCKLQDQ X6, X9
+	PUNPCKHQDQ X7, X6
+	PUNPCKHQDQ X9, X6
+	PUNPCKLQDQ X7, X9
+	MOVO       X8, X7
+	MOVO       X2, X8
+	PUNPCKHQDQ X9, X7
+	PUNPCKLQDQ X3, X9
+	PUNPCKHQDQ X9, X2
+	PUNPCKLQDQ X8, X9
+	PUNPCKHQDQ X9, X3
+	MOVQ       112(SI), X8
+	PINSRQ     $0x01, 88(SI), X8
+	MOVQ       48(SI), X9
+	PINSRQ     $0x01, 24(SI), X9
+	MOVQ       8(SI), X10
+	PINSRQ     $0x01, 96(SI), X10
+	MOVQ       64(SI), X11
+	PINSRQ     $0x01, 104(SI), X11
+	PADDQ      X8, X0
+	PADDQ      X9, X1
+	PADDQ      X2, X0
+	PADDQ      X3, X1
+	PXOR       X0, X6
+	PXOR       X1, X7
+	PSHUFD     $0xb1, X6, X6
+	PSHUFD     $0xb1, X7, X7
+	PADDQ      X6, X4
+	PADDQ      X7, X5
+	PXOR       X4, X2
+	PXOR       X5, X3
+	PSHUFB     X13, X2
+	PSHUFB     X13, X3
+	PADDQ      X10, X0
+	PADDQ      X11, X1
+	PADDQ      X2, X0
+	PADDQ      X3, X1
+	PXOR       X0, X6
+	PXOR       X1, X7
+	PSHUFB     X14, X6
+	PSHUFB     X14, X7
+	PADDQ      X6, X4
+	PADDQ      X7, X5
+	PXOR       X4, X2
+	PXOR       X5, X3
+	MOVOU      X2, X11
+	PADDQ      X2, X11
+	PSRLQ      $0x3f, X2
+	PXOR       X11, X2
+	MOVOU      X3, X11
+	PADDQ      X3, X11
+	PSRLQ      $0x3f, X3
+	PXOR       X11, X3
+	MOVO       X4, X8
+	MOVO       X5, X4
+	MOVO       X8, X5
+	MOVO       X2, X8
+	PUNPCKLQDQ X2, X9
+	PUNPCKHQDQ X3, X2
+	PUNPCKHQDQ X9, X2
+	PUNPCKLQDQ X3, X9
+	MOVO       X8, X3
+	MOVO       X6, X8
+	PUNPCKHQDQ X9, X3
+	PUNPCKLQDQ X7, X9
+	PUNPCKHQDQ X9, X6
+	PUNPCKLQDQ X8, X9
+	PUNPCKHQDQ X9, X7
+	MOVQ       16(SI), X8
+	PINSRQ     $0x01, 48(SI), X8
+	MOVQ       (SI), X9
+	PINSRQ     $0x01, 64(SI), X9
+	MOVQ       96(SI), X10
+	PINSRQ     $0x01, 80(SI), X10
+	MOVQ       88(SI), X11
+	PINSRQ     $0x01, 24(SI), X11
+	PADDQ      X8, X0
+	PADDQ      X9, X1
+	PADDQ      X2, X0
+	PADDQ      X3, X1
+	PXOR       X0, X6
+	PXOR       X1, X7
+	PSHUFD     $0xb1, X6, X6
+	PSHUFD     $0xb1, X7, X7
+	PADDQ      X6, X4
+	PADDQ      X7, X5
+	PXOR       X4, X2
+	PXOR       X5, X3
+	PSHUFB     X13, X2
+	PSHUFB     X13, X3
+	PADDQ      X10, X0
+	PADDQ      X11, X1
+	PADDQ      X2, X0
+	PADDQ      X3, X1
+	PXOR       X0, X6
+	PXOR       X1, X7
+	PSHUFB     X14, X6
+	PSHUFB     X14, X7
+	PADDQ      X6, X4
+	PADDQ      X7, X5
+	PXOR       X4, X2
+	PXOR       X5, X3
+	MOVOU      X2, X11
+	PADDQ      X2, X11
+	PSRLQ      $0x3f, X2
+	PXOR       X11, X2
+	MOVOU      X3, X11
+	PADDQ      X3, X11
+	PSRLQ      $0x3f, X3
+	PXOR       X11, X3
+	MOVO       X4, X8
+	MOVO       X5, X4
+	MOVO       X8, X5
+	MOVO       X6, X8
+	PUNPCKLQDQ X6, X9
+	PUNPCKHQDQ X7, X6
+	PUNPCKHQDQ X9, X6
+	PUNPCKLQDQ X7, X9
+	MOVO       X8, X7
+	MOVO       X2, X8
+	PUNPCKHQDQ X9, X7
+	PUNPCKLQDQ X3, X9
+	PUNPCKHQDQ X9, X2
+	PUNPCKLQDQ X8, X9
+	PUNPCKHQDQ X9, X3
+	MOVQ       32(SI), X8
+	PINSRQ     $0x01, 56(SI), X8
+	MOVQ       120(SI), X9
+	PINSRQ     $0x01, 8(SI), X9
+	MOVQ       104(SI), X10
+	PINSRQ     $0x01, 40(SI), X10
+	MOVQ       112(SI), X11
+	PINSRQ     $0x01, 72(SI), X11
+	PADDQ      X8, X0
+	PADDQ      X9, X1
+	PADDQ      X2, X0
+	PADDQ      X3, X1
+	PXOR       X0, X6
+	PXOR       X1, X7
+	PSHUFD     $0xb1, X6, X6
+	PSHUFD     $0xb1, X7, X7
+	PADDQ      X6, X4
+	PADDQ      X7, X5
+	PXOR       X4, X2
+	PXOR       X5, X3
+	PSHUFB     X13, X2
+	PSHUFB     X13, X3
+	PADDQ      X10, X0
+	PADDQ      X11, X1
+	PADDQ      X2, X0
+	PADDQ      X3, X1
+	PXOR       X0, X6
+	PXOR       X1, X7
+	PSHUFB     X14, X6
+	PSHUFB     X14, X7
+	PADDQ      X6, X4
+	PADDQ      X7, X5
+	PXOR       X4, X2
+	PXOR       X5, X3
+	MOVOU      X2, X11
+	PADDQ      X2, X11
+	PSRLQ      $0x3f, X2
+	PXOR       X11, X2
+	MOVOU      X3, X11
+	PADDQ      X3, X11
+	PSRLQ      $0x3f, X3
+	PXOR       X11, X3
+	MOVO       X4, X8
+	MOVO       X5, X4
+	MOVO       X8, X5
+	MOVO       X2, X8
+	PUNPCKLQDQ X2, X9
+	PUNPCKHQDQ X3, X2
+	PUNPCKHQDQ X9, X2
+	PUNPCKLQDQ X3, X9
+	MOVO       X8, X3
+	MOVO       X6, X8
+	PUNPCKHQDQ X9, X3
+	PUNPCKLQDQ X7, X9
+	PUNPCKHQDQ X9, X6
+	PUNPCKLQDQ X8, X9
+	PUNPCKHQDQ X9, X7
+	MOVQ       96(SI), X8
+	PINSRQ     $0x01, 8(SI), X8
+	MOVQ       112(SI), X9
+	PINSRQ     $0x01, 32(SI), X9
+	MOVQ       40(SI), X10
+	PINSRQ     $0x01, 120(SI), X10
+	MOVQ       104(SI), X11
+	PINSRQ     $0x01, 80(SI), X11
+	PADDQ      X8, X0
+	PADDQ      X9, X1
+	PADDQ      X2, X0
+	PADDQ      X3, X1
+	PXOR       X0, X6
+	PXOR       X1, X7
+	PSHUFD     $0xb1, X6, X6
+	PSHUFD     $0xb1, X7, X7
+	PADDQ      X6, X4
+	PADDQ      X7, X5
+	PXOR       X4, X2
+	PXOR       X5, X3
+	PSHUFB     X13, X2
+	PSHUFB     X13, X3
+	PADDQ      X10, X0
+	PADDQ      X11, X1
+	PADDQ      X2, X0
+	PADDQ      X3, X1
+	PXOR       X0, X6
+	PXOR       X1, X7
+	PSHUFB     X14, X6
+	PSHUFB     X14, X7
+	PADDQ      X6, X4
+	PADDQ      X7, X5
+	PXOR       X4, X2
+	PXOR       X5, X3
+	MOVOU      X2, X11
+	PADDQ      X2, X11
+	PSRLQ      $0x3f, X2
+	PXOR       X11, X2
+	MOVOU      X3, X11
+	PADDQ      X3, X11
+	PSRLQ      $0x3f, X3
+	PXOR       X11, X3
+	MOVO       X4, X8
+	MOVO       X5, X4
+	MOVO       X8, X5
+	MOVO       X6, X8
+	PUNPCKLQDQ X6, X9
+	PUNPCKHQDQ X7, X6
+	PUNPCKHQDQ X9, X6
+	PUNPCKLQDQ X7, X9
+	MOVO       X8, X7
+	MOVO       X2, X8
+	PUNPCKHQDQ X9, X7
+	PUNPCKLQDQ X3, X9
+	PUNPCKHQDQ X9, X2
+	PUNPCKLQDQ X8, X9
+	PUNPCKHQDQ X9, X3
+	MOVQ       (SI), X8
+	PINSRQ     $0x01, 48(SI), X8
+	MOVQ       72(SI), X9
+	PINSRQ     $0x01, 64(SI), X9
+	MOVQ       56(SI), X10
+	PINSRQ     $0x01, 24(SI), X10
+	MOVQ       16(SI), X11
+	PINSRQ     $0x01, 88(SI), X11
+	PADDQ      X8, X0
+	PADDQ      X9, X1
+	PADDQ      X2, X0
+	PADDQ      X3, X1
+	PXOR       X0, X6
+	PXOR       X1, X7
+	PSHUFD     $0xb1, X6, X6
+	PSHUFD     $0xb1, X7, X7
+	PADDQ      X6, X4
+	PADDQ      X7, X5
+	PXOR       X4, X2
+	PXOR       X5, X3
+	PSHUFB     X13, X2
+	PSHUFB     X13, X3
+	PADDQ      X10, X0
+	PADDQ      X11, X1
+	PADDQ      X2, X0
+	PADDQ      X3, X1
+	PXOR       X0, X6
+	PXOR       X1, X7
+	PSHUFB     X14, X6
+	PSHUFB     X14, X7
+	PADDQ      X6, X4
+	PADDQ      X7, X5
+	PXOR       X4, X2
+	PXOR       X5, X3
+	MOVOU      X2, X11
+	PADDQ      X2, X11
+	PSRLQ      $0x3f, X2
+	PXOR       X11, X2
+	MOVOU      X3, X11
+	PADDQ      X3, X11
+	PSRLQ      $0x3f, X3
+	PXOR       X11, X3
+	MOVO       X4, X8
+	MOVO       X5, X4
+	MOVO       X8, X5
+	MOVO       X2, X8
+	PUNPCKLQDQ X2, X9
+	PUNPCKHQDQ X3, X2
+	PUNPCKHQDQ X9, X2
+	PUNPCKLQDQ X3, X9
+	MOVO       X8, X3
+	MOVO       X6, X8
+	PUNPCKHQDQ X9, X3
+	PUNPCKLQDQ X7, X9
+	PUNPCKHQDQ X9, X6
+	PUNPCKLQDQ X8, X9
+	PUNPCKHQDQ X9, X7
+	MOVQ       104(SI), X8
+	PINSRQ     $0x01, 56(SI), X8
+	MOVQ       96(SI), X9
+	PINSRQ     $0x01, 24(SI), X9
+	MOVQ       88(SI), X10
+	PINSRQ     $0x01, 112(SI), X10
+	MOVQ       8(SI), X11
+	PINSRQ     $0x01, 72(SI), X11
+	PADDQ      X8, X0
+	PADDQ      X9, X1
+	PADDQ      X2, X0
+	PADDQ      X3, X1
+	PXOR       X0, X6
+	PXOR       X1, X7
+	PSHUFD     $0xb1, X6, X6
+	PSHUFD     $0xb1, X7, X7
+	PADDQ      X6, X4
+	PADDQ      X7, X5
+	PXOR       X4, X2
+	PXOR       X5, X3
+	PSHUFB     X13, X2
+	PSHUFB     X13, X3
+	PADDQ      X10, X0
+	PADDQ      X11, X1
+	PADDQ      X2, X0
+	PADDQ      X3, X1
+	PXOR       X0, X6
+	PXOR       X1, X7
+	PSHUFB     X14, X6
+	PSHUFB     X14, X7
+	PADDQ      X6, X4
+	PADDQ      X7, X5
+	PXOR       X4, X2
+	PXOR       X5, X3
+	MOVOU      X2, X11
+	PADDQ      X2, X11
+	PSRLQ      $0x3f, X2
+	PXOR       X11, X2
+	MOVOU      X3, X11
+	PADDQ      X3, X11
+	PSRLQ      $0x3f, X3
+	PXOR       X11, X3
+	MOVO       X4, X8
+	MOVO       X5, X4
+	MOVO       X8, X5
+	MOVO       X6, X8
+	PUNPCKLQDQ X6, X9
+	PUNPCKHQDQ X7, X6
+	PUNPCKHQDQ X9, X6
+	PUNPCKLQDQ X7, X9
+	MOVO       X8, X7
+	MOVO       X2, X8
+	PUNPCKHQDQ X9, X7
+	PUNPCKLQDQ X3, X9
+	PUNPCKHQDQ X9, X2
+	PUNPCKLQDQ X8, X9
+	PUNPCKHQDQ X9, X3
+	MOVQ       40(SI), X8
+	PINSRQ     $0x01, 120(SI), X8
+	MOVQ       64(SI), X9
+	PINSRQ     $0x01, 16(SI), X9
+	MOVQ       (SI), X10
+	PINSRQ     $0x01, 32(SI), X10
+	MOVQ       48(SI), X11
+	PINSRQ     $0x01, 80(SI), X11
+	PADDQ      X8, X0
+	PADDQ      X9, X1
+	PADDQ      X2, X0
+	PADDQ      X3, X1
+	PXOR       X0, X6
+	PXOR       X1, X7
+	PSHUFD     $0xb1, X6, X6
+	PSHUFD     $0xb1, X7, X7
+	PADDQ      X6, X4
+	PADDQ      X7, X5
+	PXOR       X4, X2
+	PXOR       X5, X3
+	PSHUFB     X13, X2
+	PSHUFB     X13, X3
+	PADDQ      X10, X0
+	PADDQ      X11, X1
+	PADDQ      X2, X0
+	PADDQ      X3, X1
+	PXOR       X0, X6
+	PXOR       X1, X7
+	PSHUFB     X14, X6
+	PSHUFB     X14, X7
+	PADDQ      X6, X4
+	PADDQ      X7, X5
+	PXOR       X4, X2
+	PXOR       X5, X3
+	MOVOU      X2, X11
+	PADDQ      X2, X11
+	PSRLQ      $0x3f, X2
+	PXOR       X11, X2
+	MOVOU      X3, X11
+	PADDQ      X3, X11
+	PSRLQ      $0x3f, X3
+	PXOR       X11, X3
+	MOVO       X4, X8
+	MOVO       X5, X4
+	MOVO       X8, X5
+	MOVO       X2, X8
+	PUNPCKLQDQ X2, X9
+	PUNPCKHQDQ X3, X2
+	PUNPCKHQDQ X9, X2
+	PUNPCKLQDQ X3, X9
+	MOVO       X8, X3
+	MOVO       X6, X8
+	PUNPCKHQDQ X9, X3
+	PUNPCKLQDQ X7, X9
+	PUNPCKHQDQ X9, X6
+	PUNPCKLQDQ X8, X9
+	PUNPCKHQDQ X9, X7
+	MOVQ       48(SI), X8
+	PINSRQ     $0x01, 112(SI), X8
+	MOVQ       88(SI), X9
+	PINSRQ     $0x01, (SI), X9
+	MOVQ       120(SI), X10
+	PINSRQ     $0x01, 72(SI), X10
+	MOVQ       24(SI), X11
+	PINSRQ     $0x01, 64(SI), X11
+	PADDQ      X8, X0
+	PADDQ      X9, X1
+	PADDQ      X2, X0
+	PADDQ      X3, X1
+	PXOR       X0, X6
+	PXOR       X1, X7
+	PSHUFD     $0xb1, X6, X6
+	PSHUFD     $0xb1, X7, X7
+	PADDQ      X6, X4
+	PADDQ      X7, X5
+	PXOR       X4, X2
+	PXOR       X5, X3
+	PSHUFB     X13, X2
+	PSHUFB     X13, X3
+	PADDQ      X10, X0
+	PADDQ      X11, X1
+	PADDQ      X2, X0
+	PADDQ      X3, X1
+	PXOR       X0, X6
+	PXOR       X1, X7
+	PSHUFB     X14, X6
+	PSHUFB     X14, X7
+	PADDQ      X6, X4
+	PADDQ      X7, X5
+	PXOR       X4, X2
+	PXOR       X5, X3
+	MOVOU      X2, X11
+	PADDQ      X2, X11
+	PSRLQ      $0x3f, X2
+	PXOR       X11, X2
+	MOVOU      X3, X11
+	PADDQ      X3, X11
+	PSRLQ      $0x3f, X3
+	PXOR       X11, X3
+	MOVO       X4, X8
+	MOVO       X5, X4
+	MOVO       X8, X5
+	MOVO       X6, X8
+	PUNPCKLQDQ X6, X9
+	PUNPCKHQDQ X7, X6
+	PUNPCKHQDQ X9, X6
+	PUNPCKLQDQ X7, X9
+	MOVO       X8, X7
+	MOVO       X2, X8
+	PUNPCKHQDQ X9, X7
+	PUNPCKLQDQ X3, X9
+	PUNPCKHQDQ X9, X2
+	PUNPCKLQDQ X8, X9
+	PUNPCKHQDQ X9, X3
+	MOVQ       96(SI), X8
+	PINSRQ     $0x01, 104(SI), X8
+	MOVQ       8(SI), X9
+	PINSRQ     $0x01, 80(SI), X9
+	MOVQ       16(SI), X10
+	PINSRQ     $0x01, 56(SI), X10
+	MOVQ       32(SI), X11
+	PINSRQ     $0x01, 40(SI), X11
+	PADDQ      X8, X0
+	PADDQ      X9, X1
+	PADDQ      X2, X0
+	PADDQ      X3, X1
+	PXOR       X0, X6
+	PXOR       X1, X7
+	PSHUFD     $0xb1, X6, X6
+	PSHUFD     $0xb1, X7, X7
+	PADDQ      X6, X4
+	PADDQ      X7, X5
+	PXOR       X4, X2
+	PXOR       X5, X3
+	PSHUFB     X13, X2
+	PSHUFB     X13, X3
+	PADDQ      X10, X0
+	PADDQ      X11, X1
+	PADDQ      X2, X0
+	PADDQ      X3, X1
+	PXOR       X0, X6
+	PXOR       X1, X7
+	PSHUFB     X14, X6
+	PSHUFB     X14, X7
+	PADDQ      X6, X4
+	PADDQ      X7, X5
+	PXOR       X4, X2
+	PXOR       X5, X3
+	MOVOU      X2, X11
+	PADDQ      X2, X11
+	PSRLQ      $0x3f, X2
+	PXOR       X11, X2
+	MOVOU      X3, X11
+	PADDQ      X3, X11
+	PSRLQ      $0x3f, X3
+	PXOR       X11, X3
+	MOVO       X4, X8
+	MOVO       X5, X4
+	MOVO       X8, X5
+	MOVO       X2, X8
+	PUNPCKLQDQ X2, X9
+	PUNPCKHQDQ X3, X2
+	PUNPCKHQDQ X9, X2
+	PUNPCKLQDQ X3, X9
+	MOVO       X8, X3
+	MOVO       X6, X8
+	PUNPCKHQDQ X9, X3
+	PUNPCKLQDQ X7, X9
+	PUNPCKHQDQ X9, X6
+	PUNPCKLQDQ X8, X9
+	PUNPCKHQDQ X9, X7
+	MOVQ       80(SI), X8
+	PINSRQ     $0x01, 64(SI), X8
+	MOVQ       56(SI), X9
+	PINSRQ     $0x01, 8(SI), X9
+	MOVQ       16(SI), X10
+	PINSRQ     $0x01, 32(SI), X10
+	MOVQ       48(SI), X11
+	PINSRQ     $0x01, 40(SI), X11
+	PADDQ      X8, X0
+	PADDQ      X9, X1
+	PADDQ      X2, X0
+	PADDQ      X3, X1
+	PXOR       X0, X6
+	PXOR       X1, X7
+	PSHUFD     $0xb1, X6, X6
+	PSHUFD     $0xb1, X7, X7
+	PADDQ      X6, X4
+	PADDQ      X7, X5
+	PXOR       X4, X2
+	PXOR       X5, X3
+	PSHUFB     X13, X2
+	PSHUFB     X13, X3
+	PADDQ      X10, X0
+	PADDQ      X11, X1
+	PADDQ      X2, X0
+	PADDQ      X3, X1
+	PXOR       X0, X6
+	PXOR       X1, X7
+	PSHUFB     X14, X6
+	PSHUFB     X14, X7
+	PADDQ      X6, X4
+	PADDQ      X7, X5
+	PXOR       X4, X2
+	PXOR       X5, X3
+	MOVOU      X2, X11
+	PADDQ      X2, X11
+	PSRLQ      $0x3f, X2
+	PXOR       X11, X2
+	MOVOU      X3, X11
+	PADDQ      X3, X11
+	PSRLQ      $0x3f, X3
+	PXOR       X11, X3
+	MOVO       X4, X8
+	MOVO       X5, X4
+	MOVO       X8, X5
+	MOVO       X6, X8
+	PUNPCKLQDQ X6, X9
+	PUNPCKHQDQ X7, X6
+	PUNPCKHQDQ X9, X6
+	PUNPCKLQDQ X7, X9
+	MOVO       X8, X7
+	MOVO       X2, X8
+	PUNPCKHQDQ X9, X7
+	PUNPCKLQDQ X3, X9
+	PUNPCKHQDQ X9, X2
+	PUNPCKLQDQ X8, X9
+	PUNPCKHQDQ X9, X3
+	MOVQ       120(SI), X8
+	PINSRQ     $0x01, 72(SI), X8
+	MOVQ       24(SI), X9
+	PINSRQ     $0x01, 104(SI), X9
+	MOVQ       88(SI), X10
+	PINSRQ     $0x01, 112(SI), X10
+	MOVQ       96(SI), X11
+	PINSRQ     $0x01, (SI), X11
+	PADDQ      X8, X0
+	PADDQ      X9, X1
+	PADDQ      X2, X0
+	PADDQ      X3, X1
+	PXOR       X0, X6
+	PXOR       X1, X7
+	PSHUFD     $0xb1, X6, X6
+	PSHUFD     $0xb1, X7, X7
+	PADDQ      X6, X4
+	PADDQ      X7, X5
+	PXOR       X4, X2
+	PXOR       X5, X3
+	PSHUFB     X13, X2
+	PSHUFB     X13, X3
+	PADDQ      X10, X0
+	PADDQ      X11, X1
+	PADDQ      X2, X0
+	PADDQ      X3, X1
+	PXOR       X0, X6
+	PXOR       X1, X7
+	PSHUFB     X14, X6
+	PSHUFB     X14, X7
+	PADDQ      X6, X4
+	PADDQ      X7, X5
+	PXOR       X4, X2
+	PXOR       X5, X3
+	MOVOU      X2, X11
+	PADDQ      X2, X11
+	PSRLQ      $0x3f, X2
+	PXOR       X11, X2
+	MOVOU      X3, X11
+	PADDQ      X3, X11
+	PSRLQ      $0x3f, X3
+	PXOR       X11, X3
+	MOVO       X4, X8
+	MOVO       X5, X4
+	MOVO       X8, X5
+	MOVO       X2, X8
+	PUNPCKLQDQ X2, X9
+	PUNPCKHQDQ X3, X2
+	PUNPCKHQDQ X9, X2
+	PUNPCKLQDQ X3, X9
+	MOVO       X8, X3
+	MOVO       X6, X8
+	PUNPCKHQDQ X9, X3
+	PUNPCKLQDQ X7, X9
+	PUNPCKHQDQ X9, X6
+	PUNPCKLQDQ X8, X9
+	PUNPCKHQDQ X9, X7
+	PADDQ      16(R10), X0
+	PADDQ      32(R10), X1
+	PADDQ      X2, X0
+	PADDQ      X3, X1
+	PXOR       X0, X6
+	PXOR       X1, X7
+	PSHUFD     $0xb1, X6, X6
+	PSHUFD     $0xb1, X7, X7
+	PADDQ      X6, X4
+	PADDQ      X7, X5
+	PXOR       X4, X2
+	PXOR       X5, X3
+	PSHUFB     X13, X2
+	PSHUFB     X13, X3
+	PADDQ      48(R10), X0
+	PADDQ      64(R10), X1
+	PADDQ      X2, X0
+	PADDQ      X3, X1
+	PXOR       X0, X6
+	PXOR       X1, X7
+	PSHUFB     X14, X6
+	PSHUFB     X14, X7
+	PADDQ      X6, X4
+	PADDQ      X7, X5
+	PXOR       X4, X2
+	PXOR       X5, X3
+	MOVOU      X2, X11
+	PADDQ      X2, X11
+	PSRLQ      $0x3f, X2
+	PXOR       X11, X2
+	MOVOU      X3, X11
+	PADDQ      X3, X11
+	PSRLQ      $0x3f, X3
+	PXOR       X11, X3
+	MOVO       X4, X8
+	MOVO       X5, X4
+	MOVO       X8, X5
+	MOVO       X6, X8
+	PUNPCKLQDQ X6, X9
+	PUNPCKHQDQ X7, X6
+	PUNPCKHQDQ X9, X6
+	PUNPCKLQDQ X7, X9
+	MOVO       X8, X7
+	MOVO       X2, X8
+	PUNPCKHQDQ X9, X7
+	PUNPCKLQDQ X3, X9
+	PUNPCKHQDQ X9, X2
+	PUNPCKLQDQ X8, X9
+	PUNPCKHQDQ X9, X3
+	PADDQ      80(R10), X0
+	PADDQ      96(R10), X1
+	PADDQ      X2, X0
+	PADDQ      X3, X1
+	PXOR       X0, X6
+	PXOR       X1, X7
+	PSHUFD     $0xb1, X6, X6
+	PSHUFD     $0xb1, X7, X7
+	PADDQ      X6, X4
+	PADDQ      X7, X5
+	PXOR       X4, X2
+	PXOR       X5, X3
+	PSHUFB     X13, X2
+	PSHUFB     X13, X3
+	PADDQ      112(R10), X0
+	PADDQ      128(R10), X1
+	PADDQ      X2, X0
+	PADDQ      X3, X1
+	PXOR       X0, X6
+	PXOR       X1, X7
+	PSHUFB     X14, X6
+	PSHUFB     X14, X7
+	PADDQ      X6, X4
+	PADDQ      X7, X5
+	PXOR       X4, X2
+	PXOR       X5, X3
+	MOVOU      X2, X11
+	PADDQ      X2, X11
+	PSRLQ      $0x3f, X2
+	PXOR       X11, X2
+	MOVOU      X3, X11
+	PADDQ      X3, X11
+	PSRLQ      $0x3f, X3
+	PXOR       X11, X3
+	MOVO       X4, X8
+	MOVO       X5, X4
+	MOVO       X8, X5
+	MOVO       X2, X8
+	PUNPCKLQDQ X2, X9
+	PUNPCKHQDQ X3, X2
+	PUNPCKHQDQ X9, X2
+	PUNPCKLQDQ X3, X9
+	MOVO       X8, X3
+	MOVO       X6, X8
+	PUNPCKHQDQ X9, X3
+	PUNPCKLQDQ X7, X9
+	PUNPCKHQDQ X9, X6
+	PUNPCKLQDQ X8, X9
+	PUNPCKHQDQ X9, X7
+	PADDQ      144(R10), X0
+	PADDQ      160(R10), X1
+	PADDQ      X2, X0
+	PADDQ      X3, X1
+	PXOR       X0, X6
+	PXOR       X1, X7
+	PSHUFD     $0xb1, X6, X6
+	PSHUFD     $0xb1, X7, X7
+	PADDQ      X6, X4
+	PADDQ      X7, X5
+	PXOR       X4, X2
+	PXOR       X5, X3
+	PSHUFB     X13, X2
+	PSHUFB     X13, X3
+	PADDQ      176(R10), X0
+	PADDQ      192(R10), X1
+	PADDQ      X2, X0
+	PADDQ      X3, X1
+	PXOR       X0, X6
+	PXOR       X1, X7
+	PSHUFB     X14, X6
+	PSHUFB     X14, X7
+	PADDQ      X6, X4
+	PADDQ      X7, X5
+	PXOR       X4, X2
+	PXOR       X5, X3
+	MOVOU      X2, X11
+	PADDQ      X2, X11
+	PSRLQ      $0x3f, X2
+	PXOR       X11, X2
+	MOVOU      X3, X11
+	PADDQ      X3, X11
+	PSRLQ      $0x3f, X3
+	PXOR       X11, X3
+	MOVO       X4, X8
+	MOVO       X5, X4
+	MOVO       X8, X5
+	MOVO       X6, X8
+	PUNPCKLQDQ X6, X9
+	PUNPCKHQDQ X7, X6
+	PUNPCKHQDQ X9, X6
+	PUNPCKLQDQ X7, X9
+	MOVO       X8, X7
+	MOVO       X2, X8
+	PUNPCKHQDQ X9, X7
+	PUNPCKLQDQ X3, X9
+	PUNPCKHQDQ X9, X2
+	PUNPCKLQDQ X8, X9
+	PUNPCKHQDQ X9, X3
+	PADDQ      208(R10), X0
+	PADDQ      224(R10), X1
+	PADDQ      X2, X0
+	PADDQ      X3, X1
+	PXOR       X0, X6
+	PXOR       X1, X7
+	PSHUFD     $0xb1, X6, X6
+	PSHUFD     $0xb1, X7, X7
+	PADDQ      X6, X4
+	PADDQ      X7, X5
+	PXOR       X4, X2
+	PXOR       X5, X3
+	PSHUFB     X13, X2
+	PSHUFB     X13, X3
+	PADDQ      240(R10), X0
+	PADDQ      256(R10), X1
+	PADDQ      X2, X0
+	PADDQ      X3, X1
+	PXOR       X0, X6
+	PXOR       X1, X7
+	PSHUFB     X14, X6
+	PSHUFB     X14, X7
+	PADDQ      X6, X4
+	PADDQ      X7, X5
+	PXOR       X4, X2
+	PXOR       X5, X3
+	MOVOU      X2, X11
+	PADDQ      X2, X11
+	PSRLQ      $0x3f, X2
+	PXOR       X11, X2
+	MOVOU      X3, X11
+	PADDQ      X3, X11
+	PSRLQ      $0x3f, X3
+	PXOR       X11, X3
+	MOVO       X4, X8
+	MOVO       X5, X4
+	MOVO       X8, X5
+	MOVO       X2, X8
+	PUNPCKLQDQ X2, X9
+	PUNPCKHQDQ X3, X2
+	PUNPCKHQDQ X9, X2
+	PUNPCKLQDQ X3, X9
+	MOVO       X8, X3
+	MOVO       X6, X8
+	PUNPCKHQDQ X9, X3
+	PUNPCKLQDQ X7, X9
+	PUNPCKHQDQ X9, X6
+	PUNPCKLQDQ X8, X9
+	PUNPCKHQDQ X9, X7
+	MOVOU      32(AX), X10
+	MOVOU      48(AX), X11
+	PXOR       X0, X12
+	PXOR       X1, X15
+	PXOR       X2, X10
+	PXOR       X3, X11
+	PXOR       X4, X12
+	PXOR       X5, X15
+	PXOR       X6, X10
+	PXOR       X7, X11
+	MOVOU      X10, 32(AX)
+	MOVOU      X11, 48(AX)
+	LEAQ       128(SI), SI
+	SUBQ       $0x80, DI
+	JNE        loop
+	MOVOU      X12, (AX)
+	MOVOU      X15, 16(AX)
+	MOVQ       R8, (BX)
+	MOVQ       R9, 8(BX)
+	RET
 
-	HALF_ROUND(X0, X1, X2, X3, X4, X5, X6, X7, 144(R10), 160(R10), 176(R10), 192(R10), X11, X13, X14)
-	SHUFFLE(X2, X3, X4, X5, X6, X7, X8, X9)
-	HALF_ROUND(X0, X1, X2, X3, X4, X5, X6, X7, 208(R10), 224(R10), 240(R10), 256(R10), X11, X13, X14)
-	SHUFFLE_INV(X2, X3, X4, X5, X6, X7, X8, X9)
+DATA ·iv3<>+0(SB)/8, $0x1f83d9abfb41bd6b
+DATA ·iv3<>+8(SB)/8, $0x5be0cd19137e2179
+GLOBL ·iv3<>(SB), RODATA|NOPTR, $16
 
-	MOVOU 32(AX), X10
-	MOVOU 48(AX), X11
-	PXOR  X0, X12
-	PXOR  X1, X15
-	PXOR  X2, X10
-	PXOR  X3, X11
-	PXOR  X4, X12
-	PXOR  X5, X15
-	PXOR  X6, X10
-	PXOR  X7, X11
-	MOVOU X10, 32(AX)
-	MOVOU X11, 48(AX)
+DATA ·c40<>+0(SB)/8, $0x0201000706050403
+DATA ·c40<>+8(SB)/8, $0x0a09080f0e0d0c0b
+GLOBL ·c40<>(SB), RODATA|NOPTR, $16
 
-	LEAQ 128(SI), SI
-	SUBQ $128, DI
-	JNE  loop
+DATA ·c48<>+0(SB)/8, $0x0100070605040302
+DATA ·c48<>+8(SB)/8, $0x09080f0e0d0c0b0a
+GLOBL ·c48<>(SB), RODATA|NOPTR, $16
 
-	MOVOU X12, 0(AX)
-	MOVOU X15, 16(AX)
+DATA ·iv0<>+0(SB)/8, $0x6a09e667f3bcc908
+DATA ·iv0<>+8(SB)/8, $0xbb67ae8584caa73b
+GLOBL ·iv0<>(SB), RODATA|NOPTR, $16
 
-	MOVQ R8, 0(BX)
-	MOVQ R9, 8(BX)
+DATA ·iv1<>+0(SB)/8, $0x3c6ef372fe94f82b
+DATA ·iv1<>+8(SB)/8, $0xa54ff53a5f1d36f1
+GLOBL ·iv1<>(SB), RODATA|NOPTR, $16
 
-	RET
+DATA ·iv2<>+0(SB)/8, $0x510e527fade682d1
+DATA ·iv2<>+8(SB)/8, $0x9b05688c2b3e6c1f
+GLOBL ·iv2<>(SB), RODATA|NOPTR, $16
diff --git a/vendor/golang.org/x/crypto/chacha20poly1305/chacha20poly1305_amd64.s b/vendor/golang.org/x/crypto/chacha20poly1305/chacha20poly1305_amd64.s
index 731d2ac6dbc11..fd5ee845f9faa 100644
--- a/vendor/golang.org/x/crypto/chacha20poly1305/chacha20poly1305_amd64.s
+++ b/vendor/golang.org/x/crypto/chacha20poly1305/chacha20poly1305_amd64.s
@@ -1,2715 +1,9762 @@
-// Copyright 2016 The Go Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-// This file was originally from https://golang.org/cl/24717 by Vlad Krasnov of CloudFlare.
+// Code generated by command: go run chacha20poly1305_amd64_asm.go -out ../chacha20poly1305_amd64.s -pkg chacha20poly1305. DO NOT EDIT.
 
 //go:build gc && !purego
 
 #include "textflag.h"
-// General register allocation
-#define oup DI
-#define inp SI
-#define inl BX
-#define adp CX // free to reuse, after we hash the additional data
-#define keyp R8 // free to reuse, when we copy the key to stack
-#define itr2 R9 // general iterator
-#define itr1 CX // general iterator
-#define acc0 R10
-#define acc1 R11
-#define acc2 R12
-#define t0 R13
-#define t1 R14
-#define t2 R15
-#define t3 R8
-// Register and stack allocation for the SSE code
-#define rStore (0*16)(BP)
-#define sStore (1*16)(BP)
-#define state1Store (2*16)(BP)
-#define state2Store (3*16)(BP)
-#define tmpStore (4*16)(BP)
-#define ctr0Store (5*16)(BP)
-#define ctr1Store (6*16)(BP)
-#define ctr2Store (7*16)(BP)
-#define ctr3Store (8*16)(BP)
-#define A0 X0
-#define A1 X1
-#define A2 X2
-#define B0 X3
-#define B1 X4
-#define B2 X5
-#define C0 X6
-#define C1 X7
-#define C2 X8
-#define D0 X9
-#define D1 X10
-#define D2 X11
-#define T0 X12
-#define T1 X13
-#define T2 X14
-#define T3 X15
-#define A3 T0
-#define B3 T1
-#define C3 T2
-#define D3 T3
-// Register and stack allocation for the AVX2 code
-#define rsStoreAVX2 (0*32)(BP)
-#define state1StoreAVX2 (1*32)(BP)
-#define state2StoreAVX2 (2*32)(BP)
-#define ctr0StoreAVX2 (3*32)(BP)
-#define ctr1StoreAVX2 (4*32)(BP)
-#define ctr2StoreAVX2 (5*32)(BP)
-#define ctr3StoreAVX2 (6*32)(BP)
-#define tmpStoreAVX2 (7*32)(BP) // 256 bytes on stack
-#define AA0 Y0
-#define AA1 Y5
-#define AA2 Y6
-#define AA3 Y7
-#define BB0 Y14
-#define BB1 Y9
-#define BB2 Y10
-#define BB3 Y11
-#define CC0 Y12
-#define CC1 Y13
-#define CC2 Y8
-#define CC3 Y15
-#define DD0 Y4
-#define DD1 Y1
-#define DD2 Y2
-#define DD3 Y3
-#define TT0 DD3
-#define TT1 AA3
-#define TT2 BB3
-#define TT3 CC3
-// ChaCha20 constants
-DATA ·chacha20Constants<>+0x00(SB)/4, $0x61707865
-DATA ·chacha20Constants<>+0x04(SB)/4, $0x3320646e
-DATA ·chacha20Constants<>+0x08(SB)/4, $0x79622d32
-DATA ·chacha20Constants<>+0x0c(SB)/4, $0x6b206574
-DATA ·chacha20Constants<>+0x10(SB)/4, $0x61707865
-DATA ·chacha20Constants<>+0x14(SB)/4, $0x3320646e
-DATA ·chacha20Constants<>+0x18(SB)/4, $0x79622d32
-DATA ·chacha20Constants<>+0x1c(SB)/4, $0x6b206574
-// <<< 16 with PSHUFB
-DATA ·rol16<>+0x00(SB)/8, $0x0504070601000302
-DATA ·rol16<>+0x08(SB)/8, $0x0D0C0F0E09080B0A
-DATA ·rol16<>+0x10(SB)/8, $0x0504070601000302
-DATA ·rol16<>+0x18(SB)/8, $0x0D0C0F0E09080B0A
-// <<< 8 with PSHUFB
-DATA ·rol8<>+0x00(SB)/8, $0x0605040702010003
-DATA ·rol8<>+0x08(SB)/8, $0x0E0D0C0F0A09080B
-DATA ·rol8<>+0x10(SB)/8, $0x0605040702010003
-DATA ·rol8<>+0x18(SB)/8, $0x0E0D0C0F0A09080B
-
-DATA ·avx2InitMask<>+0x00(SB)/8, $0x0
-DATA ·avx2InitMask<>+0x08(SB)/8, $0x0
-DATA ·avx2InitMask<>+0x10(SB)/8, $0x1
-DATA ·avx2InitMask<>+0x18(SB)/8, $0x0
-
-DATA ·avx2IncMask<>+0x00(SB)/8, $0x2
-DATA ·avx2IncMask<>+0x08(SB)/8, $0x0
-DATA ·avx2IncMask<>+0x10(SB)/8, $0x2
-DATA ·avx2IncMask<>+0x18(SB)/8, $0x0
-// Poly1305 key clamp
-DATA ·polyClampMask<>+0x00(SB)/8, $0x0FFFFFFC0FFFFFFF
-DATA ·polyClampMask<>+0x08(SB)/8, $0x0FFFFFFC0FFFFFFC
-DATA ·polyClampMask<>+0x10(SB)/8, $0xFFFFFFFFFFFFFFFF
-DATA ·polyClampMask<>+0x18(SB)/8, $0xFFFFFFFFFFFFFFFF
-
-DATA ·sseIncMask<>+0x00(SB)/8, $0x1
-DATA ·sseIncMask<>+0x08(SB)/8, $0x0
-// To load/store the last < 16 bytes in a buffer
-DATA ·andMask<>+0x00(SB)/8, $0x00000000000000ff
-DATA ·andMask<>+0x08(SB)/8, $0x0000000000000000
-DATA ·andMask<>+0x10(SB)/8, $0x000000000000ffff
-DATA ·andMask<>+0x18(SB)/8, $0x0000000000000000
-DATA ·andMask<>+0x20(SB)/8, $0x0000000000ffffff
-DATA ·andMask<>+0x28(SB)/8, $0x0000000000000000
-DATA ·andMask<>+0x30(SB)/8, $0x00000000ffffffff
-DATA ·andMask<>+0x38(SB)/8, $0x0000000000000000
-DATA ·andMask<>+0x40(SB)/8, $0x000000ffffffffff
-DATA ·andMask<>+0x48(SB)/8, $0x0000000000000000
-DATA ·andMask<>+0x50(SB)/8, $0x0000ffffffffffff
-DATA ·andMask<>+0x58(SB)/8, $0x0000000000000000
-DATA ·andMask<>+0x60(SB)/8, $0x00ffffffffffffff
-DATA ·andMask<>+0x68(SB)/8, $0x0000000000000000
-DATA ·andMask<>+0x70(SB)/8, $0xffffffffffffffff
-DATA ·andMask<>+0x78(SB)/8, $0x0000000000000000
-DATA ·andMask<>+0x80(SB)/8, $0xffffffffffffffff
-DATA ·andMask<>+0x88(SB)/8, $0x00000000000000ff
-DATA ·andMask<>+0x90(SB)/8, $0xffffffffffffffff
-DATA ·andMask<>+0x98(SB)/8, $0x000000000000ffff
-DATA ·andMask<>+0xa0(SB)/8, $0xffffffffffffffff
-DATA ·andMask<>+0xa8(SB)/8, $0x0000000000ffffff
-DATA ·andMask<>+0xb0(SB)/8, $0xffffffffffffffff
-DATA ·andMask<>+0xb8(SB)/8, $0x00000000ffffffff
-DATA ·andMask<>+0xc0(SB)/8, $0xffffffffffffffff
-DATA ·andMask<>+0xc8(SB)/8, $0x000000ffffffffff
-DATA ·andMask<>+0xd0(SB)/8, $0xffffffffffffffff
-DATA ·andMask<>+0xd8(SB)/8, $0x0000ffffffffffff
-DATA ·andMask<>+0xe0(SB)/8, $0xffffffffffffffff
-DATA ·andMask<>+0xe8(SB)/8, $0x00ffffffffffffff
-
-GLOBL ·chacha20Constants<>(SB), (NOPTR+RODATA), $32
-GLOBL ·rol16<>(SB), (NOPTR+RODATA), $32
-GLOBL ·rol8<>(SB), (NOPTR+RODATA), $32
-GLOBL ·sseIncMask<>(SB), (NOPTR+RODATA), $16
-GLOBL ·avx2IncMask<>(SB), (NOPTR+RODATA), $32
-GLOBL ·avx2InitMask<>(SB), (NOPTR+RODATA), $32
-GLOBL ·polyClampMask<>(SB), (NOPTR+RODATA), $32
-GLOBL ·andMask<>(SB), (NOPTR+RODATA), $240
-// No PALIGNR in Go ASM yet (but VPALIGNR is present).
-#define shiftB0Left BYTE $0x66; BYTE $0x0f; BYTE $0x3a; BYTE $0x0f; BYTE $0xdb; BYTE $0x04 // PALIGNR $4, X3, X3
-#define shiftB1Left BYTE $0x66; BYTE $0x0f; BYTE $0x3a; BYTE $0x0f; BYTE $0xe4; BYTE $0x04 // PALIGNR $4, X4, X4
-#define shiftB2Left BYTE $0x66; BYTE $0x0f; BYTE $0x3a; BYTE $0x0f; BYTE $0xed; BYTE $0x04 // PALIGNR $4, X5, X5
-#define shiftB3Left BYTE $0x66; BYTE $0x45; BYTE $0x0f; BYTE $0x3a; BYTE $0x0f; BYTE $0xed; BYTE $0x04 // PALIGNR $4, X13, X13
-#define shiftC0Left BYTE $0x66; BYTE $0x0f; BYTE $0x3a; BYTE $0x0f; BYTE $0xf6; BYTE $0x08 // PALIGNR $8, X6, X6
-#define shiftC1Left BYTE $0x66; BYTE $0x0f; BYTE $0x3a; BYTE $0x0f; BYTE $0xff; BYTE $0x08 // PALIGNR $8, X7, X7
-#define shiftC2Left BYTE $0x66; BYTE $0x45; BYTE $0x0f; BYTE $0x3a; BYTE $0x0f; BYTE $0xc0; BYTE $0x08 // PALIGNR $8, X8, X8
-#define shiftC3Left BYTE $0x66; BYTE $0x45; BYTE $0x0f; BYTE $0x3a; BYTE $0x0f; BYTE $0xf6; BYTE $0x08 // PALIGNR $8, X14, X14
-#define shiftD0Left BYTE $0x66; BYTE $0x45; BYTE $0x0f; BYTE $0x3a; BYTE $0x0f; BYTE $0xc9; BYTE $0x0c // PALIGNR $12, X9, X9
-#define shiftD1Left BYTE $0x66; BYTE $0x45; BYTE $0x0f; BYTE $0x3a; BYTE $0x0f; BYTE $0xd2; BYTE $0x0c // PALIGNR $12, X10, X10
-#define shiftD2Left BYTE $0x66; BYTE $0x45; BYTE $0x0f; BYTE $0x3a; BYTE $0x0f; BYTE $0xdb; BYTE $0x0c // PALIGNR $12, X11, X11
-#define shiftD3Left BYTE $0x66; BYTE $0x45; BYTE $0x0f; BYTE $0x3a; BYTE $0x0f; BYTE $0xff; BYTE $0x0c // PALIGNR $12, X15, X15
-#define shiftB0Right BYTE $0x66; BYTE $0x0f; BYTE $0x3a; BYTE $0x0f; BYTE $0xdb; BYTE $0x0c // PALIGNR $12, X3, X3
-#define shiftB1Right BYTE $0x66; BYTE $0x0f; BYTE $0x3a; BYTE $0x0f; BYTE $0xe4; BYTE $0x0c // PALIGNR $12, X4, X4
-#define shiftB2Right BYTE $0x66; BYTE $0x0f; BYTE $0x3a; BYTE $0x0f; BYTE $0xed; BYTE $0x0c // PALIGNR $12, X5, X5
-#define shiftB3Right BYTE $0x66; BYTE $0x45; BYTE $0x0f; BYTE $0x3a; BYTE $0x0f; BYTE $0xed; BYTE $0x0c // PALIGNR $12, X13, X13
-#define shiftC0Right shiftC0Left
-#define shiftC1Right shiftC1Left
-#define shiftC2Right shiftC2Left
-#define shiftC3Right shiftC3Left
-#define shiftD0Right BYTE $0x66; BYTE $0x45; BYTE $0x0f; BYTE $0x3a; BYTE $0x0f; BYTE $0xc9; BYTE $0x04 // PALIGNR $4, X9, X9
-#define shiftD1Right BYTE $0x66; BYTE $0x45; BYTE $0x0f; BYTE $0x3a; BYTE $0x0f; BYTE $0xd2; BYTE $0x04 // PALIGNR $4, X10, X10
-#define shiftD2Right BYTE $0x66; BYTE $0x45; BYTE $0x0f; BYTE $0x3a; BYTE $0x0f; BYTE $0xdb; BYTE $0x04 // PALIGNR $4, X11, X11
-#define shiftD3Right BYTE $0x66; BYTE $0x45; BYTE $0x0f; BYTE $0x3a; BYTE $0x0f; BYTE $0xff; BYTE $0x04 // PALIGNR $4, X15, X15
-
-// Some macros
-
-// ROL rotates the uint32s in register R left by N bits, using temporary T.
-#define ROL(N, R, T) \
-	MOVO R, T; PSLLL $(N), T; PSRLL $(32-(N)), R; PXOR T, R
-
-// ROL16 rotates the uint32s in register R left by 16, using temporary T if needed.
-#ifdef GOAMD64_v2
-#define ROL16(R, T) PSHUFB ·rol16<>(SB), R
-#else
-#define ROL16(R, T) ROL(16, R, T)
-#endif
-
-// ROL8 rotates the uint32s in register R left by 8, using temporary T if needed.
-#ifdef GOAMD64_v2
-#define ROL8(R, T) PSHUFB ·rol8<>(SB), R
-#else
-#define ROL8(R, T) ROL(8, R, T)
-#endif
-
-#define chachaQR(A, B, C, D, T) \
-	PADDD B, A; PXOR A, D; ROL16(D, T) \
-	PADDD D, C; PXOR C, B; MOVO B, T; PSLLL $12, T; PSRLL $20, B; PXOR T, B \
-	PADDD B, A; PXOR A, D; ROL8(D, T) \
-	PADDD D, C; PXOR C, B; MOVO B, T; PSLLL $7, T; PSRLL $25, B; PXOR T, B
-
-#define chachaQR_AVX2(A, B, C, D, T) \
-	VPADDD B, A, A; VPXOR A, D, D; VPSHUFB ·rol16<>(SB), D, D                         \
-	VPADDD D, C, C; VPXOR C, B, B; VPSLLD $12, B, T; VPSRLD $20, B, B; VPXOR T, B, B \
-	VPADDD B, A, A; VPXOR A, D, D; VPSHUFB ·rol8<>(SB), D, D                          \
-	VPADDD D, C, C; VPXOR C, B, B; VPSLLD $7, B, T; VPSRLD $25, B, B; VPXOR T, B, B
-
-#define polyAdd(S) ADDQ S, acc0; ADCQ 8+S, acc1; ADCQ $1, acc2
-#define polyMulStage1 MOVQ (0*8)(BP), AX; MOVQ AX, t2; MULQ acc0; MOVQ AX, t0; MOVQ DX, t1; MOVQ (0*8)(BP), AX; MULQ acc1; IMULQ acc2, t2; ADDQ AX, t1; ADCQ DX, t2
-#define polyMulStage2 MOVQ (1*8)(BP), AX; MOVQ AX, t3; MULQ acc0; ADDQ AX, t1; ADCQ $0, DX; MOVQ DX, acc0; MOVQ (1*8)(BP), AX; MULQ acc1; ADDQ AX, t2; ADCQ $0, DX
-#define polyMulStage3 IMULQ acc2, t3; ADDQ acc0, t2; ADCQ DX, t3
-#define polyMulReduceStage MOVQ t0, acc0; MOVQ t1, acc1; MOVQ t2, acc2; ANDQ $3, acc2; MOVQ t2, t0; ANDQ $-4, t0; MOVQ t3, t1; SHRQ $2, t3, t2; SHRQ $2, t3; ADDQ t0, acc0; ADCQ t1, acc1; ADCQ $0, acc2; ADDQ t2, acc0; ADCQ t3, acc1; ADCQ $0, acc2
-
-#define polyMulStage1_AVX2 MOVQ (0*8)(BP), DX; MOVQ DX, t2; MULXQ acc0, t0, t1; IMULQ acc2, t2; MULXQ acc1, AX, DX; ADDQ AX, t1; ADCQ DX, t2
-#define polyMulStage2_AVX2 MOVQ (1*8)(BP), DX; MULXQ acc0, acc0, AX; ADDQ acc0, t1; MULXQ acc1, acc1, t3; ADCQ acc1, t2; ADCQ $0, t3
-#define polyMulStage3_AVX2 IMULQ acc2, DX; ADDQ AX, t2; ADCQ DX, t3
-
-#define polyMul polyMulStage1; polyMulStage2; polyMulStage3; polyMulReduceStage
-#define polyMulAVX2 polyMulStage1_AVX2; polyMulStage2_AVX2; polyMulStage3_AVX2; polyMulReduceStage
-// ----------------------------------------------------------------------------
+
+// func polyHashADInternal<>()
 TEXT polyHashADInternal<>(SB), NOSPLIT, $0
-	// adp points to beginning of additional data
-	// itr2 holds ad length
-	XORQ acc0, acc0
-	XORQ acc1, acc1
-	XORQ acc2, acc2
-	CMPQ itr2, $13
-	JNE  hashADLoop
-
-openFastTLSAD:
-	// Special treatment for the TLS case of 13 bytes
-	MOVQ (adp), acc0
-	MOVQ 5(adp), acc1
-	SHRQ $24, acc1
-	MOVQ $1, acc2
-	polyMul
+	// Hack: Must declare #define macros inside of a function due to Avo constraints
+	// ROL rotates the uint32s in register R left by N bits, using temporary T.
+	#define ROL(N, R, T) \
+		MOVO R, T; \
+		PSLLL $(N), T; \
+		PSRLL $(32-(N)), R; \
+		PXOR T, R
+
+	// ROL8 rotates the uint32s in register R left by 8, using temporary T if needed.
+	#ifdef GOAMD64_v2
+		#define ROL8(R, T) PSHUFB ·rol8<>(SB), R
+	#else
+		#define ROL8(R, T) ROL(8, R, T)
+	#endif
+
+	// ROL16 rotates the uint32s in register R left by 16, using temporary T if needed.
+	#ifdef GOAMD64_v2
+		#define ROL16(R, T) PSHUFB ·rol16<>(SB), R
+	#else
+		#define ROL16(R, T) ROL(16, R, T)
+	#endif
+	XORQ  R10, R10
+	XORQ  R11, R11
+	XORQ  R12, R12
+	CMPQ  R9, $0x0d
+	JNE   hashADLoop
+	MOVQ  (CX), R10
+	MOVQ  5(CX), R11
+	SHRQ  $0x18, R11
+	MOVQ  $0x00000001, R12
+	MOVQ  (BP), AX
+	MOVQ  AX, R15
+	MULQ  R10
+	MOVQ  AX, R13
+	MOVQ  DX, R14
+	MOVQ  (BP), AX
+	MULQ  R11
+	IMULQ R12, R15
+	ADDQ  AX, R14
+	ADCQ  DX, R15
+	MOVQ  8(BP), AX
+	MOVQ  AX, R8
+	MULQ  R10
+	ADDQ  AX, R14
+	ADCQ  $0x00, DX
+	MOVQ  DX, R10
+	MOVQ  8(BP), AX
+	MULQ  R11
+	ADDQ  AX, R15
+	ADCQ  $0x00, DX
+	IMULQ R12, R8
+	ADDQ  R10, R15
+	ADCQ  DX, R8
+	MOVQ  R13, R10
+	MOVQ  R14, R11
+	MOVQ  R15, R12
+	ANDQ  $0x03, R12
+	MOVQ  R15, R13
+	ANDQ  $-4, R13
+	MOVQ  R8, R14
+	SHRQ  $0x02, R8, R15
+	SHRQ  $0x02, R8
+	ADDQ  R13, R10
+	ADCQ  R14, R11
+	ADCQ  $0x00, R12
+	ADDQ  R15, R10
+	ADCQ  R8, R11
+	ADCQ  $0x00, R12
 	RET
 
 hashADLoop:
 	// Hash in 16 byte chunks
-	CMPQ itr2, $16
-	JB   hashADTail
-	polyAdd(0(adp))
-	LEAQ (1*16)(adp), adp
-	SUBQ $16, itr2
-	polyMul
-	JMP  hashADLoop
+	CMPQ  R9, $0x10
+	JB    hashADTail
+	ADDQ  (CX), R10
+	ADCQ  8(CX), R11
+	ADCQ  $0x01, R12
+	LEAQ  16(CX), CX
+	SUBQ  $0x10, R9
+	MOVQ  (BP), AX
+	MOVQ  AX, R15
+	MULQ  R10
+	MOVQ  AX, R13
+	MOVQ  DX, R14
+	MOVQ  (BP), AX
+	MULQ  R11
+	IMULQ R12, R15
+	ADDQ  AX, R14
+	ADCQ  DX, R15
+	MOVQ  8(BP), AX
+	MOVQ  AX, R8
+	MULQ  R10
+	ADDQ  AX, R14
+	ADCQ  $0x00, DX
+	MOVQ  DX, R10
+	MOVQ  8(BP), AX
+	MULQ  R11
+	ADDQ  AX, R15
+	ADCQ  $0x00, DX
+	IMULQ R12, R8
+	ADDQ  R10, R15
+	ADCQ  DX, R8
+	MOVQ  R13, R10
+	MOVQ  R14, R11
+	MOVQ  R15, R12
+	ANDQ  $0x03, R12
+	MOVQ  R15, R13
+	ANDQ  $-4, R13
+	MOVQ  R8, R14
+	SHRQ  $0x02, R8, R15
+	SHRQ  $0x02, R8
+	ADDQ  R13, R10
+	ADCQ  R14, R11
+	ADCQ  $0x00, R12
+	ADDQ  R15, R10
+	ADCQ  R8, R11
+	ADCQ  $0x00, R12
+	JMP   hashADLoop
 
 hashADTail:
-	CMPQ itr2, $0
+	CMPQ R9, $0x00
 	JE   hashADDone
 
 	// Hash last < 16 byte tail
-	XORQ t0, t0
-	XORQ t1, t1
-	XORQ t2, t2
-	ADDQ itr2, adp
+	XORQ R13, R13
+	XORQ R14, R14
+	XORQ R15, R15
+	ADDQ R9, CX
 
 hashADTailLoop:
-	SHLQ $8, t0, t1
-	SHLQ $8, t0
-	MOVB -1(adp), t2
-	XORQ t2, t0
-	DECQ adp
-	DECQ itr2
-	JNE  hashADTailLoop
-
-hashADTailFinish:
-	ADDQ t0, acc0; ADCQ t1, acc1; ADCQ $1, acc2
-	polyMul
-
-	// Finished AD
+	SHLQ  $0x08, R13, R14
+	SHLQ  $0x08, R13
+	MOVB  -1(CX), R15
+	XORQ  R15, R13
+	DECQ  CX
+	DECQ  R9
+	JNE   hashADTailLoop
+	ADDQ  R13, R10
+	ADCQ  R14, R11
+	ADCQ  $0x01, R12
+	MOVQ  (BP), AX
+	MOVQ  AX, R15
+	MULQ  R10
+	MOVQ  AX, R13
+	MOVQ  DX, R14
+	MOVQ  (BP), AX
+	MULQ  R11
+	IMULQ R12, R15
+	ADDQ  AX, R14
+	ADCQ  DX, R15
+	MOVQ  8(BP), AX
+	MOVQ  AX, R8
+	MULQ  R10
+	ADDQ  AX, R14
+	ADCQ  $0x00, DX
+	MOVQ  DX, R10
+	MOVQ  8(BP), AX
+	MULQ  R11
+	ADDQ  AX, R15
+	ADCQ  $0x00, DX
+	IMULQ R12, R8
+	ADDQ  R10, R15
+	ADCQ  DX, R8
+	MOVQ  R13, R10
+	MOVQ  R14, R11
+	MOVQ  R15, R12
+	ANDQ  $0x03, R12
+	MOVQ  R15, R13
+	ANDQ  $-4, R13
+	MOVQ  R8, R14
+	SHRQ  $0x02, R8, R15
+	SHRQ  $0x02, R8
+	ADDQ  R13, R10
+	ADCQ  R14, R11
+	ADCQ  $0x00, R12
+	ADDQ  R15, R10
+	ADCQ  R8, R11
+	ADCQ  $0x00, R12
+
 hashADDone:
 	RET
 
-// ----------------------------------------------------------------------------
-// func chacha20Poly1305Open(dst, key, src, ad []byte) bool
-TEXT ·chacha20Poly1305Open(SB), 0, $288-97
+// func chacha20Poly1305Open(dst []byte, key []uint32, src []byte, ad []byte) bool
+// Requires: AVX, AVX2, BMI2, CMOV, SSE2
+TEXT ·chacha20Poly1305Open(SB), $288-97
 	// For aligned stack access
 	MOVQ SP, BP
-	ADDQ $32, BP
+	ADDQ $0x20, BP
 	ANDQ $-32, BP
-	MOVQ dst+0(FP), oup
-	MOVQ key+24(FP), keyp
-	MOVQ src+48(FP), inp
-	MOVQ src_len+56(FP), inl
-	MOVQ ad+72(FP), adp
+	MOVQ dst_base+0(FP), DI
+	MOVQ key_base+24(FP), R8
+	MOVQ src_base+48(FP), SI
+	MOVQ src_len+56(FP), BX
+	MOVQ ad_base+72(FP), CX
 
 	// Check for AVX2 support
-	CMPB ·useAVX2(SB), $1
+	CMPB ·useAVX2+0(SB), $0x01
 	JE   chacha20Poly1305Open_AVX2
 
 	// Special optimization, for very short buffers
-	CMPQ inl, $128
-	JBE  openSSE128 // About 16% faster
+	CMPQ BX, $0x80
+	JBE  openSSE128
 
 	// For long buffers, prepare the poly key first
-	MOVOU ·chacha20Constants<>(SB), A0
-	MOVOU (1*16)(keyp), B0
-	MOVOU (2*16)(keyp), C0
-	MOVOU (3*16)(keyp), D0
-	MOVO  D0, T1
+	MOVOU ·chacha20Constants<>+0(SB), X0
+	MOVOU 16(R8), X3
+	MOVOU 32(R8), X6
+	MOVOU 48(R8), X9
+	MOVO  X9, X13
 
 	// Store state on stack for future use
-	MOVO B0, state1Store
-	MOVO C0, state2Store
-	MOVO D0, ctr3Store
-	MOVQ $10, itr2
+	MOVO X3, 32(BP)
+	MOVO X6, 48(BP)
+	MOVO X9, 128(BP)
+	MOVQ $0x0000000a, R9
 
 openSSEPreparePolyKey:
-	chachaQR(A0, B0, C0, D0, T0)
-	shiftB0Left;  shiftC0Left; shiftD0Left
-	chachaQR(A0, B0, C0, D0, T0)
-	shiftB0Right; shiftC0Right; shiftD0Right
-	DECQ          itr2
-	JNE           openSSEPreparePolyKey
+	PADDD X3, X0
+	PXOR  X0, X9
+	ROL16(X9, X12)
+	PADDD X9, X6
+	PXOR  X6, X3
+	MOVO  X3, X12
+	PSLLL $0x0c, X12
+	PSRLL $0x14, X3
+	PXOR  X12, X3
+	PADDD X3, X0
+	PXOR  X0, X9
+	ROL8(X9, X12)
+	PADDD X9, X6
+	PXOR  X6, X3
+	MOVO  X3, X12
+	PSLLL $0x07, X12
+	PSRLL $0x19, X3
+	PXOR  X12, X3
+	BYTE  $0x66
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xdb
+	BYTE  $0x04
+	BYTE  $0x66
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xf6
+	BYTE  $0x08
+	BYTE  $0x66
+	BYTE  $0x45
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xc9
+	BYTE  $0x0c
+	PADDD X3, X0
+	PXOR  X0, X9
+	ROL16(X9, X12)
+	PADDD X9, X6
+	PXOR  X6, X3
+	MOVO  X3, X12
+	PSLLL $0x0c, X12
+	PSRLL $0x14, X3
+	PXOR  X12, X3
+	PADDD X3, X0
+	PXOR  X0, X9
+	ROL8(X9, X12)
+	PADDD X9, X6
+	PXOR  X6, X3
+	MOVO  X3, X12
+	PSLLL $0x07, X12
+	PSRLL $0x19, X3
+	PXOR  X12, X3
+	BYTE  $0x66
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xdb
+	BYTE  $0x0c
+	BYTE  $0x66
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xf6
+	BYTE  $0x08
+	BYTE  $0x66
+	BYTE  $0x45
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xc9
+	BYTE  $0x04
+	DECQ  R9
+	JNE   openSSEPreparePolyKey
 
 	// A0|B0 hold the Poly1305 32-byte key, C0,D0 can be discarded
-	PADDL ·chacha20Constants<>(SB), A0; PADDL state1Store, B0
+	PADDL ·chacha20Constants<>+0(SB), X0
+	PADDL 32(BP), X3
 
 	// Clamp and store the key
-	PAND ·polyClampMask<>(SB), A0
-	MOVO A0, rStore; MOVO B0, sStore
+	PAND ·polyClampMask<>+0(SB), X0
+	MOVO X0, (BP)
+	MOVO X3, 16(BP)
 
 	// Hash AAD
-	MOVQ ad_len+80(FP), itr2
+	MOVQ ad_len+80(FP), R9
 	CALL polyHashADInternal<>(SB)
 
 openSSEMainLoop:
-	CMPQ inl, $256
+	CMPQ BX, $0x00000100
 	JB   openSSEMainLoopDone
 
 	// Load state, increment counter blocks
-	MOVO ·chacha20Constants<>(SB), A0; MOVO state1Store, B0; MOVO state2Store, C0; MOVO ctr3Store, D0; PADDL ·sseIncMask<>(SB), D0
-	MOVO A0, A1; MOVO B0, B1; MOVO C0, C1; MOVO D0, D1; PADDL ·sseIncMask<>(SB), D1
-	MOVO A1, A2; MOVO B1, B2; MOVO C1, C2; MOVO D1, D2; PADDL ·sseIncMask<>(SB), D2
-	MOVO A2, A3; MOVO B2, B3; MOVO C2, C3; MOVO D2, D3; PADDL ·sseIncMask<>(SB), D3
+	MOVO  ·chacha20Constants<>+0(SB), X0
+	MOVO  32(BP), X3
+	MOVO  48(BP), X6
+	MOVO  128(BP), X9
+	PADDL ·sseIncMask<>+0(SB), X9
+	MOVO  X0, X1
+	MOVO  X3, X4
+	MOVO  X6, X7
+	MOVO  X9, X10
+	PADDL ·sseIncMask<>+0(SB), X10
+	MOVO  X1, X2
+	MOVO  X4, X5
+	MOVO  X7, X8
+	MOVO  X10, X11
+	PADDL ·sseIncMask<>+0(SB), X11
+	MOVO  X2, X12
+	MOVO  X5, X13
+	MOVO  X8, X14
+	MOVO  X11, X15
+	PADDL ·sseIncMask<>+0(SB), X15
 
 	// Store counters
-	MOVO D0, ctr0Store; MOVO D1, ctr1Store; MOVO D2, ctr2Store; MOVO D3, ctr3Store
+	MOVO X9, 80(BP)
+	MOVO X10, 96(BP)
+	MOVO X11, 112(BP)
+	MOVO X15, 128(BP)
 
-	// There are 10 ChaCha20 iterations of 2QR each, so for 6 iterations we hash 2 blocks, and for the remaining 4 only 1 block - for a total of 16
-	MOVQ $4, itr1
-	MOVQ inp, itr2
+	// There are 10 ChaCha20 iterations of 2QR each, so for 6 iterations we hash
+	// 2 blocks, and for the remaining 4 only 1 block - for a total of 16
+	MOVQ $0x00000004, CX
+	MOVQ SI, R9
 
 openSSEInternalLoop:
-	MOVO          C3, tmpStore
-	chachaQR(A0, B0, C0, D0, C3); chachaQR(A1, B1, C1, D1, C3); chachaQR(A2, B2, C2, D2, C3)
-	MOVO          tmpStore, C3
-	MOVO          C1, tmpStore
-	chachaQR(A3, B3, C3, D3, C1)
-	MOVO          tmpStore, C1
-	polyAdd(0(itr2))
-	shiftB0Left;  shiftB1Left; shiftB2Left; shiftB3Left
-	shiftC0Left;  shiftC1Left; shiftC2Left; shiftC3Left
-	shiftD0Left;  shiftD1Left; shiftD2Left; shiftD3Left
-	polyMulStage1
-	polyMulStage2
-	LEAQ          (2*8)(itr2), itr2
-	MOVO          C3, tmpStore
-	chachaQR(A0, B0, C0, D0, C3); chachaQR(A1, B1, C1, D1, C3); chachaQR(A2, B2, C2, D2, C3)
-	MOVO          tmpStore, C3
-	MOVO          C1, tmpStore
-	polyMulStage3
-	chachaQR(A3, B3, C3, D3, C1)
-	MOVO          tmpStore, C1
-	polyMulReduceStage
-	shiftB0Right; shiftB1Right; shiftB2Right; shiftB3Right
-	shiftC0Right; shiftC1Right; shiftC2Right; shiftC3Right
-	shiftD0Right; shiftD1Right; shiftD2Right; shiftD3Right
-	DECQ          itr1
-	JGE           openSSEInternalLoop
-
-	polyAdd(0(itr2))
-	polyMul
-	LEAQ (2*8)(itr2), itr2
-
-	CMPQ itr1, $-6
-	JG   openSSEInternalLoop
+	MOVO  X14, 64(BP)
+	PADDD X3, X0
+	PXOR  X0, X9
+	ROL16(X9, X14)
+	PADDD X9, X6
+	PXOR  X6, X3
+	MOVO  X3, X14
+	PSLLL $0x0c, X14
+	PSRLL $0x14, X3
+	PXOR  X14, X3
+	PADDD X3, X0
+	PXOR  X0, X9
+	ROL8(X9, X14)
+	PADDD X9, X6
+	PXOR  X6, X3
+	MOVO  X3, X14
+	PSLLL $0x07, X14
+	PSRLL $0x19, X3
+	PXOR  X14, X3
+	PADDD X4, X1
+	PXOR  X1, X10
+	ROL16(X10, X14)
+	PADDD X10, X7
+	PXOR  X7, X4
+	MOVO  X4, X14
+	PSLLL $0x0c, X14
+	PSRLL $0x14, X4
+	PXOR  X14, X4
+	PADDD X4, X1
+	PXOR  X1, X10
+	ROL8(X10, X14)
+	PADDD X10, X7
+	PXOR  X7, X4
+	MOVO  X4, X14
+	PSLLL $0x07, X14
+	PSRLL $0x19, X4
+	PXOR  X14, X4
+	PADDD X5, X2
+	PXOR  X2, X11
+	ROL16(X11, X14)
+	PADDD X11, X8
+	PXOR  X8, X5
+	MOVO  X5, X14
+	PSLLL $0x0c, X14
+	PSRLL $0x14, X5
+	PXOR  X14, X5
+	PADDD X5, X2
+	PXOR  X2, X11
+	ROL8(X11, X14)
+	PADDD X11, X8
+	PXOR  X8, X5
+	MOVO  X5, X14
+	PSLLL $0x07, X14
+	PSRLL $0x19, X5
+	PXOR  X14, X5
+	MOVO  64(BP), X14
+	MOVO  X7, 64(BP)
+	PADDD X13, X12
+	PXOR  X12, X15
+	ROL16(X15, X7)
+	PADDD X15, X14
+	PXOR  X14, X13
+	MOVO  X13, X7
+	PSLLL $0x0c, X7
+	PSRLL $0x14, X13
+	PXOR  X7, X13
+	PADDD X13, X12
+	PXOR  X12, X15
+	ROL8(X15, X7)
+	PADDD X15, X14
+	PXOR  X14, X13
+	MOVO  X13, X7
+	PSLLL $0x07, X7
+	PSRLL $0x19, X13
+	PXOR  X7, X13
+	MOVO  64(BP), X7
+	ADDQ  (R9), R10
+	ADCQ  8(R9), R11
+	ADCQ  $0x01, R12
+	BYTE  $0x66
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xdb
+	BYTE  $0x04
+	BYTE  $0x66
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xe4
+	BYTE  $0x04
+	BYTE  $0x66
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xed
+	BYTE  $0x04
+	BYTE  $0x66
+	BYTE  $0x45
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xed
+	BYTE  $0x04
+	BYTE  $0x66
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xf6
+	BYTE  $0x08
+	BYTE  $0x66
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xff
+	BYTE  $0x08
+	BYTE  $0x66
+	BYTE  $0x45
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xc0
+	BYTE  $0x08
+	BYTE  $0x66
+	BYTE  $0x45
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xf6
+	BYTE  $0x08
+	BYTE  $0x66
+	BYTE  $0x45
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xc9
+	BYTE  $0x0c
+	BYTE  $0x66
+	BYTE  $0x45
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xd2
+	BYTE  $0x0c
+	BYTE  $0x66
+	BYTE  $0x45
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xdb
+	BYTE  $0x0c
+	BYTE  $0x66
+	BYTE  $0x45
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xff
+	BYTE  $0x0c
+	MOVQ  (BP), AX
+	MOVQ  AX, R15
+	MULQ  R10
+	MOVQ  AX, R13
+	MOVQ  DX, R14
+	MOVQ  (BP), AX
+	MULQ  R11
+	IMULQ R12, R15
+	ADDQ  AX, R14
+	ADCQ  DX, R15
+	MOVQ  8(BP), AX
+	MOVQ  AX, R8
+	MULQ  R10
+	ADDQ  AX, R14
+	ADCQ  $0x00, DX
+	MOVQ  DX, R10
+	MOVQ  8(BP), AX
+	MULQ  R11
+	ADDQ  AX, R15
+	ADCQ  $0x00, DX
+	LEAQ  16(R9), R9
+	MOVO  X14, 64(BP)
+	PADDD X3, X0
+	PXOR  X0, X9
+	ROL16(X9, X14)
+	PADDD X9, X6
+	PXOR  X6, X3
+	MOVO  X3, X14
+	PSLLL $0x0c, X14
+	PSRLL $0x14, X3
+	PXOR  X14, X3
+	PADDD X3, X0
+	PXOR  X0, X9
+	ROL8(X9, X14)
+	PADDD X9, X6
+	PXOR  X6, X3
+	MOVO  X3, X14
+	PSLLL $0x07, X14
+	PSRLL $0x19, X3
+	PXOR  X14, X3
+	PADDD X4, X1
+	PXOR  X1, X10
+	ROL16(X10, X14)
+	PADDD X10, X7
+	PXOR  X7, X4
+	MOVO  X4, X14
+	PSLLL $0x0c, X14
+	PSRLL $0x14, X4
+	PXOR  X14, X4
+	PADDD X4, X1
+	PXOR  X1, X10
+	ROL8(X10, X14)
+	PADDD X10, X7
+	PXOR  X7, X4
+	MOVO  X4, X14
+	PSLLL $0x07, X14
+	PSRLL $0x19, X4
+	PXOR  X14, X4
+	PADDD X5, X2
+	PXOR  X2, X11
+	ROL16(X11, X14)
+	PADDD X11, X8
+	PXOR  X8, X5
+	MOVO  X5, X14
+	PSLLL $0x0c, X14
+	PSRLL $0x14, X5
+	PXOR  X14, X5
+	PADDD X5, X2
+	PXOR  X2, X11
+	ROL8(X11, X14)
+	PADDD X11, X8
+	PXOR  X8, X5
+	MOVO  X5, X14
+	PSLLL $0x07, X14
+	PSRLL $0x19, X5
+	PXOR  X14, X5
+	MOVO  64(BP), X14
+	MOVO  X7, 64(BP)
+	IMULQ R12, R8
+	ADDQ  R10, R15
+	ADCQ  DX, R8
+	PADDD X13, X12
+	PXOR  X12, X15
+	ROL16(X15, X7)
+	PADDD X15, X14
+	PXOR  X14, X13
+	MOVO  X13, X7
+	PSLLL $0x0c, X7
+	PSRLL $0x14, X13
+	PXOR  X7, X13
+	PADDD X13, X12
+	PXOR  X12, X15
+	ROL8(X15, X7)
+	PADDD X15, X14
+	PXOR  X14, X13
+	MOVO  X13, X7
+	PSLLL $0x07, X7
+	PSRLL $0x19, X13
+	PXOR  X7, X13
+	MOVO  64(BP), X7
+	MOVQ  R13, R10
+	MOVQ  R14, R11
+	MOVQ  R15, R12
+	ANDQ  $0x03, R12
+	MOVQ  R15, R13
+	ANDQ  $-4, R13
+	MOVQ  R8, R14
+	SHRQ  $0x02, R8, R15
+	SHRQ  $0x02, R8
+	ADDQ  R13, R10
+	ADCQ  R14, R11
+	ADCQ  $0x00, R12
+	ADDQ  R15, R10
+	ADCQ  R8, R11
+	ADCQ  $0x00, R12
+	BYTE  $0x66
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xdb
+	BYTE  $0x0c
+	BYTE  $0x66
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xe4
+	BYTE  $0x0c
+	BYTE  $0x66
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xed
+	BYTE  $0x0c
+	BYTE  $0x66
+	BYTE  $0x45
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xed
+	BYTE  $0x0c
+	BYTE  $0x66
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xf6
+	BYTE  $0x08
+	BYTE  $0x66
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xff
+	BYTE  $0x08
+	BYTE  $0x66
+	BYTE  $0x45
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xc0
+	BYTE  $0x08
+	BYTE  $0x66
+	BYTE  $0x45
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xf6
+	BYTE  $0x08
+	BYTE  $0x66
+	BYTE  $0x45
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xc9
+	BYTE  $0x04
+	BYTE  $0x66
+	BYTE  $0x45
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xd2
+	BYTE  $0x04
+	BYTE  $0x66
+	BYTE  $0x45
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xdb
+	BYTE  $0x04
+	BYTE  $0x66
+	BYTE  $0x45
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xff
+	BYTE  $0x04
+	DECQ  CX
+	JGE   openSSEInternalLoop
+	ADDQ  (R9), R10
+	ADCQ  8(R9), R11
+	ADCQ  $0x01, R12
+	MOVQ  (BP), AX
+	MOVQ  AX, R15
+	MULQ  R10
+	MOVQ  AX, R13
+	MOVQ  DX, R14
+	MOVQ  (BP), AX
+	MULQ  R11
+	IMULQ R12, R15
+	ADDQ  AX, R14
+	ADCQ  DX, R15
+	MOVQ  8(BP), AX
+	MOVQ  AX, R8
+	MULQ  R10
+	ADDQ  AX, R14
+	ADCQ  $0x00, DX
+	MOVQ  DX, R10
+	MOVQ  8(BP), AX
+	MULQ  R11
+	ADDQ  AX, R15
+	ADCQ  $0x00, DX
+	IMULQ R12, R8
+	ADDQ  R10, R15
+	ADCQ  DX, R8
+	MOVQ  R13, R10
+	MOVQ  R14, R11
+	MOVQ  R15, R12
+	ANDQ  $0x03, R12
+	MOVQ  R15, R13
+	ANDQ  $-4, R13
+	MOVQ  R8, R14
+	SHRQ  $0x02, R8, R15
+	SHRQ  $0x02, R8
+	ADDQ  R13, R10
+	ADCQ  R14, R11
+	ADCQ  $0x00, R12
+	ADDQ  R15, R10
+	ADCQ  R8, R11
+	ADCQ  $0x00, R12
+	LEAQ  16(R9), R9
+	CMPQ  CX, $-6
+	JG    openSSEInternalLoop
 
 	// Add in the state
-	PADDD ·chacha20Constants<>(SB), A0; PADDD ·chacha20Constants<>(SB), A1; PADDD ·chacha20Constants<>(SB), A2; PADDD ·chacha20Constants<>(SB), A3
-	PADDD state1Store, B0; PADDD state1Store, B1; PADDD state1Store, B2; PADDD state1Store, B3
-	PADDD state2Store, C0; PADDD state2Store, C1; PADDD state2Store, C2; PADDD state2Store, C3
-	PADDD ctr0Store, D0; PADDD ctr1Store, D1; PADDD ctr2Store, D2; PADDD ctr3Store, D3
+	PADDD ·chacha20Constants<>+0(SB), X0
+	PADDD ·chacha20Constants<>+0(SB), X1
+	PADDD ·chacha20Constants<>+0(SB), X2
+	PADDD ·chacha20Constants<>+0(SB), X12
+	PADDD 32(BP), X3
+	PADDD 32(BP), X4
+	PADDD 32(BP), X5
+	PADDD 32(BP), X13
+	PADDD 48(BP), X6
+	PADDD 48(BP), X7
+	PADDD 48(BP), X8
+	PADDD 48(BP), X14
+	PADDD 80(BP), X9
+	PADDD 96(BP), X10
+	PADDD 112(BP), X11
+	PADDD 128(BP), X15
 
 	// Load - xor - store
-	MOVO  D3, tmpStore
-	MOVOU (0*16)(inp), D3; PXOR D3, A0; MOVOU A0, (0*16)(oup)
-	MOVOU (1*16)(inp), D3; PXOR D3, B0; MOVOU B0, (1*16)(oup)
-	MOVOU (2*16)(inp), D3; PXOR D3, C0; MOVOU C0, (2*16)(oup)
-	MOVOU (3*16)(inp), D3; PXOR D3, D0; MOVOU D0, (3*16)(oup)
-	MOVOU (4*16)(inp), D0; PXOR D0, A1; MOVOU A1, (4*16)(oup)
-	MOVOU (5*16)(inp), D0; PXOR D0, B1; MOVOU B1, (5*16)(oup)
-	MOVOU (6*16)(inp), D0; PXOR D0, C1; MOVOU C1, (6*16)(oup)
-	MOVOU (7*16)(inp), D0; PXOR D0, D1; MOVOU D1, (7*16)(oup)
-	MOVOU (8*16)(inp), D0; PXOR D0, A2; MOVOU A2, (8*16)(oup)
-	MOVOU (9*16)(inp), D0; PXOR D0, B2; MOVOU B2, (9*16)(oup)
-	MOVOU (10*16)(inp), D0; PXOR D0, C2; MOVOU C2, (10*16)(oup)
-	MOVOU (11*16)(inp), D0; PXOR D0, D2; MOVOU D2, (11*16)(oup)
-	MOVOU (12*16)(inp), D0; PXOR D0, A3; MOVOU A3, (12*16)(oup)
-	MOVOU (13*16)(inp), D0; PXOR D0, B3; MOVOU B3, (13*16)(oup)
-	MOVOU (14*16)(inp), D0; PXOR D0, C3; MOVOU C3, (14*16)(oup)
-	MOVOU (15*16)(inp), D0; PXOR tmpStore, D0; MOVOU D0, (15*16)(oup)
-	LEAQ  256(inp), inp
-	LEAQ  256(oup), oup
-	SUBQ  $256, inl
+	MOVO  X15, 64(BP)
+	MOVOU (SI), X15
+	PXOR  X15, X0
+	MOVOU X0, (DI)
+	MOVOU 16(SI), X15
+	PXOR  X15, X3
+	MOVOU X3, 16(DI)
+	MOVOU 32(SI), X15
+	PXOR  X15, X6
+	MOVOU X6, 32(DI)
+	MOVOU 48(SI), X15
+	PXOR  X15, X9
+	MOVOU X9, 48(DI)
+	MOVOU 64(SI), X9
+	PXOR  X9, X1
+	MOVOU X1, 64(DI)
+	MOVOU 80(SI), X9
+	PXOR  X9, X4
+	MOVOU X4, 80(DI)
+	MOVOU 96(SI), X9
+	PXOR  X9, X7
+	MOVOU X7, 96(DI)
+	MOVOU 112(SI), X9
+	PXOR  X9, X10
+	MOVOU X10, 112(DI)
+	MOVOU 128(SI), X9
+	PXOR  X9, X2
+	MOVOU X2, 128(DI)
+	MOVOU 144(SI), X9
+	PXOR  X9, X5
+	MOVOU X5, 144(DI)
+	MOVOU 160(SI), X9
+	PXOR  X9, X8
+	MOVOU X8, 160(DI)
+	MOVOU 176(SI), X9
+	PXOR  X9, X11
+	MOVOU X11, 176(DI)
+	MOVOU 192(SI), X9
+	PXOR  X9, X12
+	MOVOU X12, 192(DI)
+	MOVOU 208(SI), X9
+	PXOR  X9, X13
+	MOVOU X13, 208(DI)
+	MOVOU 224(SI), X9
+	PXOR  X9, X14
+	MOVOU X14, 224(DI)
+	MOVOU 240(SI), X9
+	PXOR  64(BP), X9
+	MOVOU X9, 240(DI)
+	LEAQ  256(SI), SI
+	LEAQ  256(DI), DI
+	SUBQ  $0x00000100, BX
 	JMP   openSSEMainLoop
 
 openSSEMainLoopDone:
 	// Handle the various tail sizes efficiently
-	TESTQ inl, inl
+	TESTQ BX, BX
 	JE    openSSEFinalize
-	CMPQ  inl, $64
+	CMPQ  BX, $0x40
 	JBE   openSSETail64
-	CMPQ  inl, $128
+	CMPQ  BX, $0x80
 	JBE   openSSETail128
-	CMPQ  inl, $192
+	CMPQ  BX, $0xc0
 	JBE   openSSETail192
 	JMP   openSSETail256
 
 openSSEFinalize:
 	// Hash in the PT, AAD lengths
-	ADDQ ad_len+80(FP), acc0; ADCQ src_len+56(FP), acc1; ADCQ $1, acc2
-	polyMul
+	ADDQ  ad_len+80(FP), R10
+	ADCQ  src_len+56(FP), R11
+	ADCQ  $0x01, R12
+	MOVQ  (BP), AX
+	MOVQ  AX, R15
+	MULQ  R10
+	MOVQ  AX, R13
+	MOVQ  DX, R14
+	MOVQ  (BP), AX
+	MULQ  R11
+	IMULQ R12, R15
+	ADDQ  AX, R14
+	ADCQ  DX, R15
+	MOVQ  8(BP), AX
+	MOVQ  AX, R8
+	MULQ  R10
+	ADDQ  AX, R14
+	ADCQ  $0x00, DX
+	MOVQ  DX, R10
+	MOVQ  8(BP), AX
+	MULQ  R11
+	ADDQ  AX, R15
+	ADCQ  $0x00, DX
+	IMULQ R12, R8
+	ADDQ  R10, R15
+	ADCQ  DX, R8
+	MOVQ  R13, R10
+	MOVQ  R14, R11
+	MOVQ  R15, R12
+	ANDQ  $0x03, R12
+	MOVQ  R15, R13
+	ANDQ  $-4, R13
+	MOVQ  R8, R14
+	SHRQ  $0x02, R8, R15
+	SHRQ  $0x02, R8
+	ADDQ  R13, R10
+	ADCQ  R14, R11
+	ADCQ  $0x00, R12
+	ADDQ  R15, R10
+	ADCQ  R8, R11
+	ADCQ  $0x00, R12
 
 	// Final reduce
-	MOVQ    acc0, t0
-	MOVQ    acc1, t1
-	MOVQ    acc2, t2
-	SUBQ    $-5, acc0
-	SBBQ    $-1, acc1
-	SBBQ    $3, acc2
-	CMOVQCS t0, acc0
-	CMOVQCS t1, acc1
-	CMOVQCS t2, acc2
+	MOVQ    R10, R13
+	MOVQ    R11, R14
+	MOVQ    R12, R15
+	SUBQ    $-5, R10
+	SBBQ    $-1, R11
+	SBBQ    $0x03, R12
+	CMOVQCS R13, R10
+	CMOVQCS R14, R11
+	CMOVQCS R15, R12
 
 	// Add in the "s" part of the key
-	ADDQ 0+sStore, acc0
-	ADCQ 8+sStore, acc1
+	ADDQ 16(BP), R10
+	ADCQ 24(BP), R11
 
 	// Finally, constant time compare to the tag at the end of the message
 	XORQ    AX, AX
-	MOVQ    $1, DX
-	XORQ    (0*8)(inp), acc0
-	XORQ    (1*8)(inp), acc1
-	ORQ     acc1, acc0
+	MOVQ    $0x00000001, DX
+	XORQ    (SI), R10
+	XORQ    8(SI), R11
+	ORQ     R11, R10
 	CMOVQEQ DX, AX
 
 	// Return true iff tags are equal
 	MOVB AX, ret+96(FP)
 	RET
 
-// ----------------------------------------------------------------------------
-// Special optimization for buffers smaller than 129 bytes
 openSSE128:
-	// For up to 128 bytes of ciphertext and 64 bytes for the poly key, we require to process three blocks
-	MOVOU ·chacha20Constants<>(SB), A0; MOVOU (1*16)(keyp), B0; MOVOU (2*16)(keyp), C0; MOVOU (3*16)(keyp), D0
-	MOVO  A0, A1; MOVO B0, B1; MOVO C0, C1; MOVO D0, D1; PADDL ·sseIncMask<>(SB), D1
-	MOVO  A1, A2; MOVO B1, B2; MOVO C1, C2; MOVO D1, D2; PADDL ·sseIncMask<>(SB), D2
-	MOVO  B0, T1; MOVO C0, T2; MOVO D1, T3
-	MOVQ  $10, itr2
+	MOVOU ·chacha20Constants<>+0(SB), X0
+	MOVOU 16(R8), X3
+	MOVOU 32(R8), X6
+	MOVOU 48(R8), X9
+	MOVO  X0, X1
+	MOVO  X3, X4
+	MOVO  X6, X7
+	MOVO  X9, X10
+	PADDL ·sseIncMask<>+0(SB), X10
+	MOVO  X1, X2
+	MOVO  X4, X5
+	MOVO  X7, X8
+	MOVO  X10, X11
+	PADDL ·sseIncMask<>+0(SB), X11
+	MOVO  X3, X13
+	MOVO  X6, X14
+	MOVO  X10, X15
+	MOVQ  $0x0000000a, R9
 
 openSSE128InnerCipherLoop:
-	chachaQR(A0, B0, C0, D0, T0); chachaQR(A1, B1, C1, D1, T0); chachaQR(A2, B2, C2, D2, T0)
-	shiftB0Left;  shiftB1Left; shiftB2Left
-	shiftC0Left;  shiftC1Left; shiftC2Left
-	shiftD0Left;  shiftD1Left; shiftD2Left
-	chachaQR(A0, B0, C0, D0, T0); chachaQR(A1, B1, C1, D1, T0); chachaQR(A2, B2, C2, D2, T0)
-	shiftB0Right; shiftB1Right; shiftB2Right
-	shiftC0Right; shiftC1Right; shiftC2Right
-	shiftD0Right; shiftD1Right; shiftD2Right
-	DECQ          itr2
-	JNE           openSSE128InnerCipherLoop
+	PADDD X3, X0
+	PXOR  X0, X9
+	ROL16(X9, X12)
+	PADDD X9, X6
+	PXOR  X6, X3
+	MOVO  X3, X12
+	PSLLL $0x0c, X12
+	PSRLL $0x14, X3
+	PXOR  X12, X3
+	PADDD X3, X0
+	PXOR  X0, X9
+	ROL8(X9, X12)
+	PADDD X9, X6
+	PXOR  X6, X3
+	MOVO  X3, X12
+	PSLLL $0x07, X12
+	PSRLL $0x19, X3
+	PXOR  X12, X3
+	PADDD X4, X1
+	PXOR  X1, X10
+	ROL16(X10, X12)
+	PADDD X10, X7
+	PXOR  X7, X4
+	MOVO  X4, X12
+	PSLLL $0x0c, X12
+	PSRLL $0x14, X4
+	PXOR  X12, X4
+	PADDD X4, X1
+	PXOR  X1, X10
+	ROL8(X10, X12)
+	PADDD X10, X7
+	PXOR  X7, X4
+	MOVO  X4, X12
+	PSLLL $0x07, X12
+	PSRLL $0x19, X4
+	PXOR  X12, X4
+	PADDD X5, X2
+	PXOR  X2, X11
+	ROL16(X11, X12)
+	PADDD X11, X8
+	PXOR  X8, X5
+	MOVO  X5, X12
+	PSLLL $0x0c, X12
+	PSRLL $0x14, X5
+	PXOR  X12, X5
+	PADDD X5, X2
+	PXOR  X2, X11
+	ROL8(X11, X12)
+	PADDD X11, X8
+	PXOR  X8, X5
+	MOVO  X5, X12
+	PSLLL $0x07, X12
+	PSRLL $0x19, X5
+	PXOR  X12, X5
+	BYTE  $0x66
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xdb
+	BYTE  $0x04
+	BYTE  $0x66
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xe4
+	BYTE  $0x04
+	BYTE  $0x66
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xed
+	BYTE  $0x04
+	BYTE  $0x66
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xf6
+	BYTE  $0x08
+	BYTE  $0x66
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xff
+	BYTE  $0x08
+	BYTE  $0x66
+	BYTE  $0x45
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xc0
+	BYTE  $0x08
+	BYTE  $0x66
+	BYTE  $0x45
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xc9
+	BYTE  $0x0c
+	BYTE  $0x66
+	BYTE  $0x45
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xd2
+	BYTE  $0x0c
+	BYTE  $0x66
+	BYTE  $0x45
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xdb
+	BYTE  $0x0c
+	PADDD X3, X0
+	PXOR  X0, X9
+	ROL16(X9, X12)
+	PADDD X9, X6
+	PXOR  X6, X3
+	MOVO  X3, X12
+	PSLLL $0x0c, X12
+	PSRLL $0x14, X3
+	PXOR  X12, X3
+	PADDD X3, X0
+	PXOR  X0, X9
+	ROL8(X9, X12)
+	PADDD X9, X6
+	PXOR  X6, X3
+	MOVO  X3, X12
+	PSLLL $0x07, X12
+	PSRLL $0x19, X3
+	PXOR  X12, X3
+	PADDD X4, X1
+	PXOR  X1, X10
+	ROL16(X10, X12)
+	PADDD X10, X7
+	PXOR  X7, X4
+	MOVO  X4, X12
+	PSLLL $0x0c, X12
+	PSRLL $0x14, X4
+	PXOR  X12, X4
+	PADDD X4, X1
+	PXOR  X1, X10
+	ROL8(X10, X12)
+	PADDD X10, X7
+	PXOR  X7, X4
+	MOVO  X4, X12
+	PSLLL $0x07, X12
+	PSRLL $0x19, X4
+	PXOR  X12, X4
+	PADDD X5, X2
+	PXOR  X2, X11
+	ROL16(X11, X12)
+	PADDD X11, X8
+	PXOR  X8, X5
+	MOVO  X5, X12
+	PSLLL $0x0c, X12
+	PSRLL $0x14, X5
+	PXOR  X12, X5
+	PADDD X5, X2
+	PXOR  X2, X11
+	ROL8(X11, X12)
+	PADDD X11, X8
+	PXOR  X8, X5
+	MOVO  X5, X12
+	PSLLL $0x07, X12
+	PSRLL $0x19, X5
+	PXOR  X12, X5
+	BYTE  $0x66
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xdb
+	BYTE  $0x0c
+	BYTE  $0x66
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xe4
+	BYTE  $0x0c
+	BYTE  $0x66
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xed
+	BYTE  $0x0c
+	BYTE  $0x66
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xf6
+	BYTE  $0x08
+	BYTE  $0x66
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xff
+	BYTE  $0x08
+	BYTE  $0x66
+	BYTE  $0x45
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xc0
+	BYTE  $0x08
+	BYTE  $0x66
+	BYTE  $0x45
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xc9
+	BYTE  $0x04
+	BYTE  $0x66
+	BYTE  $0x45
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xd2
+	BYTE  $0x04
+	BYTE  $0x66
+	BYTE  $0x45
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xdb
+	BYTE  $0x04
+	DECQ  R9
+	JNE   openSSE128InnerCipherLoop
 
 	// A0|B0 hold the Poly1305 32-byte key, C0,D0 can be discarded
-	PADDL ·chacha20Constants<>(SB), A0; PADDL ·chacha20Constants<>(SB), A1; PADDL ·chacha20Constants<>(SB), A2
-	PADDL T1, B0; PADDL T1, B1; PADDL T1, B2
-	PADDL T2, C1; PADDL T2, C2
-	PADDL T3, D1; PADDL ·sseIncMask<>(SB), T3; PADDL T3, D2
+	PADDL ·chacha20Constants<>+0(SB), X0
+	PADDL ·chacha20Constants<>+0(SB), X1
+	PADDL ·chacha20Constants<>+0(SB), X2
+	PADDL X13, X3
+	PADDL X13, X4
+	PADDL X13, X5
+	PADDL X14, X7
+	PADDL X14, X8
+	PADDL X15, X10
+	PADDL ·sseIncMask<>+0(SB), X15
+	PADDL X15, X11
 
 	// Clamp and store the key
-	PAND  ·polyClampMask<>(SB), A0
-	MOVOU A0, rStore; MOVOU B0, sStore
+	PAND  ·polyClampMask<>+0(SB), X0
+	MOVOU X0, (BP)
+	MOVOU X3, 16(BP)
 
 	// Hash
-	MOVQ ad_len+80(FP), itr2
+	MOVQ ad_len+80(FP), R9
 	CALL polyHashADInternal<>(SB)
 
 openSSE128Open:
-	CMPQ inl, $16
+	CMPQ BX, $0x10
 	JB   openSSETail16
-	SUBQ $16, inl
+	SUBQ $0x10, BX
 
 	// Load for hashing
-	polyAdd(0(inp))
+	ADDQ (SI), R10
+	ADCQ 8(SI), R11
+	ADCQ $0x01, R12
 
 	// Load for decryption
-	MOVOU (inp), T0; PXOR T0, A1; MOVOU A1, (oup)
-	LEAQ  (1*16)(inp), inp
-	LEAQ  (1*16)(oup), oup
-	polyMul
+	MOVOU (SI), X12
+	PXOR  X12, X1
+	MOVOU X1, (DI)
+	LEAQ  16(SI), SI
+	LEAQ  16(DI), DI
+	MOVQ  (BP), AX
+	MOVQ  AX, R15
+	MULQ  R10
+	MOVQ  AX, R13
+	MOVQ  DX, R14
+	MOVQ  (BP), AX
+	MULQ  R11
+	IMULQ R12, R15
+	ADDQ  AX, R14
+	ADCQ  DX, R15
+	MOVQ  8(BP), AX
+	MOVQ  AX, R8
+	MULQ  R10
+	ADDQ  AX, R14
+	ADCQ  $0x00, DX
+	MOVQ  DX, R10
+	MOVQ  8(BP), AX
+	MULQ  R11
+	ADDQ  AX, R15
+	ADCQ  $0x00, DX
+	IMULQ R12, R8
+	ADDQ  R10, R15
+	ADCQ  DX, R8
+	MOVQ  R13, R10
+	MOVQ  R14, R11
+	MOVQ  R15, R12
+	ANDQ  $0x03, R12
+	MOVQ  R15, R13
+	ANDQ  $-4, R13
+	MOVQ  R8, R14
+	SHRQ  $0x02, R8, R15
+	SHRQ  $0x02, R8
+	ADDQ  R13, R10
+	ADCQ  R14, R11
+	ADCQ  $0x00, R12
+	ADDQ  R15, R10
+	ADCQ  R8, R11
+	ADCQ  $0x00, R12
 
 	// Shift the stream "left"
-	MOVO B1, A1
-	MOVO C1, B1
-	MOVO D1, C1
-	MOVO A2, D1
-	MOVO B2, A2
-	MOVO C2, B2
-	MOVO D2, C2
+	MOVO X4, X1
+	MOVO X7, X4
+	MOVO X10, X7
+	MOVO X2, X10
+	MOVO X5, X2
+	MOVO X8, X5
+	MOVO X11, X8
 	JMP  openSSE128Open
 
 openSSETail16:
-	TESTQ inl, inl
+	TESTQ BX, BX
 	JE    openSSEFinalize
 
 	// We can safely load the CT from the end, because it is padded with the MAC
-	MOVQ   inl, itr2
-	SHLQ   $4, itr2
-	LEAQ   ·andMask<>(SB), t0
-	MOVOU  (inp), T0
-	ADDQ   inl, inp
-	PAND   -16(t0)(itr2*1), T0
-	MOVO   T0, 0+tmpStore
-	MOVQ   T0, t0
-	MOVQ   8+tmpStore, t1
-	PXOR   A1, T0
+	MOVQ  BX, R9
+	SHLQ  $0x04, R9
+	LEAQ  ·andMask<>+0(SB), R13
+	MOVOU (SI), X12
+	ADDQ  BX, SI
+	PAND  -16(R13)(R9*1), X12
+	MOVO  X12, 64(BP)
+	MOVQ  X12, R13
+	MOVQ  72(BP), R14
+	PXOR  X1, X12
 
 	// We can only store one byte at a time, since plaintext can be shorter than 16 bytes
 openSSETail16Store:
-	MOVQ T0, t3
-	MOVB t3, (oup)
-	PSRLDQ $1, T0
-	INCQ   oup
-	DECQ   inl
+	MOVQ   X12, R8
+	MOVB   R8, (DI)
+	PSRLDQ $0x01, X12
+	INCQ   DI
+	DECQ   BX
 	JNE    openSSETail16Store
-	ADDQ   t0, acc0; ADCQ t1, acc1; ADCQ $1, acc2
-	polyMul
+	ADDQ   R13, R10
+	ADCQ   R14, R11
+	ADCQ   $0x01, R12
+	MOVQ   (BP), AX
+	MOVQ   AX, R15
+	MULQ   R10
+	MOVQ   AX, R13
+	MOVQ   DX, R14
+	MOVQ   (BP), AX
+	MULQ   R11
+	IMULQ  R12, R15
+	ADDQ   AX, R14
+	ADCQ   DX, R15
+	MOVQ   8(BP), AX
+	MOVQ   AX, R8
+	MULQ   R10
+	ADDQ   AX, R14
+	ADCQ   $0x00, DX
+	MOVQ   DX, R10
+	MOVQ   8(BP), AX
+	MULQ   R11
+	ADDQ   AX, R15
+	ADCQ   $0x00, DX
+	IMULQ  R12, R8
+	ADDQ   R10, R15
+	ADCQ   DX, R8
+	MOVQ   R13, R10
+	MOVQ   R14, R11
+	MOVQ   R15, R12
+	ANDQ   $0x03, R12
+	MOVQ   R15, R13
+	ANDQ   $-4, R13
+	MOVQ   R8, R14
+	SHRQ   $0x02, R8, R15
+	SHRQ   $0x02, R8
+	ADDQ   R13, R10
+	ADCQ   R14, R11
+	ADCQ   $0x00, R12
+	ADDQ   R15, R10
+	ADCQ   R8, R11
+	ADCQ   $0x00, R12
 	JMP    openSSEFinalize
 
-// ----------------------------------------------------------------------------
-// Special optimization for the last 64 bytes of ciphertext
 openSSETail64:
-	// Need to decrypt up to 64 bytes - prepare single block
-	MOVO ·chacha20Constants<>(SB), A0; MOVO state1Store, B0; MOVO state2Store, C0; MOVO ctr3Store, D0; PADDL ·sseIncMask<>(SB), D0; MOVO D0, ctr0Store
-	XORQ itr2, itr2
-	MOVQ inl, itr1
-	CMPQ itr1, $16
-	JB   openSSETail64LoopB
+	MOVO  ·chacha20Constants<>+0(SB), X0
+	MOVO  32(BP), X3
+	MOVO  48(BP), X6
+	MOVO  128(BP), X9
+	PADDL ·sseIncMask<>+0(SB), X9
+	MOVO  X9, 80(BP)
+	XORQ  R9, R9
+	MOVQ  BX, CX
+	CMPQ  CX, $0x10
+	JB    openSSETail64LoopB
 
 openSSETail64LoopA:
-	// Perform ChaCha rounds, while hashing the remaining input
-	polyAdd(0(inp)(itr2*1))
-	polyMul
-	SUBQ $16, itr1
+	ADDQ  (SI)(R9*1), R10
+	ADCQ  8(SI)(R9*1), R11
+	ADCQ  $0x01, R12
+	MOVQ  (BP), AX
+	MOVQ  AX, R15
+	MULQ  R10
+	MOVQ  AX, R13
+	MOVQ  DX, R14
+	MOVQ  (BP), AX
+	MULQ  R11
+	IMULQ R12, R15
+	ADDQ  AX, R14
+	ADCQ  DX, R15
+	MOVQ  8(BP), AX
+	MOVQ  AX, R8
+	MULQ  R10
+	ADDQ  AX, R14
+	ADCQ  $0x00, DX
+	MOVQ  DX, R10
+	MOVQ  8(BP), AX
+	MULQ  R11
+	ADDQ  AX, R15
+	ADCQ  $0x00, DX
+	IMULQ R12, R8
+	ADDQ  R10, R15
+	ADCQ  DX, R8
+	MOVQ  R13, R10
+	MOVQ  R14, R11
+	MOVQ  R15, R12
+	ANDQ  $0x03, R12
+	MOVQ  R15, R13
+	ANDQ  $-4, R13
+	MOVQ  R8, R14
+	SHRQ  $0x02, R8, R15
+	SHRQ  $0x02, R8
+	ADDQ  R13, R10
+	ADCQ  R14, R11
+	ADCQ  $0x00, R12
+	ADDQ  R15, R10
+	ADCQ  R8, R11
+	ADCQ  $0x00, R12
+	SUBQ  $0x10, CX
 
 openSSETail64LoopB:
-	ADDQ          $16, itr2
-	chachaQR(A0, B0, C0, D0, T0)
-	shiftB0Left;  shiftC0Left; shiftD0Left
-	chachaQR(A0, B0, C0, D0, T0)
-	shiftB0Right; shiftC0Right; shiftD0Right
-
-	CMPQ itr1, $16
-	JAE  openSSETail64LoopA
-
-	CMPQ itr2, $160
-	JNE  openSSETail64LoopB
-
-	PADDL ·chacha20Constants<>(SB), A0; PADDL state1Store, B0; PADDL state2Store, C0; PADDL ctr0Store, D0
+	ADDQ  $0x10, R9
+	PADDD X3, X0
+	PXOR  X0, X9
+	ROL16(X9, X12)
+	PADDD X9, X6
+	PXOR  X6, X3
+	MOVO  X3, X12
+	PSLLL $0x0c, X12
+	PSRLL $0x14, X3
+	PXOR  X12, X3
+	PADDD X3, X0
+	PXOR  X0, X9
+	ROL8(X9, X12)
+	PADDD X9, X6
+	PXOR  X6, X3
+	MOVO  X3, X12
+	PSLLL $0x07, X12
+	PSRLL $0x19, X3
+	PXOR  X12, X3
+	BYTE  $0x66
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xdb
+	BYTE  $0x04
+	BYTE  $0x66
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xf6
+	BYTE  $0x08
+	BYTE  $0x66
+	BYTE  $0x45
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xc9
+	BYTE  $0x0c
+	PADDD X3, X0
+	PXOR  X0, X9
+	ROL16(X9, X12)
+	PADDD X9, X6
+	PXOR  X6, X3
+	MOVO  X3, X12
+	PSLLL $0x0c, X12
+	PSRLL $0x14, X3
+	PXOR  X12, X3
+	PADDD X3, X0
+	PXOR  X0, X9
+	ROL8(X9, X12)
+	PADDD X9, X6
+	PXOR  X6, X3
+	MOVO  X3, X12
+	PSLLL $0x07, X12
+	PSRLL $0x19, X3
+	PXOR  X12, X3
+	BYTE  $0x66
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xdb
+	BYTE  $0x0c
+	BYTE  $0x66
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xf6
+	BYTE  $0x08
+	BYTE  $0x66
+	BYTE  $0x45
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xc9
+	BYTE  $0x04
+	CMPQ  CX, $0x10
+	JAE   openSSETail64LoopA
+	CMPQ  R9, $0xa0
+	JNE   openSSETail64LoopB
+	PADDL ·chacha20Constants<>+0(SB), X0
+	PADDL 32(BP), X3
+	PADDL 48(BP), X6
+	PADDL 80(BP), X9
 
 openSSETail64DecLoop:
-	CMPQ  inl, $16
+	CMPQ  BX, $0x10
 	JB    openSSETail64DecLoopDone
-	SUBQ  $16, inl
-	MOVOU (inp), T0
-	PXOR  T0, A0
-	MOVOU A0, (oup)
-	LEAQ  16(inp), inp
-	LEAQ  16(oup), oup
-	MOVO  B0, A0
-	MOVO  C0, B0
-	MOVO  D0, C0
+	SUBQ  $0x10, BX
+	MOVOU (SI), X12
+	PXOR  X12, X0
+	MOVOU X0, (DI)
+	LEAQ  16(SI), SI
+	LEAQ  16(DI), DI
+	MOVO  X3, X0
+	MOVO  X6, X3
+	MOVO  X9, X6
 	JMP   openSSETail64DecLoop
 
 openSSETail64DecLoopDone:
-	MOVO A0, A1
+	MOVO X0, X1
 	JMP  openSSETail16
 
-// ----------------------------------------------------------------------------
-// Special optimization for the last 128 bytes of ciphertext
 openSSETail128:
-	// Need to decrypt up to 128 bytes - prepare two blocks
-	MOVO ·chacha20Constants<>(SB), A1; MOVO state1Store, B1; MOVO state2Store, C1; MOVO ctr3Store, D1; PADDL ·sseIncMask<>(SB), D1; MOVO D1, ctr0Store
-	MOVO A1, A0; MOVO B1, B0; MOVO C1, C0; MOVO D1, D0; PADDL ·sseIncMask<>(SB), D0; MOVO D0, ctr1Store
-	XORQ itr2, itr2
-	MOVQ inl, itr1
-	ANDQ $-16, itr1
+	MOVO  ·chacha20Constants<>+0(SB), X1
+	MOVO  32(BP), X4
+	MOVO  48(BP), X7
+	MOVO  128(BP), X10
+	PADDL ·sseIncMask<>+0(SB), X10
+	MOVO  X10, 80(BP)
+	MOVO  X1, X0
+	MOVO  X4, X3
+	MOVO  X7, X6
+	MOVO  X10, X9
+	PADDL ·sseIncMask<>+0(SB), X9
+	MOVO  X9, 96(BP)
+	XORQ  R9, R9
+	MOVQ  BX, CX
+	ANDQ  $-16, CX
 
 openSSETail128LoopA:
-	// Perform ChaCha rounds, while hashing the remaining input
-	polyAdd(0(inp)(itr2*1))
-	polyMul
+	ADDQ  (SI)(R9*1), R10
+	ADCQ  8(SI)(R9*1), R11
+	ADCQ  $0x01, R12
+	MOVQ  (BP), AX
+	MOVQ  AX, R15
+	MULQ  R10
+	MOVQ  AX, R13
+	MOVQ  DX, R14
+	MOVQ  (BP), AX
+	MULQ  R11
+	IMULQ R12, R15
+	ADDQ  AX, R14
+	ADCQ  DX, R15
+	MOVQ  8(BP), AX
+	MOVQ  AX, R8
+	MULQ  R10
+	ADDQ  AX, R14
+	ADCQ  $0x00, DX
+	MOVQ  DX, R10
+	MOVQ  8(BP), AX
+	MULQ  R11
+	ADDQ  AX, R15
+	ADCQ  $0x00, DX
+	IMULQ R12, R8
+	ADDQ  R10, R15
+	ADCQ  DX, R8
+	MOVQ  R13, R10
+	MOVQ  R14, R11
+	MOVQ  R15, R12
+	ANDQ  $0x03, R12
+	MOVQ  R15, R13
+	ANDQ  $-4, R13
+	MOVQ  R8, R14
+	SHRQ  $0x02, R8, R15
+	SHRQ  $0x02, R8
+	ADDQ  R13, R10
+	ADCQ  R14, R11
+	ADCQ  $0x00, R12
+	ADDQ  R15, R10
+	ADCQ  R8, R11
+	ADCQ  $0x00, R12
 
 openSSETail128LoopB:
-	ADDQ          $16, itr2
-	chachaQR(A0, B0, C0, D0, T0); chachaQR(A1, B1, C1, D1, T0)
-	shiftB0Left;  shiftC0Left; shiftD0Left
-	shiftB1Left;  shiftC1Left; shiftD1Left
-	chachaQR(A0, B0, C0, D0, T0); chachaQR(A1, B1, C1, D1, T0)
-	shiftB0Right; shiftC0Right; shiftD0Right
-	shiftB1Right; shiftC1Right; shiftD1Right
-
-	CMPQ itr2, itr1
-	JB   openSSETail128LoopA
-
-	CMPQ itr2, $160
-	JNE  openSSETail128LoopB
-
-	PADDL ·chacha20Constants<>(SB), A0; PADDL ·chacha20Constants<>(SB), A1
-	PADDL state1Store, B0; PADDL state1Store, B1
-	PADDL state2Store, C0; PADDL state2Store, C1
-	PADDL ctr1Store, D0; PADDL ctr0Store, D1
-
-	MOVOU (0*16)(inp), T0; MOVOU (1*16)(inp), T1; MOVOU (2*16)(inp), T2; MOVOU (3*16)(inp), T3
-	PXOR  T0, A1; PXOR T1, B1; PXOR T2, C1; PXOR T3, D1
-	MOVOU A1, (0*16)(oup); MOVOU B1, (1*16)(oup); MOVOU C1, (2*16)(oup); MOVOU D1, (3*16)(oup)
-
-	SUBQ $64, inl
-	LEAQ 64(inp), inp
-	LEAQ 64(oup), oup
-	JMP  openSSETail64DecLoop
-
-// ----------------------------------------------------------------------------
-// Special optimization for the last 192 bytes of ciphertext
+	ADDQ  $0x10, R9
+	PADDD X3, X0
+	PXOR  X0, X9
+	ROL16(X9, X12)
+	PADDD X9, X6
+	PXOR  X6, X3
+	MOVO  X3, X12
+	PSLLL $0x0c, X12
+	PSRLL $0x14, X3
+	PXOR  X12, X3
+	PADDD X3, X0
+	PXOR  X0, X9
+	ROL8(X9, X12)
+	PADDD X9, X6
+	PXOR  X6, X3
+	MOVO  X3, X12
+	PSLLL $0x07, X12
+	PSRLL $0x19, X3
+	PXOR  X12, X3
+	PADDD X4, X1
+	PXOR  X1, X10
+	ROL16(X10, X12)
+	PADDD X10, X7
+	PXOR  X7, X4
+	MOVO  X4, X12
+	PSLLL $0x0c, X12
+	PSRLL $0x14, X4
+	PXOR  X12, X4
+	PADDD X4, X1
+	PXOR  X1, X10
+	ROL8(X10, X12)
+	PADDD X10, X7
+	PXOR  X7, X4
+	MOVO  X4, X12
+	PSLLL $0x07, X12
+	PSRLL $0x19, X4
+	PXOR  X12, X4
+	BYTE  $0x66
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xdb
+	BYTE  $0x04
+	BYTE  $0x66
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xf6
+	BYTE  $0x08
+	BYTE  $0x66
+	BYTE  $0x45
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xc9
+	BYTE  $0x0c
+	BYTE  $0x66
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xe4
+	BYTE  $0x04
+	BYTE  $0x66
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xff
+	BYTE  $0x08
+	BYTE  $0x66
+	BYTE  $0x45
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xd2
+	BYTE  $0x0c
+	PADDD X3, X0
+	PXOR  X0, X9
+	ROL16(X9, X12)
+	PADDD X9, X6
+	PXOR  X6, X3
+	MOVO  X3, X12
+	PSLLL $0x0c, X12
+	PSRLL $0x14, X3
+	PXOR  X12, X3
+	PADDD X3, X0
+	PXOR  X0, X9
+	ROL8(X9, X12)
+	PADDD X9, X6
+	PXOR  X6, X3
+	MOVO  X3, X12
+	PSLLL $0x07, X12
+	PSRLL $0x19, X3
+	PXOR  X12, X3
+	PADDD X4, X1
+	PXOR  X1, X10
+	ROL16(X10, X12)
+	PADDD X10, X7
+	PXOR  X7, X4
+	MOVO  X4, X12
+	PSLLL $0x0c, X12
+	PSRLL $0x14, X4
+	PXOR  X12, X4
+	PADDD X4, X1
+	PXOR  X1, X10
+	ROL8(X10, X12)
+	PADDD X10, X7
+	PXOR  X7, X4
+	MOVO  X4, X12
+	PSLLL $0x07, X12
+	PSRLL $0x19, X4
+	PXOR  X12, X4
+	BYTE  $0x66
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xdb
+	BYTE  $0x0c
+	BYTE  $0x66
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xf6
+	BYTE  $0x08
+	BYTE  $0x66
+	BYTE  $0x45
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xc9
+	BYTE  $0x04
+	BYTE  $0x66
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xe4
+	BYTE  $0x0c
+	BYTE  $0x66
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xff
+	BYTE  $0x08
+	BYTE  $0x66
+	BYTE  $0x45
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xd2
+	BYTE  $0x04
+	CMPQ  R9, CX
+	JB    openSSETail128LoopA
+	CMPQ  R9, $0xa0
+	JNE   openSSETail128LoopB
+	PADDL ·chacha20Constants<>+0(SB), X0
+	PADDL ·chacha20Constants<>+0(SB), X1
+	PADDL 32(BP), X3
+	PADDL 32(BP), X4
+	PADDL 48(BP), X6
+	PADDL 48(BP), X7
+	PADDL 96(BP), X9
+	PADDL 80(BP), X10
+	MOVOU (SI), X12
+	MOVOU 16(SI), X13
+	MOVOU 32(SI), X14
+	MOVOU 48(SI), X15
+	PXOR  X12, X1
+	PXOR  X13, X4
+	PXOR  X14, X7
+	PXOR  X15, X10
+	MOVOU X1, (DI)
+	MOVOU X4, 16(DI)
+	MOVOU X7, 32(DI)
+	MOVOU X10, 48(DI)
+	SUBQ  $0x40, BX
+	LEAQ  64(SI), SI
+	LEAQ  64(DI), DI
+	JMP   openSSETail64DecLoop
+
 openSSETail192:
-	// Need to decrypt up to 192 bytes - prepare three blocks
-	MOVO ·chacha20Constants<>(SB), A2; MOVO state1Store, B2; MOVO state2Store, C2; MOVO ctr3Store, D2; PADDL ·sseIncMask<>(SB), D2; MOVO D2, ctr0Store
-	MOVO A2, A1; MOVO B2, B1; MOVO C2, C1; MOVO D2, D1; PADDL ·sseIncMask<>(SB), D1; MOVO D1, ctr1Store
-	MOVO A1, A0; MOVO B1, B0; MOVO C1, C0; MOVO D1, D0; PADDL ·sseIncMask<>(SB), D0; MOVO D0, ctr2Store
-
-	MOVQ    inl, itr1
-	MOVQ    $160, itr2
-	CMPQ    itr1, $160
-	CMOVQGT itr2, itr1
-	ANDQ    $-16, itr1
-	XORQ    itr2, itr2
+	MOVO    ·chacha20Constants<>+0(SB), X2
+	MOVO    32(BP), X5
+	MOVO    48(BP), X8
+	MOVO    128(BP), X11
+	PADDL   ·sseIncMask<>+0(SB), X11
+	MOVO    X11, 80(BP)
+	MOVO    X2, X1
+	MOVO    X5, X4
+	MOVO    X8, X7
+	MOVO    X11, X10
+	PADDL   ·sseIncMask<>+0(SB), X10
+	MOVO    X10, 96(BP)
+	MOVO    X1, X0
+	MOVO    X4, X3
+	MOVO    X7, X6
+	MOVO    X10, X9
+	PADDL   ·sseIncMask<>+0(SB), X9
+	MOVO    X9, 112(BP)
+	MOVQ    BX, CX
+	MOVQ    $0x000000a0, R9
+	CMPQ    CX, $0xa0
+	CMOVQGT R9, CX
+	ANDQ    $-16, CX
+	XORQ    R9, R9
 
 openSSLTail192LoopA:
-	// Perform ChaCha rounds, while hashing the remaining input
-	polyAdd(0(inp)(itr2*1))
-	polyMul
+	ADDQ  (SI)(R9*1), R10
+	ADCQ  8(SI)(R9*1), R11
+	ADCQ  $0x01, R12
+	MOVQ  (BP), AX
+	MOVQ  AX, R15
+	MULQ  R10
+	MOVQ  AX, R13
+	MOVQ  DX, R14
+	MOVQ  (BP), AX
+	MULQ  R11
+	IMULQ R12, R15
+	ADDQ  AX, R14
+	ADCQ  DX, R15
+	MOVQ  8(BP), AX
+	MOVQ  AX, R8
+	MULQ  R10
+	ADDQ  AX, R14
+	ADCQ  $0x00, DX
+	MOVQ  DX, R10
+	MOVQ  8(BP), AX
+	MULQ  R11
+	ADDQ  AX, R15
+	ADCQ  $0x00, DX
+	IMULQ R12, R8
+	ADDQ  R10, R15
+	ADCQ  DX, R8
+	MOVQ  R13, R10
+	MOVQ  R14, R11
+	MOVQ  R15, R12
+	ANDQ  $0x03, R12
+	MOVQ  R15, R13
+	ANDQ  $-4, R13
+	MOVQ  R8, R14
+	SHRQ  $0x02, R8, R15
+	SHRQ  $0x02, R8
+	ADDQ  R13, R10
+	ADCQ  R14, R11
+	ADCQ  $0x00, R12
+	ADDQ  R15, R10
+	ADCQ  R8, R11
+	ADCQ  $0x00, R12
 
 openSSLTail192LoopB:
-	ADDQ         $16, itr2
-	chachaQR(A0, B0, C0, D0, T0); chachaQR(A1, B1, C1, D1, T0); chachaQR(A2, B2, C2, D2, T0)
-	shiftB0Left; shiftC0Left; shiftD0Left
-	shiftB1Left; shiftC1Left; shiftD1Left
-	shiftB2Left; shiftC2Left; shiftD2Left
-
-	chachaQR(A0, B0, C0, D0, T0); chachaQR(A1, B1, C1, D1, T0); chachaQR(A2, B2, C2, D2, T0)
-	shiftB0Right; shiftC0Right; shiftD0Right
-	shiftB1Right; shiftC1Right; shiftD1Right
-	shiftB2Right; shiftC2Right; shiftD2Right
-
-	CMPQ itr2, itr1
-	JB   openSSLTail192LoopA
-
-	CMPQ itr2, $160
-	JNE  openSSLTail192LoopB
-
-	CMPQ inl, $176
-	JB   openSSLTail192Store
-
-	polyAdd(160(inp))
-	polyMul
-
-	CMPQ inl, $192
-	JB   openSSLTail192Store
-
-	polyAdd(176(inp))
-	polyMul
+	ADDQ  $0x10, R9
+	PADDD X3, X0
+	PXOR  X0, X9
+	ROL16(X9, X12)
+	PADDD X9, X6
+	PXOR  X6, X3
+	MOVO  X3, X12
+	PSLLL $0x0c, X12
+	PSRLL $0x14, X3
+	PXOR  X12, X3
+	PADDD X3, X0
+	PXOR  X0, X9
+	ROL8(X9, X12)
+	PADDD X9, X6
+	PXOR  X6, X3
+	MOVO  X3, X12
+	PSLLL $0x07, X12
+	PSRLL $0x19, X3
+	PXOR  X12, X3
+	PADDD X4, X1
+	PXOR  X1, X10
+	ROL16(X10, X12)
+	PADDD X10, X7
+	PXOR  X7, X4
+	MOVO  X4, X12
+	PSLLL $0x0c, X12
+	PSRLL $0x14, X4
+	PXOR  X12, X4
+	PADDD X4, X1
+	PXOR  X1, X10
+	ROL8(X10, X12)
+	PADDD X10, X7
+	PXOR  X7, X4
+	MOVO  X4, X12
+	PSLLL $0x07, X12
+	PSRLL $0x19, X4
+	PXOR  X12, X4
+	PADDD X5, X2
+	PXOR  X2, X11
+	ROL16(X11, X12)
+	PADDD X11, X8
+	PXOR  X8, X5
+	MOVO  X5, X12
+	PSLLL $0x0c, X12
+	PSRLL $0x14, X5
+	PXOR  X12, X5
+	PADDD X5, X2
+	PXOR  X2, X11
+	ROL8(X11, X12)
+	PADDD X11, X8
+	PXOR  X8, X5
+	MOVO  X5, X12
+	PSLLL $0x07, X12
+	PSRLL $0x19, X5
+	PXOR  X12, X5
+	BYTE  $0x66
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xdb
+	BYTE  $0x04
+	BYTE  $0x66
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xf6
+	BYTE  $0x08
+	BYTE  $0x66
+	BYTE  $0x45
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xc9
+	BYTE  $0x0c
+	BYTE  $0x66
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xe4
+	BYTE  $0x04
+	BYTE  $0x66
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xff
+	BYTE  $0x08
+	BYTE  $0x66
+	BYTE  $0x45
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xd2
+	BYTE  $0x0c
+	BYTE  $0x66
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xed
+	BYTE  $0x04
+	BYTE  $0x66
+	BYTE  $0x45
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xc0
+	BYTE  $0x08
+	BYTE  $0x66
+	BYTE  $0x45
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xdb
+	BYTE  $0x0c
+	PADDD X3, X0
+	PXOR  X0, X9
+	ROL16(X9, X12)
+	PADDD X9, X6
+	PXOR  X6, X3
+	MOVO  X3, X12
+	PSLLL $0x0c, X12
+	PSRLL $0x14, X3
+	PXOR  X12, X3
+	PADDD X3, X0
+	PXOR  X0, X9
+	ROL8(X9, X12)
+	PADDD X9, X6
+	PXOR  X6, X3
+	MOVO  X3, X12
+	PSLLL $0x07, X12
+	PSRLL $0x19, X3
+	PXOR  X12, X3
+	PADDD X4, X1
+	PXOR  X1, X10
+	ROL16(X10, X12)
+	PADDD X10, X7
+	PXOR  X7, X4
+	MOVO  X4, X12
+	PSLLL $0x0c, X12
+	PSRLL $0x14, X4
+	PXOR  X12, X4
+	PADDD X4, X1
+	PXOR  X1, X10
+	ROL8(X10, X12)
+	PADDD X10, X7
+	PXOR  X7, X4
+	MOVO  X4, X12
+	PSLLL $0x07, X12
+	PSRLL $0x19, X4
+	PXOR  X12, X4
+	PADDD X5, X2
+	PXOR  X2, X11
+	ROL16(X11, X12)
+	PADDD X11, X8
+	PXOR  X8, X5
+	MOVO  X5, X12
+	PSLLL $0x0c, X12
+	PSRLL $0x14, X5
+	PXOR  X12, X5
+	PADDD X5, X2
+	PXOR  X2, X11
+	ROL8(X11, X12)
+	PADDD X11, X8
+	PXOR  X8, X5
+	MOVO  X5, X12
+	PSLLL $0x07, X12
+	PSRLL $0x19, X5
+	PXOR  X12, X5
+	BYTE  $0x66
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xdb
+	BYTE  $0x0c
+	BYTE  $0x66
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xf6
+	BYTE  $0x08
+	BYTE  $0x66
+	BYTE  $0x45
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xc9
+	BYTE  $0x04
+	BYTE  $0x66
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xe4
+	BYTE  $0x0c
+	BYTE  $0x66
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xff
+	BYTE  $0x08
+	BYTE  $0x66
+	BYTE  $0x45
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xd2
+	BYTE  $0x04
+	BYTE  $0x66
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xed
+	BYTE  $0x0c
+	BYTE  $0x66
+	BYTE  $0x45
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xc0
+	BYTE  $0x08
+	BYTE  $0x66
+	BYTE  $0x45
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xdb
+	BYTE  $0x04
+	CMPQ  R9, CX
+	JB    openSSLTail192LoopA
+	CMPQ  R9, $0xa0
+	JNE   openSSLTail192LoopB
+	CMPQ  BX, $0xb0
+	JB    openSSLTail192Store
+	ADDQ  160(SI), R10
+	ADCQ  168(SI), R11
+	ADCQ  $0x01, R12
+	MOVQ  (BP), AX
+	MOVQ  AX, R15
+	MULQ  R10
+	MOVQ  AX, R13
+	MOVQ  DX, R14
+	MOVQ  (BP), AX
+	MULQ  R11
+	IMULQ R12, R15
+	ADDQ  AX, R14
+	ADCQ  DX, R15
+	MOVQ  8(BP), AX
+	MOVQ  AX, R8
+	MULQ  R10
+	ADDQ  AX, R14
+	ADCQ  $0x00, DX
+	MOVQ  DX, R10
+	MOVQ  8(BP), AX
+	MULQ  R11
+	ADDQ  AX, R15
+	ADCQ  $0x00, DX
+	IMULQ R12, R8
+	ADDQ  R10, R15
+	ADCQ  DX, R8
+	MOVQ  R13, R10
+	MOVQ  R14, R11
+	MOVQ  R15, R12
+	ANDQ  $0x03, R12
+	MOVQ  R15, R13
+	ANDQ  $-4, R13
+	MOVQ  R8, R14
+	SHRQ  $0x02, R8, R15
+	SHRQ  $0x02, R8
+	ADDQ  R13, R10
+	ADCQ  R14, R11
+	ADCQ  $0x00, R12
+	ADDQ  R15, R10
+	ADCQ  R8, R11
+	ADCQ  $0x00, R12
+	CMPQ  BX, $0xc0
+	JB    openSSLTail192Store
+	ADDQ  176(SI), R10
+	ADCQ  184(SI), R11
+	ADCQ  $0x01, R12
+	MOVQ  (BP), AX
+	MOVQ  AX, R15
+	MULQ  R10
+	MOVQ  AX, R13
+	MOVQ  DX, R14
+	MOVQ  (BP), AX
+	MULQ  R11
+	IMULQ R12, R15
+	ADDQ  AX, R14
+	ADCQ  DX, R15
+	MOVQ  8(BP), AX
+	MOVQ  AX, R8
+	MULQ  R10
+	ADDQ  AX, R14
+	ADCQ  $0x00, DX
+	MOVQ  DX, R10
+	MOVQ  8(BP), AX
+	MULQ  R11
+	ADDQ  AX, R15
+	ADCQ  $0x00, DX
+	IMULQ R12, R8
+	ADDQ  R10, R15
+	ADCQ  DX, R8
+	MOVQ  R13, R10
+	MOVQ  R14, R11
+	MOVQ  R15, R12
+	ANDQ  $0x03, R12
+	MOVQ  R15, R13
+	ANDQ  $-4, R13
+	MOVQ  R8, R14
+	SHRQ  $0x02, R8, R15
+	SHRQ  $0x02, R8
+	ADDQ  R13, R10
+	ADCQ  R14, R11
+	ADCQ  $0x00, R12
+	ADDQ  R15, R10
+	ADCQ  R8, R11
+	ADCQ  $0x00, R12
 
 openSSLTail192Store:
-	PADDL ·chacha20Constants<>(SB), A0; PADDL ·chacha20Constants<>(SB), A1; PADDL ·chacha20Constants<>(SB), A2
-	PADDL state1Store, B0; PADDL state1Store, B1; PADDL state1Store, B2
-	PADDL state2Store, C0; PADDL state2Store, C1; PADDL state2Store, C2
-	PADDL ctr2Store, D0; PADDL ctr1Store, D1; PADDL ctr0Store, D2
-
-	MOVOU (0*16)(inp), T0; MOVOU (1*16)(inp), T1; MOVOU (2*16)(inp), T2; MOVOU (3*16)(inp), T3
-	PXOR  T0, A2; PXOR T1, B2; PXOR T2, C2; PXOR T3, D2
-	MOVOU A2, (0*16)(oup); MOVOU B2, (1*16)(oup); MOVOU C2, (2*16)(oup); MOVOU D2, (3*16)(oup)
-
-	MOVOU (4*16)(inp), T0; MOVOU (5*16)(inp), T1; MOVOU (6*16)(inp), T2; MOVOU (7*16)(inp), T3
-	PXOR  T0, A1; PXOR T1, B1; PXOR T2, C1; PXOR T3, D1
-	MOVOU A1, (4*16)(oup); MOVOU B1, (5*16)(oup); MOVOU C1, (6*16)(oup); MOVOU D1, (7*16)(oup)
-
-	SUBQ $128, inl
-	LEAQ 128(inp), inp
-	LEAQ 128(oup), oup
-	JMP  openSSETail64DecLoop
-
-// ----------------------------------------------------------------------------
-// Special optimization for the last 256 bytes of ciphertext
+	PADDL ·chacha20Constants<>+0(SB), X0
+	PADDL ·chacha20Constants<>+0(SB), X1
+	PADDL ·chacha20Constants<>+0(SB), X2
+	PADDL 32(BP), X3
+	PADDL 32(BP), X4
+	PADDL 32(BP), X5
+	PADDL 48(BP), X6
+	PADDL 48(BP), X7
+	PADDL 48(BP), X8
+	PADDL 112(BP), X9
+	PADDL 96(BP), X10
+	PADDL 80(BP), X11
+	MOVOU (SI), X12
+	MOVOU 16(SI), X13
+	MOVOU 32(SI), X14
+	MOVOU 48(SI), X15
+	PXOR  X12, X2
+	PXOR  X13, X5
+	PXOR  X14, X8
+	PXOR  X15, X11
+	MOVOU X2, (DI)
+	MOVOU X5, 16(DI)
+	MOVOU X8, 32(DI)
+	MOVOU X11, 48(DI)
+	MOVOU 64(SI), X12
+	MOVOU 80(SI), X13
+	MOVOU 96(SI), X14
+	MOVOU 112(SI), X15
+	PXOR  X12, X1
+	PXOR  X13, X4
+	PXOR  X14, X7
+	PXOR  X15, X10
+	MOVOU X1, 64(DI)
+	MOVOU X4, 80(DI)
+	MOVOU X7, 96(DI)
+	MOVOU X10, 112(DI)
+	SUBQ  $0x80, BX
+	LEAQ  128(SI), SI
+	LEAQ  128(DI), DI
+	JMP   openSSETail64DecLoop
+
 openSSETail256:
-	// Need to decrypt up to 256 bytes - prepare four blocks
-	MOVO ·chacha20Constants<>(SB), A0; MOVO state1Store, B0; MOVO state2Store, C0; MOVO ctr3Store, D0; PADDL ·sseIncMask<>(SB), D0
-	MOVO A0, A1; MOVO B0, B1; MOVO C0, C1; MOVO D0, D1; PADDL ·sseIncMask<>(SB), D1
-	MOVO A1, A2; MOVO B1, B2; MOVO C1, C2; MOVO D1, D2; PADDL ·sseIncMask<>(SB), D2
-	MOVO A2, A3; MOVO B2, B3; MOVO C2, C3; MOVO D2, D3; PADDL ·sseIncMask<>(SB), D3
+	MOVO  ·chacha20Constants<>+0(SB), X0
+	MOVO  32(BP), X3
+	MOVO  48(BP), X6
+	MOVO  128(BP), X9
+	PADDL ·sseIncMask<>+0(SB), X9
+	MOVO  X0, X1
+	MOVO  X3, X4
+	MOVO  X6, X7
+	MOVO  X9, X10
+	PADDL ·sseIncMask<>+0(SB), X10
+	MOVO  X1, X2
+	MOVO  X4, X5
+	MOVO  X7, X8
+	MOVO  X10, X11
+	PADDL ·sseIncMask<>+0(SB), X11
+	MOVO  X2, X12
+	MOVO  X5, X13
+	MOVO  X8, X14
+	MOVO  X11, X15
+	PADDL ·sseIncMask<>+0(SB), X15
 
 	// Store counters
-	MOVO D0, ctr0Store; MOVO D1, ctr1Store; MOVO D2, ctr2Store; MOVO D3, ctr3Store
-	XORQ itr2, itr2
+	MOVO X9, 80(BP)
+	MOVO X10, 96(BP)
+	MOVO X11, 112(BP)
+	MOVO X15, 128(BP)
+	XORQ R9, R9
 
 openSSETail256Loop:
-	// This loop inteleaves 8 ChaCha quarter rounds with 1 poly multiplication
-	polyAdd(0(inp)(itr2*1))
-	MOVO          C3, tmpStore
-	chachaQR(A0, B0, C0, D0, C3); chachaQR(A1, B1, C1, D1, C3); chachaQR(A2, B2, C2, D2, C3)
-	MOVO          tmpStore, C3
-	MOVO          C1, tmpStore
-	chachaQR(A3, B3, C3, D3, C1)
-	MOVO          tmpStore, C1
-	shiftB0Left;  shiftB1Left; shiftB2Left; shiftB3Left
-	shiftC0Left;  shiftC1Left; shiftC2Left; shiftC3Left
-	shiftD0Left;  shiftD1Left; shiftD2Left; shiftD3Left
-	polyMulStage1
-	polyMulStage2
-	MOVO          C3, tmpStore
-	chachaQR(A0, B0, C0, D0, C3); chachaQR(A1, B1, C1, D1, C3); chachaQR(A2, B2, C2, D2, C3)
-	MOVO          tmpStore, C3
-	MOVO          C1, tmpStore
-	chachaQR(A3, B3, C3, D3, C1)
-	MOVO          tmpStore, C1
-	polyMulStage3
-	polyMulReduceStage
-	shiftB0Right; shiftB1Right; shiftB2Right; shiftB3Right
-	shiftC0Right; shiftC1Right; shiftC2Right; shiftC3Right
-	shiftD0Right; shiftD1Right; shiftD2Right; shiftD3Right
-	ADDQ          $2*8, itr2
-	CMPQ          itr2, $160
-	JB            openSSETail256Loop
-	MOVQ          inl, itr1
-	ANDQ          $-16, itr1
+	ADDQ  (SI)(R9*1), R10
+	ADCQ  8(SI)(R9*1), R11
+	ADCQ  $0x01, R12
+	MOVO  X14, 64(BP)
+	PADDD X3, X0
+	PXOR  X0, X9
+	ROL16(X9, X14)
+	PADDD X9, X6
+	PXOR  X6, X3
+	MOVO  X3, X14
+	PSLLL $0x0c, X14
+	PSRLL $0x14, X3
+	PXOR  X14, X3
+	PADDD X3, X0
+	PXOR  X0, X9
+	ROL8(X9, X14)
+	PADDD X9, X6
+	PXOR  X6, X3
+	MOVO  X3, X14
+	PSLLL $0x07, X14
+	PSRLL $0x19, X3
+	PXOR  X14, X3
+	PADDD X4, X1
+	PXOR  X1, X10
+	ROL16(X10, X14)
+	PADDD X10, X7
+	PXOR  X7, X4
+	MOVO  X4, X14
+	PSLLL $0x0c, X14
+	PSRLL $0x14, X4
+	PXOR  X14, X4
+	PADDD X4, X1
+	PXOR  X1, X10
+	ROL8(X10, X14)
+	PADDD X10, X7
+	PXOR  X7, X4
+	MOVO  X4, X14
+	PSLLL $0x07, X14
+	PSRLL $0x19, X4
+	PXOR  X14, X4
+	PADDD X5, X2
+	PXOR  X2, X11
+	ROL16(X11, X14)
+	PADDD X11, X8
+	PXOR  X8, X5
+	MOVO  X5, X14
+	PSLLL $0x0c, X14
+	PSRLL $0x14, X5
+	PXOR  X14, X5
+	PADDD X5, X2
+	PXOR  X2, X11
+	ROL8(X11, X14)
+	PADDD X11, X8
+	PXOR  X8, X5
+	MOVO  X5, X14
+	PSLLL $0x07, X14
+	PSRLL $0x19, X5
+	PXOR  X14, X5
+	MOVO  64(BP), X14
+	MOVO  X7, 64(BP)
+	PADDD X13, X12
+	PXOR  X12, X15
+	ROL16(X15, X7)
+	PADDD X15, X14
+	PXOR  X14, X13
+	MOVO  X13, X7
+	PSLLL $0x0c, X7
+	PSRLL $0x14, X13
+	PXOR  X7, X13
+	PADDD X13, X12
+	PXOR  X12, X15
+	ROL8(X15, X7)
+	PADDD X15, X14
+	PXOR  X14, X13
+	MOVO  X13, X7
+	PSLLL $0x07, X7
+	PSRLL $0x19, X13
+	PXOR  X7, X13
+	MOVO  64(BP), X7
+	BYTE  $0x66
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xdb
+	BYTE  $0x04
+	BYTE  $0x66
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xe4
+	BYTE  $0x04
+	BYTE  $0x66
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xed
+	BYTE  $0x04
+	BYTE  $0x66
+	BYTE  $0x45
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xed
+	BYTE  $0x04
+	BYTE  $0x66
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xf6
+	BYTE  $0x08
+	BYTE  $0x66
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xff
+	BYTE  $0x08
+	BYTE  $0x66
+	BYTE  $0x45
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xc0
+	BYTE  $0x08
+	BYTE  $0x66
+	BYTE  $0x45
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xf6
+	BYTE  $0x08
+	BYTE  $0x66
+	BYTE  $0x45
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xc9
+	BYTE  $0x0c
+	BYTE  $0x66
+	BYTE  $0x45
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xd2
+	BYTE  $0x0c
+	BYTE  $0x66
+	BYTE  $0x45
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xdb
+	BYTE  $0x0c
+	BYTE  $0x66
+	BYTE  $0x45
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xff
+	BYTE  $0x0c
+	MOVQ  (BP), AX
+	MOVQ  AX, R15
+	MULQ  R10
+	MOVQ  AX, R13
+	MOVQ  DX, R14
+	MOVQ  (BP), AX
+	MULQ  R11
+	IMULQ R12, R15
+	ADDQ  AX, R14
+	ADCQ  DX, R15
+	MOVQ  8(BP), AX
+	MOVQ  AX, R8
+	MULQ  R10
+	ADDQ  AX, R14
+	ADCQ  $0x00, DX
+	MOVQ  DX, R10
+	MOVQ  8(BP), AX
+	MULQ  R11
+	ADDQ  AX, R15
+	ADCQ  $0x00, DX
+	MOVO  X14, 64(BP)
+	PADDD X3, X0
+	PXOR  X0, X9
+	ROL16(X9, X14)
+	PADDD X9, X6
+	PXOR  X6, X3
+	MOVO  X3, X14
+	PSLLL $0x0c, X14
+	PSRLL $0x14, X3
+	PXOR  X14, X3
+	PADDD X3, X0
+	PXOR  X0, X9
+	ROL8(X9, X14)
+	PADDD X9, X6
+	PXOR  X6, X3
+	MOVO  X3, X14
+	PSLLL $0x07, X14
+	PSRLL $0x19, X3
+	PXOR  X14, X3
+	PADDD X4, X1
+	PXOR  X1, X10
+	ROL16(X10, X14)
+	PADDD X10, X7
+	PXOR  X7, X4
+	MOVO  X4, X14
+	PSLLL $0x0c, X14
+	PSRLL $0x14, X4
+	PXOR  X14, X4
+	PADDD X4, X1
+	PXOR  X1, X10
+	ROL8(X10, X14)
+	PADDD X10, X7
+	PXOR  X7, X4
+	MOVO  X4, X14
+	PSLLL $0x07, X14
+	PSRLL $0x19, X4
+	PXOR  X14, X4
+	PADDD X5, X2
+	PXOR  X2, X11
+	ROL16(X11, X14)
+	PADDD X11, X8
+	PXOR  X8, X5
+	MOVO  X5, X14
+	PSLLL $0x0c, X14
+	PSRLL $0x14, X5
+	PXOR  X14, X5
+	PADDD X5, X2
+	PXOR  X2, X11
+	ROL8(X11, X14)
+	PADDD X11, X8
+	PXOR  X8, X5
+	MOVO  X5, X14
+	PSLLL $0x07, X14
+	PSRLL $0x19, X5
+	PXOR  X14, X5
+	MOVO  64(BP), X14
+	MOVO  X7, 64(BP)
+	PADDD X13, X12
+	PXOR  X12, X15
+	ROL16(X15, X7)
+	PADDD X15, X14
+	PXOR  X14, X13
+	MOVO  X13, X7
+	PSLLL $0x0c, X7
+	PSRLL $0x14, X13
+	PXOR  X7, X13
+	PADDD X13, X12
+	PXOR  X12, X15
+	ROL8(X15, X7)
+	PADDD X15, X14
+	PXOR  X14, X13
+	MOVO  X13, X7
+	PSLLL $0x07, X7
+	PSRLL $0x19, X13
+	PXOR  X7, X13
+	MOVO  64(BP), X7
+	IMULQ R12, R8
+	ADDQ  R10, R15
+	ADCQ  DX, R8
+	MOVQ  R13, R10
+	MOVQ  R14, R11
+	MOVQ  R15, R12
+	ANDQ  $0x03, R12
+	MOVQ  R15, R13
+	ANDQ  $-4, R13
+	MOVQ  R8, R14
+	SHRQ  $0x02, R8, R15
+	SHRQ  $0x02, R8
+	ADDQ  R13, R10
+	ADCQ  R14, R11
+	ADCQ  $0x00, R12
+	ADDQ  R15, R10
+	ADCQ  R8, R11
+	ADCQ  $0x00, R12
+	BYTE  $0x66
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xdb
+	BYTE  $0x0c
+	BYTE  $0x66
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xe4
+	BYTE  $0x0c
+	BYTE  $0x66
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xed
+	BYTE  $0x0c
+	BYTE  $0x66
+	BYTE  $0x45
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xed
+	BYTE  $0x0c
+	BYTE  $0x66
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xf6
+	BYTE  $0x08
+	BYTE  $0x66
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xff
+	BYTE  $0x08
+	BYTE  $0x66
+	BYTE  $0x45
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xc0
+	BYTE  $0x08
+	BYTE  $0x66
+	BYTE  $0x45
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xf6
+	BYTE  $0x08
+	BYTE  $0x66
+	BYTE  $0x45
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xc9
+	BYTE  $0x04
+	BYTE  $0x66
+	BYTE  $0x45
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xd2
+	BYTE  $0x04
+	BYTE  $0x66
+	BYTE  $0x45
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xdb
+	BYTE  $0x04
+	BYTE  $0x66
+	BYTE  $0x45
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xff
+	BYTE  $0x04
+	ADDQ  $0x10, R9
+	CMPQ  R9, $0xa0
+	JB    openSSETail256Loop
+	MOVQ  BX, CX
+	ANDQ  $-16, CX
 
 openSSETail256HashLoop:
-	polyAdd(0(inp)(itr2*1))
-	polyMul
-	ADDQ $2*8, itr2
-	CMPQ itr2, itr1
-	JB   openSSETail256HashLoop
+	ADDQ  (SI)(R9*1), R10
+	ADCQ  8(SI)(R9*1), R11
+	ADCQ  $0x01, R12
+	MOVQ  (BP), AX
+	MOVQ  AX, R15
+	MULQ  R10
+	MOVQ  AX, R13
+	MOVQ  DX, R14
+	MOVQ  (BP), AX
+	MULQ  R11
+	IMULQ R12, R15
+	ADDQ  AX, R14
+	ADCQ  DX, R15
+	MOVQ  8(BP), AX
+	MOVQ  AX, R8
+	MULQ  R10
+	ADDQ  AX, R14
+	ADCQ  $0x00, DX
+	MOVQ  DX, R10
+	MOVQ  8(BP), AX
+	MULQ  R11
+	ADDQ  AX, R15
+	ADCQ  $0x00, DX
+	IMULQ R12, R8
+	ADDQ  R10, R15
+	ADCQ  DX, R8
+	MOVQ  R13, R10
+	MOVQ  R14, R11
+	MOVQ  R15, R12
+	ANDQ  $0x03, R12
+	MOVQ  R15, R13
+	ANDQ  $-4, R13
+	MOVQ  R8, R14
+	SHRQ  $0x02, R8, R15
+	SHRQ  $0x02, R8
+	ADDQ  R13, R10
+	ADCQ  R14, R11
+	ADCQ  $0x00, R12
+	ADDQ  R15, R10
+	ADCQ  R8, R11
+	ADCQ  $0x00, R12
+	ADDQ  $0x10, R9
+	CMPQ  R9, CX
+	JB    openSSETail256HashLoop
 
 	// Add in the state
-	PADDD ·chacha20Constants<>(SB), A0; PADDD ·chacha20Constants<>(SB), A1; PADDD ·chacha20Constants<>(SB), A2; PADDD ·chacha20Constants<>(SB), A3
-	PADDD state1Store, B0; PADDD state1Store, B1; PADDD state1Store, B2; PADDD state1Store, B3
-	PADDD state2Store, C0; PADDD state2Store, C1; PADDD state2Store, C2; PADDD state2Store, C3
-	PADDD ctr0Store, D0; PADDD ctr1Store, D1; PADDD ctr2Store, D2; PADDD ctr3Store, D3
-	MOVO  D3, tmpStore
+	PADDD ·chacha20Constants<>+0(SB), X0
+	PADDD ·chacha20Constants<>+0(SB), X1
+	PADDD ·chacha20Constants<>+0(SB), X2
+	PADDD ·chacha20Constants<>+0(SB), X12
+	PADDD 32(BP), X3
+	PADDD 32(BP), X4
+	PADDD 32(BP), X5
+	PADDD 32(BP), X13
+	PADDD 48(BP), X6
+	PADDD 48(BP), X7
+	PADDD 48(BP), X8
+	PADDD 48(BP), X14
+	PADDD 80(BP), X9
+	PADDD 96(BP), X10
+	PADDD 112(BP), X11
+	PADDD 128(BP), X15
+	MOVO  X15, 64(BP)
 
 	// Load - xor - store
-	MOVOU (0*16)(inp), D3; PXOR D3, A0
-	MOVOU (1*16)(inp), D3; PXOR D3, B0
-	MOVOU (2*16)(inp), D3; PXOR D3, C0
-	MOVOU (3*16)(inp), D3; PXOR D3, D0
-	MOVOU A0, (0*16)(oup)
-	MOVOU B0, (1*16)(oup)
-	MOVOU C0, (2*16)(oup)
-	MOVOU D0, (3*16)(oup)
-	MOVOU (4*16)(inp), A0; MOVOU (5*16)(inp), B0; MOVOU (6*16)(inp), C0; MOVOU (7*16)(inp), D0
-	PXOR  A0, A1; PXOR B0, B1; PXOR C0, C1; PXOR D0, D1
-	MOVOU A1, (4*16)(oup); MOVOU B1, (5*16)(oup); MOVOU C1, (6*16)(oup); MOVOU D1, (7*16)(oup)
-	MOVOU (8*16)(inp), A0; MOVOU (9*16)(inp), B0; MOVOU (10*16)(inp), C0; MOVOU (11*16)(inp), D0
-	PXOR  A0, A2; PXOR B0, B2; PXOR C0, C2; PXOR D0, D2
-	MOVOU A2, (8*16)(oup); MOVOU B2, (9*16)(oup); MOVOU C2, (10*16)(oup); MOVOU D2, (11*16)(oup)
-	LEAQ  192(inp), inp
-	LEAQ  192(oup), oup
-	SUBQ  $192, inl
-	MOVO  A3, A0
-	MOVO  B3, B0
-	MOVO  C3, C0
-	MOVO  tmpStore, D0
-
-	JMP openSSETail64DecLoop
-
-// ----------------------------------------------------------------------------
-// ------------------------- AVX2 Code ----------------------------------------
+	MOVOU (SI), X15
+	PXOR  X15, X0
+	MOVOU 16(SI), X15
+	PXOR  X15, X3
+	MOVOU 32(SI), X15
+	PXOR  X15, X6
+	MOVOU 48(SI), X15
+	PXOR  X15, X9
+	MOVOU X0, (DI)
+	MOVOU X3, 16(DI)
+	MOVOU X6, 32(DI)
+	MOVOU X9, 48(DI)
+	MOVOU 64(SI), X0
+	MOVOU 80(SI), X3
+	MOVOU 96(SI), X6
+	MOVOU 112(SI), X9
+	PXOR  X0, X1
+	PXOR  X3, X4
+	PXOR  X6, X7
+	PXOR  X9, X10
+	MOVOU X1, 64(DI)
+	MOVOU X4, 80(DI)
+	MOVOU X7, 96(DI)
+	MOVOU X10, 112(DI)
+	MOVOU 128(SI), X0
+	MOVOU 144(SI), X3
+	MOVOU 160(SI), X6
+	MOVOU 176(SI), X9
+	PXOR  X0, X2
+	PXOR  X3, X5
+	PXOR  X6, X8
+	PXOR  X9, X11
+	MOVOU X2, 128(DI)
+	MOVOU X5, 144(DI)
+	MOVOU X8, 160(DI)
+	MOVOU X11, 176(DI)
+	LEAQ  192(SI), SI
+	LEAQ  192(DI), DI
+	SUBQ  $0xc0, BX
+	MOVO  X12, X0
+	MOVO  X13, X3
+	MOVO  X14, X6
+	MOVO  64(BP), X9
+	JMP   openSSETail64DecLoop
+
 chacha20Poly1305Open_AVX2:
 	VZEROUPPER
-	VMOVDQU ·chacha20Constants<>(SB), AA0
-	BYTE    $0xc4; BYTE $0x42; BYTE $0x7d; BYTE $0x5a; BYTE $0x70; BYTE $0x10 // broadcasti128 16(r8), ymm14
-	BYTE    $0xc4; BYTE $0x42; BYTE $0x7d; BYTE $0x5a; BYTE $0x60; BYTE $0x20 // broadcasti128 32(r8), ymm12
-	BYTE    $0xc4; BYTE $0xc2; BYTE $0x7d; BYTE $0x5a; BYTE $0x60; BYTE $0x30 // broadcasti128 48(r8), ymm4
-	VPADDD  ·avx2InitMask<>(SB), DD0, DD0
+	VMOVDQU ·chacha20Constants<>+0(SB), Y0
+	BYTE    $0xc4
+	BYTE    $0x42
+	BYTE    $0x7d
+	BYTE    $0x5a
+	BYTE    $0x70
+	BYTE    $0x10
+	BYTE    $0xc4
+	BYTE    $0x42
+	BYTE    $0x7d
+	BYTE    $0x5a
+	BYTE    $0x60
+	BYTE    $0x20
+	BYTE    $0xc4
+	BYTE    $0xc2
+	BYTE    $0x7d
+	BYTE    $0x5a
+	BYTE    $0x60
+	BYTE    $0x30
+	VPADDD  ·avx2InitMask<>+0(SB), Y4, Y4
 
 	// Special optimization, for very short buffers
-	CMPQ inl, $192
+	CMPQ BX, $0xc0
 	JBE  openAVX2192
-	CMPQ inl, $320
+	CMPQ BX, $0x00000140
 	JBE  openAVX2320
 
 	// For the general key prepare the key first - as a byproduct we have 64 bytes of cipher stream
-	VMOVDQA BB0, state1StoreAVX2
-	VMOVDQA CC0, state2StoreAVX2
-	VMOVDQA DD0, ctr3StoreAVX2
-	MOVQ    $10, itr2
+	VMOVDQA Y14, 32(BP)
+	VMOVDQA Y12, 64(BP)
+	VMOVDQA Y4, 192(BP)
+	MOVQ    $0x0000000a, R9
 
 openAVX2PreparePolyKey:
-	chachaQR_AVX2(AA0, BB0, CC0, DD0, TT0)
-	VPALIGNR $4, BB0, BB0, BB0; VPALIGNR $8, CC0, CC0, CC0; VPALIGNR $12, DD0, DD0, DD0
-	chachaQR_AVX2(AA0, BB0, CC0, DD0, TT0)
-	VPALIGNR $12, BB0, BB0, BB0; VPALIGNR $8, CC0, CC0, CC0; VPALIGNR $4, DD0, DD0, DD0
-	DECQ     itr2
-	JNE      openAVX2PreparePolyKey
-
-	VPADDD ·chacha20Constants<>(SB), AA0, AA0
-	VPADDD state1StoreAVX2, BB0, BB0
-	VPADDD state2StoreAVX2, CC0, CC0
-	VPADDD ctr3StoreAVX2, DD0, DD0
-
-	VPERM2I128 $0x02, AA0, BB0, TT0
+	VPADDD     Y14, Y0, Y0
+	VPXOR      Y0, Y4, Y4
+	VPSHUFB    ·rol16<>+0(SB), Y4, Y4
+	VPADDD     Y4, Y12, Y12
+	VPXOR      Y12, Y14, Y14
+	VPSLLD     $0x0c, Y14, Y3
+	VPSRLD     $0x14, Y14, Y14
+	VPXOR      Y3, Y14, Y14
+	VPADDD     Y14, Y0, Y0
+	VPXOR      Y0, Y4, Y4
+	VPSHUFB    ·rol8<>+0(SB), Y4, Y4
+	VPADDD     Y4, Y12, Y12
+	VPXOR      Y12, Y14, Y14
+	VPSLLD     $0x07, Y14, Y3
+	VPSRLD     $0x19, Y14, Y14
+	VPXOR      Y3, Y14, Y14
+	VPALIGNR   $0x04, Y14, Y14, Y14
+	VPALIGNR   $0x08, Y12, Y12, Y12
+	VPALIGNR   $0x0c, Y4, Y4, Y4
+	VPADDD     Y14, Y0, Y0
+	VPXOR      Y0, Y4, Y4
+	VPSHUFB    ·rol16<>+0(SB), Y4, Y4
+	VPADDD     Y4, Y12, Y12
+	VPXOR      Y12, Y14, Y14
+	VPSLLD     $0x0c, Y14, Y3
+	VPSRLD     $0x14, Y14, Y14
+	VPXOR      Y3, Y14, Y14
+	VPADDD     Y14, Y0, Y0
+	VPXOR      Y0, Y4, Y4
+	VPSHUFB    ·rol8<>+0(SB), Y4, Y4
+	VPADDD     Y4, Y12, Y12
+	VPXOR      Y12, Y14, Y14
+	VPSLLD     $0x07, Y14, Y3
+	VPSRLD     $0x19, Y14, Y14
+	VPXOR      Y3, Y14, Y14
+	VPALIGNR   $0x0c, Y14, Y14, Y14
+	VPALIGNR   $0x08, Y12, Y12, Y12
+	VPALIGNR   $0x04, Y4, Y4, Y4
+	DECQ       R9
+	JNE        openAVX2PreparePolyKey
+	VPADDD     ·chacha20Constants<>+0(SB), Y0, Y0
+	VPADDD     32(BP), Y14, Y14
+	VPADDD     64(BP), Y12, Y12
+	VPADDD     192(BP), Y4, Y4
+	VPERM2I128 $0x02, Y0, Y14, Y3
 
 	// Clamp and store poly key
-	VPAND   ·polyClampMask<>(SB), TT0, TT0
-	VMOVDQA TT0, rsStoreAVX2
+	VPAND   ·polyClampMask<>+0(SB), Y3, Y3
+	VMOVDQA Y3, (BP)
 
 	// Stream for the first 64 bytes
-	VPERM2I128 $0x13, AA0, BB0, AA0
-	VPERM2I128 $0x13, CC0, DD0, BB0
+	VPERM2I128 $0x13, Y0, Y14, Y0
+	VPERM2I128 $0x13, Y12, Y4, Y14
 
 	// Hash AD + first 64 bytes
-	MOVQ ad_len+80(FP), itr2
+	MOVQ ad_len+80(FP), R9
 	CALL polyHashADInternal<>(SB)
-	XORQ itr1, itr1
+	XORQ CX, CX
 
 openAVX2InitialHash64:
-	polyAdd(0(inp)(itr1*1))
-	polyMulAVX2
-	ADDQ $16, itr1
-	CMPQ itr1, $64
-	JNE  openAVX2InitialHash64
+	ADDQ  (SI)(CX*1), R10
+	ADCQ  8(SI)(CX*1), R11
+	ADCQ  $0x01, R12
+	MOVQ  (BP), DX
+	MOVQ  DX, R15
+	MULXQ R10, R13, R14
+	IMULQ R12, R15
+	MULXQ R11, AX, DX
+	ADDQ  AX, R14
+	ADCQ  DX, R15
+	MOVQ  8(BP), DX
+	MULXQ R10, R10, AX
+	ADDQ  R10, R14
+	MULXQ R11, R11, R8
+	ADCQ  R11, R15
+	ADCQ  $0x00, R8
+	IMULQ R12, DX
+	ADDQ  AX, R15
+	ADCQ  DX, R8
+	MOVQ  R13, R10
+	MOVQ  R14, R11
+	MOVQ  R15, R12
+	ANDQ  $0x03, R12
+	MOVQ  R15, R13
+	ANDQ  $-4, R13
+	MOVQ  R8, R14
+	SHRQ  $0x02, R8, R15
+	SHRQ  $0x02, R8
+	ADDQ  R13, R10
+	ADCQ  R14, R11
+	ADCQ  $0x00, R12
+	ADDQ  R15, R10
+	ADCQ  R8, R11
+	ADCQ  $0x00, R12
+	ADDQ  $0x10, CX
+	CMPQ  CX, $0x40
+	JNE   openAVX2InitialHash64
 
 	// Decrypt the first 64 bytes
-	VPXOR   (0*32)(inp), AA0, AA0
-	VPXOR   (1*32)(inp), BB0, BB0
-	VMOVDQU AA0, (0*32)(oup)
-	VMOVDQU BB0, (1*32)(oup)
-	LEAQ    (2*32)(inp), inp
-	LEAQ    (2*32)(oup), oup
-	SUBQ    $64, inl
+	VPXOR   (SI), Y0, Y0
+	VPXOR   32(SI), Y14, Y14
+	VMOVDQU Y0, (DI)
+	VMOVDQU Y14, 32(DI)
+	LEAQ    64(SI), SI
+	LEAQ    64(DI), DI
+	SUBQ    $0x40, BX
 
 openAVX2MainLoop:
-	CMPQ inl, $512
+	CMPQ BX, $0x00000200
 	JB   openAVX2MainLoopDone
 
 	// Load state, increment counter blocks, store the incremented counters
-	VMOVDQU ·chacha20Constants<>(SB), AA0; VMOVDQA AA0, AA1; VMOVDQA AA0, AA2; VMOVDQA AA0, AA3
-	VMOVDQA state1StoreAVX2, BB0; VMOVDQA BB0, BB1; VMOVDQA BB0, BB2; VMOVDQA BB0, BB3
-	VMOVDQA state2StoreAVX2, CC0; VMOVDQA CC0, CC1; VMOVDQA CC0, CC2; VMOVDQA CC0, CC3
-	VMOVDQA ctr3StoreAVX2, DD0; VPADDD ·avx2IncMask<>(SB), DD0, DD0; VPADDD ·avx2IncMask<>(SB), DD0, DD1; VPADDD ·avx2IncMask<>(SB), DD1, DD2; VPADDD ·avx2IncMask<>(SB), DD2, DD3
-	VMOVDQA DD0, ctr0StoreAVX2; VMOVDQA DD1, ctr1StoreAVX2; VMOVDQA DD2, ctr2StoreAVX2; VMOVDQA DD3, ctr3StoreAVX2
-	XORQ    itr1, itr1
+	VMOVDQU ·chacha20Constants<>+0(SB), Y0
+	VMOVDQA Y0, Y5
+	VMOVDQA Y0, Y6
+	VMOVDQA Y0, Y7
+	VMOVDQA 32(BP), Y14
+	VMOVDQA Y14, Y9
+	VMOVDQA Y14, Y10
+	VMOVDQA Y14, Y11
+	VMOVDQA 64(BP), Y12
+	VMOVDQA Y12, Y13
+	VMOVDQA Y12, Y8
+	VMOVDQA Y12, Y15
+	VMOVDQA 192(BP), Y4
+	VPADDD  ·avx2IncMask<>+0(SB), Y4, Y4
+	VPADDD  ·avx2IncMask<>+0(SB), Y4, Y1
+	VPADDD  ·avx2IncMask<>+0(SB), Y1, Y2
+	VPADDD  ·avx2IncMask<>+0(SB), Y2, Y3
+	VMOVDQA Y4, 96(BP)
+	VMOVDQA Y1, 128(BP)
+	VMOVDQA Y2, 160(BP)
+	VMOVDQA Y3, 192(BP)
+	XORQ    CX, CX
 
 openAVX2InternalLoop:
-	// Lets just say this spaghetti loop interleaves 2 quarter rounds with 3 poly multiplications
-	// Effectively per 512 bytes of stream we hash 480 bytes of ciphertext
-	polyAdd(0*8(inp)(itr1*1))
-	VPADDD   BB0, AA0, AA0; VPADDD BB1, AA1, AA1; VPADDD BB2, AA2, AA2; VPADDD BB3, AA3, AA3
-	polyMulStage1_AVX2
-	VPXOR    AA0, DD0, DD0; VPXOR AA1, DD1, DD1; VPXOR AA2, DD2, DD2; VPXOR AA3, DD3, DD3
-	VPSHUFB  ·rol16<>(SB), DD0, DD0; VPSHUFB ·rol16<>(SB), DD1, DD1; VPSHUFB ·rol16<>(SB), DD2, DD2; VPSHUFB ·rol16<>(SB), DD3, DD3
-	polyMulStage2_AVX2
-	VPADDD   DD0, CC0, CC0; VPADDD DD1, CC1, CC1; VPADDD DD2, CC2, CC2; VPADDD DD3, CC3, CC3
-	VPXOR    CC0, BB0, BB0; VPXOR CC1, BB1, BB1; VPXOR CC2, BB2, BB2; VPXOR CC3, BB3, BB3
-	polyMulStage3_AVX2
-	VMOVDQA  CC3, tmpStoreAVX2
-	VPSLLD   $12, BB0, CC3; VPSRLD $20, BB0, BB0; VPXOR CC3, BB0, BB0
-	VPSLLD   $12, BB1, CC3; VPSRLD $20, BB1, BB1; VPXOR CC3, BB1, BB1
-	VPSLLD   $12, BB2, CC3; VPSRLD $20, BB2, BB2; VPXOR CC3, BB2, BB2
-	VPSLLD   $12, BB3, CC3; VPSRLD $20, BB3, BB3; VPXOR CC3, BB3, BB3
-	VMOVDQA  tmpStoreAVX2, CC3
-	polyMulReduceStage
-	VPADDD   BB0, AA0, AA0; VPADDD BB1, AA1, AA1; VPADDD BB2, AA2, AA2; VPADDD BB3, AA3, AA3
-	VPXOR    AA0, DD0, DD0; VPXOR AA1, DD1, DD1; VPXOR AA2, DD2, DD2; VPXOR AA3, DD3, DD3
-	VPSHUFB  ·rol8<>(SB), DD0, DD0; VPSHUFB ·rol8<>(SB), DD1, DD1; VPSHUFB ·rol8<>(SB), DD2, DD2; VPSHUFB ·rol8<>(SB), DD3, DD3
-	polyAdd(2*8(inp)(itr1*1))
-	VPADDD   DD0, CC0, CC0; VPADDD DD1, CC1, CC1; VPADDD DD2, CC2, CC2; VPADDD DD3, CC3, CC3
-	polyMulStage1_AVX2
-	VPXOR    CC0, BB0, BB0; VPXOR CC1, BB1, BB1; VPXOR CC2, BB2, BB2; VPXOR CC3, BB3, BB3
-	VMOVDQA  CC3, tmpStoreAVX2
-	VPSLLD   $7, BB0, CC3; VPSRLD $25, BB0, BB0; VPXOR CC3, BB0, BB0
-	VPSLLD   $7, BB1, CC3; VPSRLD $25, BB1, BB1; VPXOR CC3, BB1, BB1
-	VPSLLD   $7, BB2, CC3; VPSRLD $25, BB2, BB2; VPXOR CC3, BB2, BB2
-	VPSLLD   $7, BB3, CC3; VPSRLD $25, BB3, BB3; VPXOR CC3, BB3, BB3
-	VMOVDQA  tmpStoreAVX2, CC3
-	polyMulStage2_AVX2
-	VPALIGNR $4, BB0, BB0, BB0; VPALIGNR $4, BB1, BB1, BB1; VPALIGNR $4, BB2, BB2, BB2; VPALIGNR $4, BB3, BB3, BB3
-	VPALIGNR $8, CC0, CC0, CC0; VPALIGNR $8, CC1, CC1, CC1; VPALIGNR $8, CC2, CC2, CC2; VPALIGNR $8, CC3, CC3, CC3
-	VPALIGNR $12, DD0, DD0, DD0; VPALIGNR $12, DD1, DD1, DD1; VPALIGNR $12, DD2, DD2, DD2; VPALIGNR $12, DD3, DD3, DD3
-	VPADDD   BB0, AA0, AA0; VPADDD BB1, AA1, AA1; VPADDD BB2, AA2, AA2; VPADDD BB3, AA3, AA3
-	polyMulStage3_AVX2
-	VPXOR    AA0, DD0, DD0; VPXOR AA1, DD1, DD1; VPXOR AA2, DD2, DD2; VPXOR AA3, DD3, DD3
-	VPSHUFB  ·rol16<>(SB), DD0, DD0; VPSHUFB ·rol16<>(SB), DD1, DD1; VPSHUFB ·rol16<>(SB), DD2, DD2; VPSHUFB ·rol16<>(SB), DD3, DD3
-	polyMulReduceStage
-	VPADDD   DD0, CC0, CC0; VPADDD DD1, CC1, CC1; VPADDD DD2, CC2, CC2; VPADDD DD3, CC3, CC3
-	VPXOR    CC0, BB0, BB0; VPXOR CC1, BB1, BB1; VPXOR CC2, BB2, BB2; VPXOR CC3, BB3, BB3
-	polyAdd(4*8(inp)(itr1*1))
-	LEAQ     (6*8)(itr1), itr1
-	VMOVDQA  CC3, tmpStoreAVX2
-	VPSLLD   $12, BB0, CC3; VPSRLD $20, BB0, BB0; VPXOR CC3, BB0, BB0
-	VPSLLD   $12, BB1, CC3; VPSRLD $20, BB1, BB1; VPXOR CC3, BB1, BB1
-	VPSLLD   $12, BB2, CC3; VPSRLD $20, BB2, BB2; VPXOR CC3, BB2, BB2
-	VPSLLD   $12, BB3, CC3; VPSRLD $20, BB3, BB3; VPXOR CC3, BB3, BB3
-	VMOVDQA  tmpStoreAVX2, CC3
-	polyMulStage1_AVX2
-	VPADDD   BB0, AA0, AA0; VPADDD BB1, AA1, AA1; VPADDD BB2, AA2, AA2; VPADDD BB3, AA3, AA3
-	VPXOR    AA0, DD0, DD0; VPXOR AA1, DD1, DD1; VPXOR AA2, DD2, DD2; VPXOR AA3, DD3, DD3
-	polyMulStage2_AVX2
-	VPSHUFB  ·rol8<>(SB), DD0, DD0; VPSHUFB ·rol8<>(SB), DD1, DD1; VPSHUFB ·rol8<>(SB), DD2, DD2; VPSHUFB ·rol8<>(SB), DD3, DD3
-	VPADDD   DD0, CC0, CC0; VPADDD DD1, CC1, CC1; VPADDD DD2, CC2, CC2; VPADDD DD3, CC3, CC3
-	polyMulStage3_AVX2
-	VPXOR    CC0, BB0, BB0; VPXOR CC1, BB1, BB1; VPXOR CC2, BB2, BB2; VPXOR CC3, BB3, BB3
-	VMOVDQA  CC3, tmpStoreAVX2
-	VPSLLD   $7, BB0, CC3; VPSRLD $25, BB0, BB0; VPXOR CC3, BB0, BB0
-	VPSLLD   $7, BB1, CC3; VPSRLD $25, BB1, BB1; VPXOR CC3, BB1, BB1
-	VPSLLD   $7, BB2, CC3; VPSRLD $25, BB2, BB2; VPXOR CC3, BB2, BB2
-	VPSLLD   $7, BB3, CC3; VPSRLD $25, BB3, BB3; VPXOR CC3, BB3, BB3
-	VMOVDQA  tmpStoreAVX2, CC3
-	polyMulReduceStage
-	VPALIGNR $12, BB0, BB0, BB0; VPALIGNR $12, BB1, BB1, BB1; VPALIGNR $12, BB2, BB2, BB2; VPALIGNR $12, BB3, BB3, BB3
-	VPALIGNR $8, CC0, CC0, CC0; VPALIGNR $8, CC1, CC1, CC1; VPALIGNR $8, CC2, CC2, CC2; VPALIGNR $8, CC3, CC3, CC3
-	VPALIGNR $4, DD0, DD0, DD0; VPALIGNR $4, DD1, DD1, DD1; VPALIGNR $4, DD2, DD2, DD2; VPALIGNR $4, DD3, DD3, DD3
-	CMPQ     itr1, $480
+	ADDQ     (SI)(CX*1), R10
+	ADCQ     8(SI)(CX*1), R11
+	ADCQ     $0x01, R12
+	VPADDD   Y14, Y0, Y0
+	VPADDD   Y9, Y5, Y5
+	VPADDD   Y10, Y6, Y6
+	VPADDD   Y11, Y7, Y7
+	MOVQ     (BP), DX
+	MOVQ     DX, R15
+	MULXQ    R10, R13, R14
+	IMULQ    R12, R15
+	MULXQ    R11, AX, DX
+	ADDQ     AX, R14
+	ADCQ     DX, R15
+	VPXOR    Y0, Y4, Y4
+	VPXOR    Y5, Y1, Y1
+	VPXOR    Y6, Y2, Y2
+	VPXOR    Y7, Y3, Y3
+	VPSHUFB  ·rol16<>+0(SB), Y4, Y4
+	VPSHUFB  ·rol16<>+0(SB), Y1, Y1
+	VPSHUFB  ·rol16<>+0(SB), Y2, Y2
+	VPSHUFB  ·rol16<>+0(SB), Y3, Y3
+	MOVQ     8(BP), DX
+	MULXQ    R10, R10, AX
+	ADDQ     R10, R14
+	MULXQ    R11, R11, R8
+	ADCQ     R11, R15
+	ADCQ     $0x00, R8
+	VPADDD   Y4, Y12, Y12
+	VPADDD   Y1, Y13, Y13
+	VPADDD   Y2, Y8, Y8
+	VPADDD   Y3, Y15, Y15
+	VPXOR    Y12, Y14, Y14
+	VPXOR    Y13, Y9, Y9
+	VPXOR    Y8, Y10, Y10
+	VPXOR    Y15, Y11, Y11
+	IMULQ    R12, DX
+	ADDQ     AX, R15
+	ADCQ     DX, R8
+	VMOVDQA  Y15, 224(BP)
+	VPSLLD   $0x0c, Y14, Y15
+	VPSRLD   $0x14, Y14, Y14
+	VPXOR    Y15, Y14, Y14
+	VPSLLD   $0x0c, Y9, Y15
+	VPSRLD   $0x14, Y9, Y9
+	VPXOR    Y15, Y9, Y9
+	VPSLLD   $0x0c, Y10, Y15
+	VPSRLD   $0x14, Y10, Y10
+	VPXOR    Y15, Y10, Y10
+	VPSLLD   $0x0c, Y11, Y15
+	VPSRLD   $0x14, Y11, Y11
+	VPXOR    Y15, Y11, Y11
+	VMOVDQA  224(BP), Y15
+	MOVQ     R13, R10
+	MOVQ     R14, R11
+	MOVQ     R15, R12
+	ANDQ     $0x03, R12
+	MOVQ     R15, R13
+	ANDQ     $-4, R13
+	MOVQ     R8, R14
+	SHRQ     $0x02, R8, R15
+	SHRQ     $0x02, R8
+	ADDQ     R13, R10
+	ADCQ     R14, R11
+	ADCQ     $0x00, R12
+	ADDQ     R15, R10
+	ADCQ     R8, R11
+	ADCQ     $0x00, R12
+	VPADDD   Y14, Y0, Y0
+	VPADDD   Y9, Y5, Y5
+	VPADDD   Y10, Y6, Y6
+	VPADDD   Y11, Y7, Y7
+	VPXOR    Y0, Y4, Y4
+	VPXOR    Y5, Y1, Y1
+	VPXOR    Y6, Y2, Y2
+	VPXOR    Y7, Y3, Y3
+	VPSHUFB  ·rol8<>+0(SB), Y4, Y4
+	VPSHUFB  ·rol8<>+0(SB), Y1, Y1
+	VPSHUFB  ·rol8<>+0(SB), Y2, Y2
+	VPSHUFB  ·rol8<>+0(SB), Y3, Y3
+	ADDQ     16(SI)(CX*1), R10
+	ADCQ     24(SI)(CX*1), R11
+	ADCQ     $0x01, R12
+	VPADDD   Y4, Y12, Y12
+	VPADDD   Y1, Y13, Y13
+	VPADDD   Y2, Y8, Y8
+	VPADDD   Y3, Y15, Y15
+	MOVQ     (BP), DX
+	MOVQ     DX, R15
+	MULXQ    R10, R13, R14
+	IMULQ    R12, R15
+	MULXQ    R11, AX, DX
+	ADDQ     AX, R14
+	ADCQ     DX, R15
+	VPXOR    Y12, Y14, Y14
+	VPXOR    Y13, Y9, Y9
+	VPXOR    Y8, Y10, Y10
+	VPXOR    Y15, Y11, Y11
+	VMOVDQA  Y15, 224(BP)
+	VPSLLD   $0x07, Y14, Y15
+	VPSRLD   $0x19, Y14, Y14
+	VPXOR    Y15, Y14, Y14
+	VPSLLD   $0x07, Y9, Y15
+	VPSRLD   $0x19, Y9, Y9
+	VPXOR    Y15, Y9, Y9
+	VPSLLD   $0x07, Y10, Y15
+	VPSRLD   $0x19, Y10, Y10
+	VPXOR    Y15, Y10, Y10
+	VPSLLD   $0x07, Y11, Y15
+	VPSRLD   $0x19, Y11, Y11
+	VPXOR    Y15, Y11, Y11
+	VMOVDQA  224(BP), Y15
+	MOVQ     8(BP), DX
+	MULXQ    R10, R10, AX
+	ADDQ     R10, R14
+	MULXQ    R11, R11, R8
+	ADCQ     R11, R15
+	ADCQ     $0x00, R8
+	VPALIGNR $0x04, Y14, Y14, Y14
+	VPALIGNR $0x04, Y9, Y9, Y9
+	VPALIGNR $0x04, Y10, Y10, Y10
+	VPALIGNR $0x04, Y11, Y11, Y11
+	VPALIGNR $0x08, Y12, Y12, Y12
+	VPALIGNR $0x08, Y13, Y13, Y13
+	VPALIGNR $0x08, Y8, Y8, Y8
+	VPALIGNR $0x08, Y15, Y15, Y15
+	VPALIGNR $0x0c, Y4, Y4, Y4
+	VPALIGNR $0x0c, Y1, Y1, Y1
+	VPALIGNR $0x0c, Y2, Y2, Y2
+	VPALIGNR $0x0c, Y3, Y3, Y3
+	VPADDD   Y14, Y0, Y0
+	VPADDD   Y9, Y5, Y5
+	VPADDD   Y10, Y6, Y6
+	VPADDD   Y11, Y7, Y7
+	IMULQ    R12, DX
+	ADDQ     AX, R15
+	ADCQ     DX, R8
+	VPXOR    Y0, Y4, Y4
+	VPXOR    Y5, Y1, Y1
+	VPXOR    Y6, Y2, Y2
+	VPXOR    Y7, Y3, Y3
+	VPSHUFB  ·rol16<>+0(SB), Y4, Y4
+	VPSHUFB  ·rol16<>+0(SB), Y1, Y1
+	VPSHUFB  ·rol16<>+0(SB), Y2, Y2
+	VPSHUFB  ·rol16<>+0(SB), Y3, Y3
+	MOVQ     R13, R10
+	MOVQ     R14, R11
+	MOVQ     R15, R12
+	ANDQ     $0x03, R12
+	MOVQ     R15, R13
+	ANDQ     $-4, R13
+	MOVQ     R8, R14
+	SHRQ     $0x02, R8, R15
+	SHRQ     $0x02, R8
+	ADDQ     R13, R10
+	ADCQ     R14, R11
+	ADCQ     $0x00, R12
+	ADDQ     R15, R10
+	ADCQ     R8, R11
+	ADCQ     $0x00, R12
+	VPADDD   Y4, Y12, Y12
+	VPADDD   Y1, Y13, Y13
+	VPADDD   Y2, Y8, Y8
+	VPADDD   Y3, Y15, Y15
+	VPXOR    Y12, Y14, Y14
+	VPXOR    Y13, Y9, Y9
+	VPXOR    Y8, Y10, Y10
+	VPXOR    Y15, Y11, Y11
+	ADDQ     32(SI)(CX*1), R10
+	ADCQ     40(SI)(CX*1), R11
+	ADCQ     $0x01, R12
+	LEAQ     48(CX), CX
+	VMOVDQA  Y15, 224(BP)
+	VPSLLD   $0x0c, Y14, Y15
+	VPSRLD   $0x14, Y14, Y14
+	VPXOR    Y15, Y14, Y14
+	VPSLLD   $0x0c, Y9, Y15
+	VPSRLD   $0x14, Y9, Y9
+	VPXOR    Y15, Y9, Y9
+	VPSLLD   $0x0c, Y10, Y15
+	VPSRLD   $0x14, Y10, Y10
+	VPXOR    Y15, Y10, Y10
+	VPSLLD   $0x0c, Y11, Y15
+	VPSRLD   $0x14, Y11, Y11
+	VPXOR    Y15, Y11, Y11
+	VMOVDQA  224(BP), Y15
+	MOVQ     (BP), DX
+	MOVQ     DX, R15
+	MULXQ    R10, R13, R14
+	IMULQ    R12, R15
+	MULXQ    R11, AX, DX
+	ADDQ     AX, R14
+	ADCQ     DX, R15
+	VPADDD   Y14, Y0, Y0
+	VPADDD   Y9, Y5, Y5
+	VPADDD   Y10, Y6, Y6
+	VPADDD   Y11, Y7, Y7
+	VPXOR    Y0, Y4, Y4
+	VPXOR    Y5, Y1, Y1
+	VPXOR    Y6, Y2, Y2
+	VPXOR    Y7, Y3, Y3
+	MOVQ     8(BP), DX
+	MULXQ    R10, R10, AX
+	ADDQ     R10, R14
+	MULXQ    R11, R11, R8
+	ADCQ     R11, R15
+	ADCQ     $0x00, R8
+	VPSHUFB  ·rol8<>+0(SB), Y4, Y4
+	VPSHUFB  ·rol8<>+0(SB), Y1, Y1
+	VPSHUFB  ·rol8<>+0(SB), Y2, Y2
+	VPSHUFB  ·rol8<>+0(SB), Y3, Y3
+	VPADDD   Y4, Y12, Y12
+	VPADDD   Y1, Y13, Y13
+	VPADDD   Y2, Y8, Y8
+	VPADDD   Y3, Y15, Y15
+	IMULQ    R12, DX
+	ADDQ     AX, R15
+	ADCQ     DX, R8
+	VPXOR    Y12, Y14, Y14
+	VPXOR    Y13, Y9, Y9
+	VPXOR    Y8, Y10, Y10
+	VPXOR    Y15, Y11, Y11
+	VMOVDQA  Y15, 224(BP)
+	VPSLLD   $0x07, Y14, Y15
+	VPSRLD   $0x19, Y14, Y14
+	VPXOR    Y15, Y14, Y14
+	VPSLLD   $0x07, Y9, Y15
+	VPSRLD   $0x19, Y9, Y9
+	VPXOR    Y15, Y9, Y9
+	VPSLLD   $0x07, Y10, Y15
+	VPSRLD   $0x19, Y10, Y10
+	VPXOR    Y15, Y10, Y10
+	VPSLLD   $0x07, Y11, Y15
+	VPSRLD   $0x19, Y11, Y11
+	VPXOR    Y15, Y11, Y11
+	VMOVDQA  224(BP), Y15
+	MOVQ     R13, R10
+	MOVQ     R14, R11
+	MOVQ     R15, R12
+	ANDQ     $0x03, R12
+	MOVQ     R15, R13
+	ANDQ     $-4, R13
+	MOVQ     R8, R14
+	SHRQ     $0x02, R8, R15
+	SHRQ     $0x02, R8
+	ADDQ     R13, R10
+	ADCQ     R14, R11
+	ADCQ     $0x00, R12
+	ADDQ     R15, R10
+	ADCQ     R8, R11
+	ADCQ     $0x00, R12
+	VPALIGNR $0x0c, Y14, Y14, Y14
+	VPALIGNR $0x0c, Y9, Y9, Y9
+	VPALIGNR $0x0c, Y10, Y10, Y10
+	VPALIGNR $0x0c, Y11, Y11, Y11
+	VPALIGNR $0x08, Y12, Y12, Y12
+	VPALIGNR $0x08, Y13, Y13, Y13
+	VPALIGNR $0x08, Y8, Y8, Y8
+	VPALIGNR $0x08, Y15, Y15, Y15
+	VPALIGNR $0x04, Y4, Y4, Y4
+	VPALIGNR $0x04, Y1, Y1, Y1
+	VPALIGNR $0x04, Y2, Y2, Y2
+	VPALIGNR $0x04, Y3, Y3, Y3
+	CMPQ     CX, $0x000001e0
 	JNE      openAVX2InternalLoop
-
-	VPADDD  ·chacha20Constants<>(SB), AA0, AA0; VPADDD ·chacha20Constants<>(SB), AA1, AA1; VPADDD ·chacha20Constants<>(SB), AA2, AA2; VPADDD ·chacha20Constants<>(SB), AA3, AA3
-	VPADDD  state1StoreAVX2, BB0, BB0; VPADDD state1StoreAVX2, BB1, BB1; VPADDD state1StoreAVX2, BB2, BB2; VPADDD state1StoreAVX2, BB3, BB3
-	VPADDD  state2StoreAVX2, CC0, CC0; VPADDD state2StoreAVX2, CC1, CC1; VPADDD state2StoreAVX2, CC2, CC2; VPADDD state2StoreAVX2, CC3, CC3
-	VPADDD  ctr0StoreAVX2, DD0, DD0; VPADDD ctr1StoreAVX2, DD1, DD1; VPADDD ctr2StoreAVX2, DD2, DD2; VPADDD ctr3StoreAVX2, DD3, DD3
-	VMOVDQA CC3, tmpStoreAVX2
+	VPADDD   ·chacha20Constants<>+0(SB), Y0, Y0
+	VPADDD   ·chacha20Constants<>+0(SB), Y5, Y5
+	VPADDD   ·chacha20Constants<>+0(SB), Y6, Y6
+	VPADDD   ·chacha20Constants<>+0(SB), Y7, Y7
+	VPADDD   32(BP), Y14, Y14
+	VPADDD   32(BP), Y9, Y9
+	VPADDD   32(BP), Y10, Y10
+	VPADDD   32(BP), Y11, Y11
+	VPADDD   64(BP), Y12, Y12
+	VPADDD   64(BP), Y13, Y13
+	VPADDD   64(BP), Y8, Y8
+	VPADDD   64(BP), Y15, Y15
+	VPADDD   96(BP), Y4, Y4
+	VPADDD   128(BP), Y1, Y1
+	VPADDD   160(BP), Y2, Y2
+	VPADDD   192(BP), Y3, Y3
+	VMOVDQA  Y15, 224(BP)
 
 	// We only hashed 480 of the 512 bytes available - hash the remaining 32 here
-	polyAdd(480(inp))
-	polyMulAVX2
-	VPERM2I128 $0x02, AA0, BB0, CC3; VPERM2I128 $0x13, AA0, BB0, BB0; VPERM2I128 $0x02, CC0, DD0, AA0; VPERM2I128 $0x13, CC0, DD0, CC0
-	VPXOR      (0*32)(inp), CC3, CC3; VPXOR (1*32)(inp), AA0, AA0; VPXOR (2*32)(inp), BB0, BB0; VPXOR (3*32)(inp), CC0, CC0
-	VMOVDQU    CC3, (0*32)(oup); VMOVDQU AA0, (1*32)(oup); VMOVDQU BB0, (2*32)(oup); VMOVDQU CC0, (3*32)(oup)
-	VPERM2I128 $0x02, AA1, BB1, AA0; VPERM2I128 $0x02, CC1, DD1, BB0; VPERM2I128 $0x13, AA1, BB1, CC0; VPERM2I128 $0x13, CC1, DD1, DD0
-	VPXOR      (4*32)(inp), AA0, AA0; VPXOR (5*32)(inp), BB0, BB0; VPXOR (6*32)(inp), CC0, CC0; VPXOR (7*32)(inp), DD0, DD0
-	VMOVDQU    AA0, (4*32)(oup); VMOVDQU BB0, (5*32)(oup); VMOVDQU CC0, (6*32)(oup); VMOVDQU DD0, (7*32)(oup)
+	ADDQ       480(SI), R10
+	ADCQ       488(SI), R11
+	ADCQ       $0x01, R12
+	MOVQ       (BP), DX
+	MOVQ       DX, R15
+	MULXQ      R10, R13, R14
+	IMULQ      R12, R15
+	MULXQ      R11, AX, DX
+	ADDQ       AX, R14
+	ADCQ       DX, R15
+	MOVQ       8(BP), DX
+	MULXQ      R10, R10, AX
+	ADDQ       R10, R14
+	MULXQ      R11, R11, R8
+	ADCQ       R11, R15
+	ADCQ       $0x00, R8
+	IMULQ      R12, DX
+	ADDQ       AX, R15
+	ADCQ       DX, R8
+	MOVQ       R13, R10
+	MOVQ       R14, R11
+	MOVQ       R15, R12
+	ANDQ       $0x03, R12
+	MOVQ       R15, R13
+	ANDQ       $-4, R13
+	MOVQ       R8, R14
+	SHRQ       $0x02, R8, R15
+	SHRQ       $0x02, R8
+	ADDQ       R13, R10
+	ADCQ       R14, R11
+	ADCQ       $0x00, R12
+	ADDQ       R15, R10
+	ADCQ       R8, R11
+	ADCQ       $0x00, R12
+	VPERM2I128 $0x02, Y0, Y14, Y15
+	VPERM2I128 $0x13, Y0, Y14, Y14
+	VPERM2I128 $0x02, Y12, Y4, Y0
+	VPERM2I128 $0x13, Y12, Y4, Y12
+	VPXOR      (SI), Y15, Y15
+	VPXOR      32(SI), Y0, Y0
+	VPXOR      64(SI), Y14, Y14
+	VPXOR      96(SI), Y12, Y12
+	VMOVDQU    Y15, (DI)
+	VMOVDQU    Y0, 32(DI)
+	VMOVDQU    Y14, 64(DI)
+	VMOVDQU    Y12, 96(DI)
+	VPERM2I128 $0x02, Y5, Y9, Y0
+	VPERM2I128 $0x02, Y13, Y1, Y14
+	VPERM2I128 $0x13, Y5, Y9, Y12
+	VPERM2I128 $0x13, Y13, Y1, Y4
+	VPXOR      128(SI), Y0, Y0
+	VPXOR      160(SI), Y14, Y14
+	VPXOR      192(SI), Y12, Y12
+	VPXOR      224(SI), Y4, Y4
+	VMOVDQU    Y0, 128(DI)
+	VMOVDQU    Y14, 160(DI)
+	VMOVDQU    Y12, 192(DI)
+	VMOVDQU    Y4, 224(DI)
 
 	// and here
-	polyAdd(496(inp))
-	polyMulAVX2
-	VPERM2I128 $0x02, AA2, BB2, AA0; VPERM2I128 $0x02, CC2, DD2, BB0; VPERM2I128 $0x13, AA2, BB2, CC0; VPERM2I128 $0x13, CC2, DD2, DD0
-	VPXOR      (8*32)(inp), AA0, AA0; VPXOR (9*32)(inp), BB0, BB0; VPXOR (10*32)(inp), CC0, CC0; VPXOR (11*32)(inp), DD0, DD0
-	VMOVDQU    AA0, (8*32)(oup); VMOVDQU BB0, (9*32)(oup); VMOVDQU CC0, (10*32)(oup); VMOVDQU DD0, (11*32)(oup)
-	VPERM2I128 $0x02, AA3, BB3, AA0; VPERM2I128 $0x02, tmpStoreAVX2, DD3, BB0; VPERM2I128 $0x13, AA3, BB3, CC0; VPERM2I128 $0x13, tmpStoreAVX2, DD3, DD0
-	VPXOR      (12*32)(inp), AA0, AA0; VPXOR (13*32)(inp), BB0, BB0; VPXOR (14*32)(inp), CC0, CC0; VPXOR (15*32)(inp), DD0, DD0
-	VMOVDQU    AA0, (12*32)(oup); VMOVDQU BB0, (13*32)(oup); VMOVDQU CC0, (14*32)(oup); VMOVDQU DD0, (15*32)(oup)
-	LEAQ       (32*16)(inp), inp
-	LEAQ       (32*16)(oup), oup
-	SUBQ       $(32*16), inl
+	ADDQ       496(SI), R10
+	ADCQ       504(SI), R11
+	ADCQ       $0x01, R12
+	MOVQ       (BP), DX
+	MOVQ       DX, R15
+	MULXQ      R10, R13, R14
+	IMULQ      R12, R15
+	MULXQ      R11, AX, DX
+	ADDQ       AX, R14
+	ADCQ       DX, R15
+	MOVQ       8(BP), DX
+	MULXQ      R10, R10, AX
+	ADDQ       R10, R14
+	MULXQ      R11, R11, R8
+	ADCQ       R11, R15
+	ADCQ       $0x00, R8
+	IMULQ      R12, DX
+	ADDQ       AX, R15
+	ADCQ       DX, R8
+	MOVQ       R13, R10
+	MOVQ       R14, R11
+	MOVQ       R15, R12
+	ANDQ       $0x03, R12
+	MOVQ       R15, R13
+	ANDQ       $-4, R13
+	MOVQ       R8, R14
+	SHRQ       $0x02, R8, R15
+	SHRQ       $0x02, R8
+	ADDQ       R13, R10
+	ADCQ       R14, R11
+	ADCQ       $0x00, R12
+	ADDQ       R15, R10
+	ADCQ       R8, R11
+	ADCQ       $0x00, R12
+	VPERM2I128 $0x02, Y6, Y10, Y0
+	VPERM2I128 $0x02, Y8, Y2, Y14
+	VPERM2I128 $0x13, Y6, Y10, Y12
+	VPERM2I128 $0x13, Y8, Y2, Y4
+	VPXOR      256(SI), Y0, Y0
+	VPXOR      288(SI), Y14, Y14
+	VPXOR      320(SI), Y12, Y12
+	VPXOR      352(SI), Y4, Y4
+	VMOVDQU    Y0, 256(DI)
+	VMOVDQU    Y14, 288(DI)
+	VMOVDQU    Y12, 320(DI)
+	VMOVDQU    Y4, 352(DI)
+	VPERM2I128 $0x02, Y7, Y11, Y0
+	VPERM2I128 $0x02, 224(BP), Y3, Y14
+	VPERM2I128 $0x13, Y7, Y11, Y12
+	VPERM2I128 $0x13, 224(BP), Y3, Y4
+	VPXOR      384(SI), Y0, Y0
+	VPXOR      416(SI), Y14, Y14
+	VPXOR      448(SI), Y12, Y12
+	VPXOR      480(SI), Y4, Y4
+	VMOVDQU    Y0, 384(DI)
+	VMOVDQU    Y14, 416(DI)
+	VMOVDQU    Y12, 448(DI)
+	VMOVDQU    Y4, 480(DI)
+	LEAQ       512(SI), SI
+	LEAQ       512(DI), DI
+	SUBQ       $0x00000200, BX
 	JMP        openAVX2MainLoop
 
 openAVX2MainLoopDone:
 	// Handle the various tail sizes efficiently
-	TESTQ inl, inl
+	TESTQ BX, BX
 	JE    openSSEFinalize
-	CMPQ  inl, $128
+	CMPQ  BX, $0x80
 	JBE   openAVX2Tail128
-	CMPQ  inl, $256
+	CMPQ  BX, $0x00000100
 	JBE   openAVX2Tail256
-	CMPQ  inl, $384
+	CMPQ  BX, $0x00000180
 	JBE   openAVX2Tail384
 	JMP   openAVX2Tail512
 
-// ----------------------------------------------------------------------------
-// Special optimization for buffers smaller than 193 bytes
 openAVX2192:
-	// For up to 192 bytes of ciphertext and 64 bytes for the poly key, we process four blocks
-	VMOVDQA AA0, AA1
-	VMOVDQA BB0, BB1
-	VMOVDQA CC0, CC1
-	VPADDD  ·avx2IncMask<>(SB), DD0, DD1
-	VMOVDQA AA0, AA2
-	VMOVDQA BB0, BB2
-	VMOVDQA CC0, CC2
-	VMOVDQA DD0, DD2
-	VMOVDQA DD1, TT3
-	MOVQ    $10, itr2
+	VMOVDQA Y0, Y5
+	VMOVDQA Y14, Y9
+	VMOVDQA Y12, Y13
+	VPADDD  ·avx2IncMask<>+0(SB), Y4, Y1
+	VMOVDQA Y0, Y6
+	VMOVDQA Y14, Y10
+	VMOVDQA Y12, Y8
+	VMOVDQA Y4, Y2
+	VMOVDQA Y1, Y15
+	MOVQ    $0x0000000a, R9
 
 openAVX2192InnerCipherLoop:
-	chachaQR_AVX2(AA0, BB0, CC0, DD0, TT0); chachaQR_AVX2(AA1, BB1, CC1, DD1, TT0)
-	VPALIGNR   $4, BB0, BB0, BB0; VPALIGNR $4, BB1, BB1, BB1
-	VPALIGNR   $8, CC0, CC0, CC0; VPALIGNR $8, CC1, CC1, CC1
-	VPALIGNR   $12, DD0, DD0, DD0; VPALIGNR $12, DD1, DD1, DD1
-	chachaQR_AVX2(AA0, BB0, CC0, DD0, TT0); chachaQR_AVX2(AA1, BB1, CC1, DD1, TT0)
-	VPALIGNR   $12, BB0, BB0, BB0; VPALIGNR $12, BB1, BB1, BB1
-	VPALIGNR   $8, CC0, CC0, CC0; VPALIGNR $8, CC1, CC1, CC1
-	VPALIGNR   $4, DD0, DD0, DD0; VPALIGNR $4, DD1, DD1, DD1
-	DECQ       itr2
+	VPADDD     Y14, Y0, Y0
+	VPXOR      Y0, Y4, Y4
+	VPSHUFB    ·rol16<>+0(SB), Y4, Y4
+	VPADDD     Y4, Y12, Y12
+	VPXOR      Y12, Y14, Y14
+	VPSLLD     $0x0c, Y14, Y3
+	VPSRLD     $0x14, Y14, Y14
+	VPXOR      Y3, Y14, Y14
+	VPADDD     Y14, Y0, Y0
+	VPXOR      Y0, Y4, Y4
+	VPSHUFB    ·rol8<>+0(SB), Y4, Y4
+	VPADDD     Y4, Y12, Y12
+	VPXOR      Y12, Y14, Y14
+	VPSLLD     $0x07, Y14, Y3
+	VPSRLD     $0x19, Y14, Y14
+	VPXOR      Y3, Y14, Y14
+	VPADDD     Y9, Y5, Y5
+	VPXOR      Y5, Y1, Y1
+	VPSHUFB    ·rol16<>+0(SB), Y1, Y1
+	VPADDD     Y1, Y13, Y13
+	VPXOR      Y13, Y9, Y9
+	VPSLLD     $0x0c, Y9, Y3
+	VPSRLD     $0x14, Y9, Y9
+	VPXOR      Y3, Y9, Y9
+	VPADDD     Y9, Y5, Y5
+	VPXOR      Y5, Y1, Y1
+	VPSHUFB    ·rol8<>+0(SB), Y1, Y1
+	VPADDD     Y1, Y13, Y13
+	VPXOR      Y13, Y9, Y9
+	VPSLLD     $0x07, Y9, Y3
+	VPSRLD     $0x19, Y9, Y9
+	VPXOR      Y3, Y9, Y9
+	VPALIGNR   $0x04, Y14, Y14, Y14
+	VPALIGNR   $0x04, Y9, Y9, Y9
+	VPALIGNR   $0x08, Y12, Y12, Y12
+	VPALIGNR   $0x08, Y13, Y13, Y13
+	VPALIGNR   $0x0c, Y4, Y4, Y4
+	VPALIGNR   $0x0c, Y1, Y1, Y1
+	VPADDD     Y14, Y0, Y0
+	VPXOR      Y0, Y4, Y4
+	VPSHUFB    ·rol16<>+0(SB), Y4, Y4
+	VPADDD     Y4, Y12, Y12
+	VPXOR      Y12, Y14, Y14
+	VPSLLD     $0x0c, Y14, Y3
+	VPSRLD     $0x14, Y14, Y14
+	VPXOR      Y3, Y14, Y14
+	VPADDD     Y14, Y0, Y0
+	VPXOR      Y0, Y4, Y4
+	VPSHUFB    ·rol8<>+0(SB), Y4, Y4
+	VPADDD     Y4, Y12, Y12
+	VPXOR      Y12, Y14, Y14
+	VPSLLD     $0x07, Y14, Y3
+	VPSRLD     $0x19, Y14, Y14
+	VPXOR      Y3, Y14, Y14
+	VPADDD     Y9, Y5, Y5
+	VPXOR      Y5, Y1, Y1
+	VPSHUFB    ·rol16<>+0(SB), Y1, Y1
+	VPADDD     Y1, Y13, Y13
+	VPXOR      Y13, Y9, Y9
+	VPSLLD     $0x0c, Y9, Y3
+	VPSRLD     $0x14, Y9, Y9
+	VPXOR      Y3, Y9, Y9
+	VPADDD     Y9, Y5, Y5
+	VPXOR      Y5, Y1, Y1
+	VPSHUFB    ·rol8<>+0(SB), Y1, Y1
+	VPADDD     Y1, Y13, Y13
+	VPXOR      Y13, Y9, Y9
+	VPSLLD     $0x07, Y9, Y3
+	VPSRLD     $0x19, Y9, Y9
+	VPXOR      Y3, Y9, Y9
+	VPALIGNR   $0x0c, Y14, Y14, Y14
+	VPALIGNR   $0x0c, Y9, Y9, Y9
+	VPALIGNR   $0x08, Y12, Y12, Y12
+	VPALIGNR   $0x08, Y13, Y13, Y13
+	VPALIGNR   $0x04, Y4, Y4, Y4
+	VPALIGNR   $0x04, Y1, Y1, Y1
+	DECQ       R9
 	JNE        openAVX2192InnerCipherLoop
-	VPADDD     AA2, AA0, AA0; VPADDD AA2, AA1, AA1
-	VPADDD     BB2, BB0, BB0; VPADDD BB2, BB1, BB1
-	VPADDD     CC2, CC0, CC0; VPADDD CC2, CC1, CC1
-	VPADDD     DD2, DD0, DD0; VPADDD TT3, DD1, DD1
-	VPERM2I128 $0x02, AA0, BB0, TT0
+	VPADDD     Y6, Y0, Y0
+	VPADDD     Y6, Y5, Y5
+	VPADDD     Y10, Y14, Y14
+	VPADDD     Y10, Y9, Y9
+	VPADDD     Y8, Y12, Y12
+	VPADDD     Y8, Y13, Y13
+	VPADDD     Y2, Y4, Y4
+	VPADDD     Y15, Y1, Y1
+	VPERM2I128 $0x02, Y0, Y14, Y3
 
 	// Clamp and store poly key
-	VPAND   ·polyClampMask<>(SB), TT0, TT0
-	VMOVDQA TT0, rsStoreAVX2
+	VPAND   ·polyClampMask<>+0(SB), Y3, Y3
+	VMOVDQA Y3, (BP)
 
 	// Stream for up to 192 bytes
-	VPERM2I128 $0x13, AA0, BB0, AA0
-	VPERM2I128 $0x13, CC0, DD0, BB0
-	VPERM2I128 $0x02, AA1, BB1, CC0
-	VPERM2I128 $0x02, CC1, DD1, DD0
-	VPERM2I128 $0x13, AA1, BB1, AA1
-	VPERM2I128 $0x13, CC1, DD1, BB1
+	VPERM2I128 $0x13, Y0, Y14, Y0
+	VPERM2I128 $0x13, Y12, Y4, Y14
+	VPERM2I128 $0x02, Y5, Y9, Y12
+	VPERM2I128 $0x02, Y13, Y1, Y4
+	VPERM2I128 $0x13, Y5, Y9, Y5
+	VPERM2I128 $0x13, Y13, Y1, Y9
 
 openAVX2ShortOpen:
 	// Hash
-	MOVQ ad_len+80(FP), itr2
+	MOVQ ad_len+80(FP), R9
 	CALL polyHashADInternal<>(SB)
 
 openAVX2ShortOpenLoop:
-	CMPQ inl, $32
+	CMPQ BX, $0x20
 	JB   openAVX2ShortTail32
-	SUBQ $32, inl
+	SUBQ $0x20, BX
 
 	// Load for hashing
-	polyAdd(0*8(inp))
-	polyMulAVX2
-	polyAdd(2*8(inp))
-	polyMulAVX2
+	ADDQ  (SI), R10
+	ADCQ  8(SI), R11
+	ADCQ  $0x01, R12
+	MOVQ  (BP), DX
+	MOVQ  DX, R15
+	MULXQ R10, R13, R14
+	IMULQ R12, R15
+	MULXQ R11, AX, DX
+	ADDQ  AX, R14
+	ADCQ  DX, R15
+	MOVQ  8(BP), DX
+	MULXQ R10, R10, AX
+	ADDQ  R10, R14
+	MULXQ R11, R11, R8
+	ADCQ  R11, R15
+	ADCQ  $0x00, R8
+	IMULQ R12, DX
+	ADDQ  AX, R15
+	ADCQ  DX, R8
+	MOVQ  R13, R10
+	MOVQ  R14, R11
+	MOVQ  R15, R12
+	ANDQ  $0x03, R12
+	MOVQ  R15, R13
+	ANDQ  $-4, R13
+	MOVQ  R8, R14
+	SHRQ  $0x02, R8, R15
+	SHRQ  $0x02, R8
+	ADDQ  R13, R10
+	ADCQ  R14, R11
+	ADCQ  $0x00, R12
+	ADDQ  R15, R10
+	ADCQ  R8, R11
+	ADCQ  $0x00, R12
+	ADDQ  16(SI), R10
+	ADCQ  24(SI), R11
+	ADCQ  $0x01, R12
+	MOVQ  (BP), DX
+	MOVQ  DX, R15
+	MULXQ R10, R13, R14
+	IMULQ R12, R15
+	MULXQ R11, AX, DX
+	ADDQ  AX, R14
+	ADCQ  DX, R15
+	MOVQ  8(BP), DX
+	MULXQ R10, R10, AX
+	ADDQ  R10, R14
+	MULXQ R11, R11, R8
+	ADCQ  R11, R15
+	ADCQ  $0x00, R8
+	IMULQ R12, DX
+	ADDQ  AX, R15
+	ADCQ  DX, R8
+	MOVQ  R13, R10
+	MOVQ  R14, R11
+	MOVQ  R15, R12
+	ANDQ  $0x03, R12
+	MOVQ  R15, R13
+	ANDQ  $-4, R13
+	MOVQ  R8, R14
+	SHRQ  $0x02, R8, R15
+	SHRQ  $0x02, R8
+	ADDQ  R13, R10
+	ADCQ  R14, R11
+	ADCQ  $0x00, R12
+	ADDQ  R15, R10
+	ADCQ  R8, R11
+	ADCQ  $0x00, R12
 
 	// Load for decryption
-	VPXOR   (inp), AA0, AA0
-	VMOVDQU AA0, (oup)
-	LEAQ    (1*32)(inp), inp
-	LEAQ    (1*32)(oup), oup
+	VPXOR   (SI), Y0, Y0
+	VMOVDQU Y0, (DI)
+	LEAQ    32(SI), SI
+	LEAQ    32(DI), DI
 
 	// Shift stream left
-	VMOVDQA BB0, AA0
-	VMOVDQA CC0, BB0
-	VMOVDQA DD0, CC0
-	VMOVDQA AA1, DD0
-	VMOVDQA BB1, AA1
-	VMOVDQA CC1, BB1
-	VMOVDQA DD1, CC1
-	VMOVDQA AA2, DD1
-	VMOVDQA BB2, AA2
+	VMOVDQA Y14, Y0
+	VMOVDQA Y12, Y14
+	VMOVDQA Y4, Y12
+	VMOVDQA Y5, Y4
+	VMOVDQA Y9, Y5
+	VMOVDQA Y13, Y9
+	VMOVDQA Y1, Y13
+	VMOVDQA Y6, Y1
+	VMOVDQA Y10, Y6
 	JMP     openAVX2ShortOpenLoop
 
 openAVX2ShortTail32:
-	CMPQ    inl, $16
-	VMOVDQA A0, A1
+	CMPQ    BX, $0x10
+	VMOVDQA X0, X1
 	JB      openAVX2ShortDone
-
-	SUBQ $16, inl
+	SUBQ    $0x10, BX
 
 	// Load for hashing
-	polyAdd(0*8(inp))
-	polyMulAVX2
+	ADDQ  (SI), R10
+	ADCQ  8(SI), R11
+	ADCQ  $0x01, R12
+	MOVQ  (BP), DX
+	MOVQ  DX, R15
+	MULXQ R10, R13, R14
+	IMULQ R12, R15
+	MULXQ R11, AX, DX
+	ADDQ  AX, R14
+	ADCQ  DX, R15
+	MOVQ  8(BP), DX
+	MULXQ R10, R10, AX
+	ADDQ  R10, R14
+	MULXQ R11, R11, R8
+	ADCQ  R11, R15
+	ADCQ  $0x00, R8
+	IMULQ R12, DX
+	ADDQ  AX, R15
+	ADCQ  DX, R8
+	MOVQ  R13, R10
+	MOVQ  R14, R11
+	MOVQ  R15, R12
+	ANDQ  $0x03, R12
+	MOVQ  R15, R13
+	ANDQ  $-4, R13
+	MOVQ  R8, R14
+	SHRQ  $0x02, R8, R15
+	SHRQ  $0x02, R8
+	ADDQ  R13, R10
+	ADCQ  R14, R11
+	ADCQ  $0x00, R12
+	ADDQ  R15, R10
+	ADCQ  R8, R11
+	ADCQ  $0x00, R12
 
 	// Load for decryption
-	VPXOR      (inp), A0, T0
-	VMOVDQU    T0, (oup)
-	LEAQ       (1*16)(inp), inp
-	LEAQ       (1*16)(oup), oup
-	VPERM2I128 $0x11, AA0, AA0, AA0
-	VMOVDQA    A0, A1
+	VPXOR      (SI), X0, X12
+	VMOVDQU    X12, (DI)
+	LEAQ       16(SI), SI
+	LEAQ       16(DI), DI
+	VPERM2I128 $0x11, Y0, Y0, Y0
+	VMOVDQA    X0, X1
 
 openAVX2ShortDone:
 	VZEROUPPER
 	JMP openSSETail16
 
-// ----------------------------------------------------------------------------
-// Special optimization for buffers smaller than 321 bytes
 openAVX2320:
-	// For up to 320 bytes of ciphertext and 64 bytes for the poly key, we process six blocks
-	VMOVDQA AA0, AA1; VMOVDQA BB0, BB1; VMOVDQA CC0, CC1; VPADDD ·avx2IncMask<>(SB), DD0, DD1
-	VMOVDQA AA0, AA2; VMOVDQA BB0, BB2; VMOVDQA CC0, CC2; VPADDD ·avx2IncMask<>(SB), DD1, DD2
-	VMOVDQA BB0, TT1; VMOVDQA CC0, TT2; VMOVDQA DD0, TT3
-	MOVQ    $10, itr2
+	VMOVDQA Y0, Y5
+	VMOVDQA Y14, Y9
+	VMOVDQA Y12, Y13
+	VPADDD  ·avx2IncMask<>+0(SB), Y4, Y1
+	VMOVDQA Y0, Y6
+	VMOVDQA Y14, Y10
+	VMOVDQA Y12, Y8
+	VPADDD  ·avx2IncMask<>+0(SB), Y1, Y2
+	VMOVDQA Y14, Y7
+	VMOVDQA Y12, Y11
+	VMOVDQA Y4, Y15
+	MOVQ    $0x0000000a, R9
 
 openAVX2320InnerCipherLoop:
-	chachaQR_AVX2(AA0, BB0, CC0, DD0, TT0); chachaQR_AVX2(AA1, BB1, CC1, DD1, TT0); chachaQR_AVX2(AA2, BB2, CC2, DD2, TT0)
-	VPALIGNR $4, BB0, BB0, BB0; VPALIGNR $4, BB1, BB1, BB1; VPALIGNR $4, BB2, BB2, BB2
-	VPALIGNR $8, CC0, CC0, CC0; VPALIGNR $8, CC1, CC1, CC1; VPALIGNR $8, CC2, CC2, CC2
-	VPALIGNR $12, DD0, DD0, DD0; VPALIGNR $12, DD1, DD1, DD1; VPALIGNR $12, DD2, DD2, DD2
-	chachaQR_AVX2(AA0, BB0, CC0, DD0, TT0); chachaQR_AVX2(AA1, BB1, CC1, DD1, TT0); chachaQR_AVX2(AA2, BB2, CC2, DD2, TT0)
-	VPALIGNR $12, BB0, BB0, BB0; VPALIGNR $12, BB1, BB1, BB1; VPALIGNR $12, BB2, BB2, BB2
-	VPALIGNR $8, CC0, CC0, CC0; VPALIGNR $8, CC1, CC1, CC1; VPALIGNR $8, CC2, CC2, CC2
-	VPALIGNR $4, DD0, DD0, DD0; VPALIGNR $4, DD1, DD1, DD1; VPALIGNR $4, DD2, DD2, DD2
-	DECQ     itr2
+	VPADDD   Y14, Y0, Y0
+	VPXOR    Y0, Y4, Y4
+	VPSHUFB  ·rol16<>+0(SB), Y4, Y4
+	VPADDD   Y4, Y12, Y12
+	VPXOR    Y12, Y14, Y14
+	VPSLLD   $0x0c, Y14, Y3
+	VPSRLD   $0x14, Y14, Y14
+	VPXOR    Y3, Y14, Y14
+	VPADDD   Y14, Y0, Y0
+	VPXOR    Y0, Y4, Y4
+	VPSHUFB  ·rol8<>+0(SB), Y4, Y4
+	VPADDD   Y4, Y12, Y12
+	VPXOR    Y12, Y14, Y14
+	VPSLLD   $0x07, Y14, Y3
+	VPSRLD   $0x19, Y14, Y14
+	VPXOR    Y3, Y14, Y14
+	VPADDD   Y9, Y5, Y5
+	VPXOR    Y5, Y1, Y1
+	VPSHUFB  ·rol16<>+0(SB), Y1, Y1
+	VPADDD   Y1, Y13, Y13
+	VPXOR    Y13, Y9, Y9
+	VPSLLD   $0x0c, Y9, Y3
+	VPSRLD   $0x14, Y9, Y9
+	VPXOR    Y3, Y9, Y9
+	VPADDD   Y9, Y5, Y5
+	VPXOR    Y5, Y1, Y1
+	VPSHUFB  ·rol8<>+0(SB), Y1, Y1
+	VPADDD   Y1, Y13, Y13
+	VPXOR    Y13, Y9, Y9
+	VPSLLD   $0x07, Y9, Y3
+	VPSRLD   $0x19, Y9, Y9
+	VPXOR    Y3, Y9, Y9
+	VPADDD   Y10, Y6, Y6
+	VPXOR    Y6, Y2, Y2
+	VPSHUFB  ·rol16<>+0(SB), Y2, Y2
+	VPADDD   Y2, Y8, Y8
+	VPXOR    Y8, Y10, Y10
+	VPSLLD   $0x0c, Y10, Y3
+	VPSRLD   $0x14, Y10, Y10
+	VPXOR    Y3, Y10, Y10
+	VPADDD   Y10, Y6, Y6
+	VPXOR    Y6, Y2, Y2
+	VPSHUFB  ·rol8<>+0(SB), Y2, Y2
+	VPADDD   Y2, Y8, Y8
+	VPXOR    Y8, Y10, Y10
+	VPSLLD   $0x07, Y10, Y3
+	VPSRLD   $0x19, Y10, Y10
+	VPXOR    Y3, Y10, Y10
+	VPALIGNR $0x04, Y14, Y14, Y14
+	VPALIGNR $0x04, Y9, Y9, Y9
+	VPALIGNR $0x04, Y10, Y10, Y10
+	VPALIGNR $0x08, Y12, Y12, Y12
+	VPALIGNR $0x08, Y13, Y13, Y13
+	VPALIGNR $0x08, Y8, Y8, Y8
+	VPALIGNR $0x0c, Y4, Y4, Y4
+	VPALIGNR $0x0c, Y1, Y1, Y1
+	VPALIGNR $0x0c, Y2, Y2, Y2
+	VPADDD   Y14, Y0, Y0
+	VPXOR    Y0, Y4, Y4
+	VPSHUFB  ·rol16<>+0(SB), Y4, Y4
+	VPADDD   Y4, Y12, Y12
+	VPXOR    Y12, Y14, Y14
+	VPSLLD   $0x0c, Y14, Y3
+	VPSRLD   $0x14, Y14, Y14
+	VPXOR    Y3, Y14, Y14
+	VPADDD   Y14, Y0, Y0
+	VPXOR    Y0, Y4, Y4
+	VPSHUFB  ·rol8<>+0(SB), Y4, Y4
+	VPADDD   Y4, Y12, Y12
+	VPXOR    Y12, Y14, Y14
+	VPSLLD   $0x07, Y14, Y3
+	VPSRLD   $0x19, Y14, Y14
+	VPXOR    Y3, Y14, Y14
+	VPADDD   Y9, Y5, Y5
+	VPXOR    Y5, Y1, Y1
+	VPSHUFB  ·rol16<>+0(SB), Y1, Y1
+	VPADDD   Y1, Y13, Y13
+	VPXOR    Y13, Y9, Y9
+	VPSLLD   $0x0c, Y9, Y3
+	VPSRLD   $0x14, Y9, Y9
+	VPXOR    Y3, Y9, Y9
+	VPADDD   Y9, Y5, Y5
+	VPXOR    Y5, Y1, Y1
+	VPSHUFB  ·rol8<>+0(SB), Y1, Y1
+	VPADDD   Y1, Y13, Y13
+	VPXOR    Y13, Y9, Y9
+	VPSLLD   $0x07, Y9, Y3
+	VPSRLD   $0x19, Y9, Y9
+	VPXOR    Y3, Y9, Y9
+	VPADDD   Y10, Y6, Y6
+	VPXOR    Y6, Y2, Y2
+	VPSHUFB  ·rol16<>+0(SB), Y2, Y2
+	VPADDD   Y2, Y8, Y8
+	VPXOR    Y8, Y10, Y10
+	VPSLLD   $0x0c, Y10, Y3
+	VPSRLD   $0x14, Y10, Y10
+	VPXOR    Y3, Y10, Y10
+	VPADDD   Y10, Y6, Y6
+	VPXOR    Y6, Y2, Y2
+	VPSHUFB  ·rol8<>+0(SB), Y2, Y2
+	VPADDD   Y2, Y8, Y8
+	VPXOR    Y8, Y10, Y10
+	VPSLLD   $0x07, Y10, Y3
+	VPSRLD   $0x19, Y10, Y10
+	VPXOR    Y3, Y10, Y10
+	VPALIGNR $0x0c, Y14, Y14, Y14
+	VPALIGNR $0x0c, Y9, Y9, Y9
+	VPALIGNR $0x0c, Y10, Y10, Y10
+	VPALIGNR $0x08, Y12, Y12, Y12
+	VPALIGNR $0x08, Y13, Y13, Y13
+	VPALIGNR $0x08, Y8, Y8, Y8
+	VPALIGNR $0x04, Y4, Y4, Y4
+	VPALIGNR $0x04, Y1, Y1, Y1
+	VPALIGNR $0x04, Y2, Y2, Y2
+	DECQ     R9
 	JNE      openAVX2320InnerCipherLoop
-
-	VMOVDQA ·chacha20Constants<>(SB), TT0
-	VPADDD  TT0, AA0, AA0; VPADDD TT0, AA1, AA1; VPADDD TT0, AA2, AA2
-	VPADDD  TT1, BB0, BB0; VPADDD TT1, BB1, BB1; VPADDD TT1, BB2, BB2
-	VPADDD  TT2, CC0, CC0; VPADDD TT2, CC1, CC1; VPADDD TT2, CC2, CC2
-	VMOVDQA ·avx2IncMask<>(SB), TT0
-	VPADDD  TT3, DD0, DD0; VPADDD TT0, TT3, TT3
-	VPADDD  TT3, DD1, DD1; VPADDD TT0, TT3, TT3
-	VPADDD  TT3, DD2, DD2
+	VMOVDQA  ·chacha20Constants<>+0(SB), Y3
+	VPADDD   Y3, Y0, Y0
+	VPADDD   Y3, Y5, Y5
+	VPADDD   Y3, Y6, Y6
+	VPADDD   Y7, Y14, Y14
+	VPADDD   Y7, Y9, Y9
+	VPADDD   Y7, Y10, Y10
+	VPADDD   Y11, Y12, Y12
+	VPADDD   Y11, Y13, Y13
+	VPADDD   Y11, Y8, Y8
+	VMOVDQA  ·avx2IncMask<>+0(SB), Y3
+	VPADDD   Y15, Y4, Y4
+	VPADDD   Y3, Y15, Y15
+	VPADDD   Y15, Y1, Y1
+	VPADDD   Y3, Y15, Y15
+	VPADDD   Y15, Y2, Y2
 
 	// Clamp and store poly key
-	VPERM2I128 $0x02, AA0, BB0, TT0
-	VPAND      ·polyClampMask<>(SB), TT0, TT0
-	VMOVDQA    TT0, rsStoreAVX2
+	VPERM2I128 $0x02, Y0, Y14, Y3
+	VPAND      ·polyClampMask<>+0(SB), Y3, Y3
+	VMOVDQA    Y3, (BP)
 
 	// Stream for up to 320 bytes
-	VPERM2I128 $0x13, AA0, BB0, AA0
-	VPERM2I128 $0x13, CC0, DD0, BB0
-	VPERM2I128 $0x02, AA1, BB1, CC0
-	VPERM2I128 $0x02, CC1, DD1, DD0
-	VPERM2I128 $0x13, AA1, BB1, AA1
-	VPERM2I128 $0x13, CC1, DD1, BB1
-	VPERM2I128 $0x02, AA2, BB2, CC1
-	VPERM2I128 $0x02, CC2, DD2, DD1
-	VPERM2I128 $0x13, AA2, BB2, AA2
-	VPERM2I128 $0x13, CC2, DD2, BB2
+	VPERM2I128 $0x13, Y0, Y14, Y0
+	VPERM2I128 $0x13, Y12, Y4, Y14
+	VPERM2I128 $0x02, Y5, Y9, Y12
+	VPERM2I128 $0x02, Y13, Y1, Y4
+	VPERM2I128 $0x13, Y5, Y9, Y5
+	VPERM2I128 $0x13, Y13, Y1, Y9
+	VPERM2I128 $0x02, Y6, Y10, Y13
+	VPERM2I128 $0x02, Y8, Y2, Y1
+	VPERM2I128 $0x13, Y6, Y10, Y6
+	VPERM2I128 $0x13, Y8, Y2, Y10
 	JMP        openAVX2ShortOpen
 
-// ----------------------------------------------------------------------------
-// Special optimization for the last 128 bytes of ciphertext
 openAVX2Tail128:
 	// Need to decrypt up to 128 bytes - prepare two blocks
-	VMOVDQA ·chacha20Constants<>(SB), AA1
-	VMOVDQA state1StoreAVX2, BB1
-	VMOVDQA state2StoreAVX2, CC1
-	VMOVDQA ctr3StoreAVX2, DD1
-	VPADDD  ·avx2IncMask<>(SB), DD1, DD1
-	VMOVDQA DD1, DD0
-
-	XORQ  itr2, itr2
-	MOVQ  inl, itr1
-	ANDQ  $-16, itr1
-	TESTQ itr1, itr1
-	JE    openAVX2Tail128LoopB
+	VMOVDQA ·chacha20Constants<>+0(SB), Y5
+	VMOVDQA 32(BP), Y9
+	VMOVDQA 64(BP), Y13
+	VMOVDQA 192(BP), Y1
+	VPADDD  ·avx2IncMask<>+0(SB), Y1, Y1
+	VMOVDQA Y1, Y4
+	XORQ    R9, R9
+	MOVQ    BX, CX
+	ANDQ    $-16, CX
+	TESTQ   CX, CX
+	JE      openAVX2Tail128LoopB
 
 openAVX2Tail128LoopA:
-	// Perform ChaCha rounds, while hashing the remaining input
-	polyAdd(0(inp)(itr2*1))
-	polyMulAVX2
+	ADDQ  (SI)(R9*1), R10
+	ADCQ  8(SI)(R9*1), R11
+	ADCQ  $0x01, R12
+	MOVQ  (BP), DX
+	MOVQ  DX, R15
+	MULXQ R10, R13, R14
+	IMULQ R12, R15
+	MULXQ R11, AX, DX
+	ADDQ  AX, R14
+	ADCQ  DX, R15
+	MOVQ  8(BP), DX
+	MULXQ R10, R10, AX
+	ADDQ  R10, R14
+	MULXQ R11, R11, R8
+	ADCQ  R11, R15
+	ADCQ  $0x00, R8
+	IMULQ R12, DX
+	ADDQ  AX, R15
+	ADCQ  DX, R8
+	MOVQ  R13, R10
+	MOVQ  R14, R11
+	MOVQ  R15, R12
+	ANDQ  $0x03, R12
+	MOVQ  R15, R13
+	ANDQ  $-4, R13
+	MOVQ  R8, R14
+	SHRQ  $0x02, R8, R15
+	SHRQ  $0x02, R8
+	ADDQ  R13, R10
+	ADCQ  R14, R11
+	ADCQ  $0x00, R12
+	ADDQ  R15, R10
+	ADCQ  R8, R11
+	ADCQ  $0x00, R12
 
 openAVX2Tail128LoopB:
-	ADDQ     $16, itr2
-	chachaQR_AVX2(AA1, BB1, CC1, DD1, TT0)
-	VPALIGNR $4, BB1, BB1, BB1
-	VPALIGNR $8, CC1, CC1, CC1
-	VPALIGNR $12, DD1, DD1, DD1
-	chachaQR_AVX2(AA1, BB1, CC1, DD1, TT0)
-	VPALIGNR $12, BB1, BB1, BB1
-	VPALIGNR $8, CC1, CC1, CC1
-	VPALIGNR $4, DD1, DD1, DD1
-	CMPQ     itr2, itr1
-	JB       openAVX2Tail128LoopA
-	CMPQ     itr2, $160
-	JNE      openAVX2Tail128LoopB
-
-	VPADDD     ·chacha20Constants<>(SB), AA1, AA1
-	VPADDD     state1StoreAVX2, BB1, BB1
-	VPADDD     state2StoreAVX2, CC1, CC1
-	VPADDD     DD0, DD1, DD1
-	VPERM2I128 $0x02, AA1, BB1, AA0; VPERM2I128 $0x02, CC1, DD1, BB0; VPERM2I128 $0x13, AA1, BB1, CC0; VPERM2I128 $0x13, CC1, DD1, DD0
+	ADDQ       $0x10, R9
+	VPADDD     Y9, Y5, Y5
+	VPXOR      Y5, Y1, Y1
+	VPSHUFB    ·rol16<>+0(SB), Y1, Y1
+	VPADDD     Y1, Y13, Y13
+	VPXOR      Y13, Y9, Y9
+	VPSLLD     $0x0c, Y9, Y3
+	VPSRLD     $0x14, Y9, Y9
+	VPXOR      Y3, Y9, Y9
+	VPADDD     Y9, Y5, Y5
+	VPXOR      Y5, Y1, Y1
+	VPSHUFB    ·rol8<>+0(SB), Y1, Y1
+	VPADDD     Y1, Y13, Y13
+	VPXOR      Y13, Y9, Y9
+	VPSLLD     $0x07, Y9, Y3
+	VPSRLD     $0x19, Y9, Y9
+	VPXOR      Y3, Y9, Y9
+	VPALIGNR   $0x04, Y9, Y9, Y9
+	VPALIGNR   $0x08, Y13, Y13, Y13
+	VPALIGNR   $0x0c, Y1, Y1, Y1
+	VPADDD     Y9, Y5, Y5
+	VPXOR      Y5, Y1, Y1
+	VPSHUFB    ·rol16<>+0(SB), Y1, Y1
+	VPADDD     Y1, Y13, Y13
+	VPXOR      Y13, Y9, Y9
+	VPSLLD     $0x0c, Y9, Y3
+	VPSRLD     $0x14, Y9, Y9
+	VPXOR      Y3, Y9, Y9
+	VPADDD     Y9, Y5, Y5
+	VPXOR      Y5, Y1, Y1
+	VPSHUFB    ·rol8<>+0(SB), Y1, Y1
+	VPADDD     Y1, Y13, Y13
+	VPXOR      Y13, Y9, Y9
+	VPSLLD     $0x07, Y9, Y3
+	VPSRLD     $0x19, Y9, Y9
+	VPXOR      Y3, Y9, Y9
+	VPALIGNR   $0x0c, Y9, Y9, Y9
+	VPALIGNR   $0x08, Y13, Y13, Y13
+	VPALIGNR   $0x04, Y1, Y1, Y1
+	CMPQ       R9, CX
+	JB         openAVX2Tail128LoopA
+	CMPQ       R9, $0xa0
+	JNE        openAVX2Tail128LoopB
+	VPADDD     ·chacha20Constants<>+0(SB), Y5, Y5
+	VPADDD     32(BP), Y9, Y9
+	VPADDD     64(BP), Y13, Y13
+	VPADDD     Y4, Y1, Y1
+	VPERM2I128 $0x02, Y5, Y9, Y0
+	VPERM2I128 $0x02, Y13, Y1, Y14
+	VPERM2I128 $0x13, Y5, Y9, Y12
+	VPERM2I128 $0x13, Y13, Y1, Y4
 
 openAVX2TailLoop:
-	CMPQ inl, $32
+	CMPQ BX, $0x20
 	JB   openAVX2Tail
-	SUBQ $32, inl
+	SUBQ $0x20, BX
 
 	// Load for decryption
-	VPXOR   (inp), AA0, AA0
-	VMOVDQU AA0, (oup)
-	LEAQ    (1*32)(inp), inp
-	LEAQ    (1*32)(oup), oup
-	VMOVDQA BB0, AA0
-	VMOVDQA CC0, BB0
-	VMOVDQA DD0, CC0
+	VPXOR   (SI), Y0, Y0
+	VMOVDQU Y0, (DI)
+	LEAQ    32(SI), SI
+	LEAQ    32(DI), DI
+	VMOVDQA Y14, Y0
+	VMOVDQA Y12, Y14
+	VMOVDQA Y4, Y12
 	JMP     openAVX2TailLoop
 
 openAVX2Tail:
-	CMPQ    inl, $16
-	VMOVDQA A0, A1
+	CMPQ    BX, $0x10
+	VMOVDQA X0, X1
 	JB      openAVX2TailDone
-	SUBQ    $16, inl
+	SUBQ    $0x10, BX
 
 	// Load for decryption
-	VPXOR      (inp), A0, T0
-	VMOVDQU    T0, (oup)
-	LEAQ       (1*16)(inp), inp
-	LEAQ       (1*16)(oup), oup
-	VPERM2I128 $0x11, AA0, AA0, AA0
-	VMOVDQA    A0, A1
+	VPXOR      (SI), X0, X12
+	VMOVDQU    X12, (DI)
+	LEAQ       16(SI), SI
+	LEAQ       16(DI), DI
+	VPERM2I128 $0x11, Y0, Y0, Y0
+	VMOVDQA    X0, X1
 
 openAVX2TailDone:
 	VZEROUPPER
 	JMP openSSETail16
 
-// ----------------------------------------------------------------------------
-// Special optimization for the last 256 bytes of ciphertext
 openAVX2Tail256:
-	// Need to decrypt up to 256 bytes - prepare four blocks
-	VMOVDQA ·chacha20Constants<>(SB), AA0; VMOVDQA AA0, AA1
-	VMOVDQA state1StoreAVX2, BB0; VMOVDQA BB0, BB1
-	VMOVDQA state2StoreAVX2, CC0; VMOVDQA CC0, CC1
-	VMOVDQA ctr3StoreAVX2, DD0
-	VPADDD  ·avx2IncMask<>(SB), DD0, DD0
-	VPADDD  ·avx2IncMask<>(SB), DD0, DD1
-	VMOVDQA DD0, TT1
-	VMOVDQA DD1, TT2
+	VMOVDQA ·chacha20Constants<>+0(SB), Y0
+	VMOVDQA Y0, Y5
+	VMOVDQA 32(BP), Y14
+	VMOVDQA Y14, Y9
+	VMOVDQA 64(BP), Y12
+	VMOVDQA Y12, Y13
+	VMOVDQA 192(BP), Y4
+	VPADDD  ·avx2IncMask<>+0(SB), Y4, Y4
+	VPADDD  ·avx2IncMask<>+0(SB), Y4, Y1
+	VMOVDQA Y4, Y7
+	VMOVDQA Y1, Y11
 
 	// Compute the number of iterations that will hash data
-	MOVQ    inl, tmpStoreAVX2
-	MOVQ    inl, itr1
-	SUBQ    $128, itr1
-	SHRQ    $4, itr1
-	MOVQ    $10, itr2
-	CMPQ    itr1, $10
-	CMOVQGT itr2, itr1
-	MOVQ    inp, inl
-	XORQ    itr2, itr2
+	MOVQ    BX, 224(BP)
+	MOVQ    BX, CX
+	SUBQ    $0x80, CX
+	SHRQ    $0x04, CX
+	MOVQ    $0x0000000a, R9
+	CMPQ    CX, $0x0a
+	CMOVQGT R9, CX
+	MOVQ    SI, BX
+	XORQ    R9, R9
 
 openAVX2Tail256LoopA:
-	polyAdd(0(inl))
-	polyMulAVX2
-	LEAQ 16(inl), inl
+	ADDQ  (BX), R10
+	ADCQ  8(BX), R11
+	ADCQ  $0x01, R12
+	MOVQ  (BP), DX
+	MOVQ  DX, R15
+	MULXQ R10, R13, R14
+	IMULQ R12, R15
+	MULXQ R11, AX, DX
+	ADDQ  AX, R14
+	ADCQ  DX, R15
+	MOVQ  8(BP), DX
+	MULXQ R10, R10, AX
+	ADDQ  R10, R14
+	MULXQ R11, R11, R8
+	ADCQ  R11, R15
+	ADCQ  $0x00, R8
+	IMULQ R12, DX
+	ADDQ  AX, R15
+	ADCQ  DX, R8
+	MOVQ  R13, R10
+	MOVQ  R14, R11
+	MOVQ  R15, R12
+	ANDQ  $0x03, R12
+	MOVQ  R15, R13
+	ANDQ  $-4, R13
+	MOVQ  R8, R14
+	SHRQ  $0x02, R8, R15
+	SHRQ  $0x02, R8
+	ADDQ  R13, R10
+	ADCQ  R14, R11
+	ADCQ  $0x00, R12
+	ADDQ  R15, R10
+	ADCQ  R8, R11
+	ADCQ  $0x00, R12
+	LEAQ  16(BX), BX
 
-	// Perform ChaCha rounds, while hashing the remaining input
 openAVX2Tail256LoopB:
-	chachaQR_AVX2(AA0, BB0, CC0, DD0, TT0); chachaQR_AVX2(AA1, BB1, CC1, DD1, TT0)
-	VPALIGNR $4, BB0, BB0, BB0; VPALIGNR $4, BB1, BB1, BB1
-	VPALIGNR $8, CC0, CC0, CC0; VPALIGNR $8, CC1, CC1, CC1
-	VPALIGNR $12, DD0, DD0, DD0; VPALIGNR $12, DD1, DD1, DD1
-	INCQ     itr2
-	chachaQR_AVX2(AA0, BB0, CC0, DD0, TT0); chachaQR_AVX2(AA1, BB1, CC1, DD1, TT0)
-	VPALIGNR $12, BB0, BB0, BB0; VPALIGNR $12, BB1, BB1, BB1
-	VPALIGNR $8, CC0, CC0, CC0; VPALIGNR $8, CC1, CC1, CC1
-	VPALIGNR $4, DD0, DD0, DD0; VPALIGNR $4, DD1, DD1, DD1
-	CMPQ     itr2, itr1
+	VPADDD   Y14, Y0, Y0
+	VPXOR    Y0, Y4, Y4
+	VPSHUFB  ·rol16<>+0(SB), Y4, Y4
+	VPADDD   Y4, Y12, Y12
+	VPXOR    Y12, Y14, Y14
+	VPSLLD   $0x0c, Y14, Y3
+	VPSRLD   $0x14, Y14, Y14
+	VPXOR    Y3, Y14, Y14
+	VPADDD   Y14, Y0, Y0
+	VPXOR    Y0, Y4, Y4
+	VPSHUFB  ·rol8<>+0(SB), Y4, Y4
+	VPADDD   Y4, Y12, Y12
+	VPXOR    Y12, Y14, Y14
+	VPSLLD   $0x07, Y14, Y3
+	VPSRLD   $0x19, Y14, Y14
+	VPXOR    Y3, Y14, Y14
+	VPADDD   Y9, Y5, Y5
+	VPXOR    Y5, Y1, Y1
+	VPSHUFB  ·rol16<>+0(SB), Y1, Y1
+	VPADDD   Y1, Y13, Y13
+	VPXOR    Y13, Y9, Y9
+	VPSLLD   $0x0c, Y9, Y3
+	VPSRLD   $0x14, Y9, Y9
+	VPXOR    Y3, Y9, Y9
+	VPADDD   Y9, Y5, Y5
+	VPXOR    Y5, Y1, Y1
+	VPSHUFB  ·rol8<>+0(SB), Y1, Y1
+	VPADDD   Y1, Y13, Y13
+	VPXOR    Y13, Y9, Y9
+	VPSLLD   $0x07, Y9, Y3
+	VPSRLD   $0x19, Y9, Y9
+	VPXOR    Y3, Y9, Y9
+	VPALIGNR $0x04, Y14, Y14, Y14
+	VPALIGNR $0x04, Y9, Y9, Y9
+	VPALIGNR $0x08, Y12, Y12, Y12
+	VPALIGNR $0x08, Y13, Y13, Y13
+	VPALIGNR $0x0c, Y4, Y4, Y4
+	VPALIGNR $0x0c, Y1, Y1, Y1
+	INCQ     R9
+	VPADDD   Y14, Y0, Y0
+	VPXOR    Y0, Y4, Y4
+	VPSHUFB  ·rol16<>+0(SB), Y4, Y4
+	VPADDD   Y4, Y12, Y12
+	VPXOR    Y12, Y14, Y14
+	VPSLLD   $0x0c, Y14, Y3
+	VPSRLD   $0x14, Y14, Y14
+	VPXOR    Y3, Y14, Y14
+	VPADDD   Y14, Y0, Y0
+	VPXOR    Y0, Y4, Y4
+	VPSHUFB  ·rol8<>+0(SB), Y4, Y4
+	VPADDD   Y4, Y12, Y12
+	VPXOR    Y12, Y14, Y14
+	VPSLLD   $0x07, Y14, Y3
+	VPSRLD   $0x19, Y14, Y14
+	VPXOR    Y3, Y14, Y14
+	VPADDD   Y9, Y5, Y5
+	VPXOR    Y5, Y1, Y1
+	VPSHUFB  ·rol16<>+0(SB), Y1, Y1
+	VPADDD   Y1, Y13, Y13
+	VPXOR    Y13, Y9, Y9
+	VPSLLD   $0x0c, Y9, Y3
+	VPSRLD   $0x14, Y9, Y9
+	VPXOR    Y3, Y9, Y9
+	VPADDD   Y9, Y5, Y5
+	VPXOR    Y5, Y1, Y1
+	VPSHUFB  ·rol8<>+0(SB), Y1, Y1
+	VPADDD   Y1, Y13, Y13
+	VPXOR    Y13, Y9, Y9
+	VPSLLD   $0x07, Y9, Y3
+	VPSRLD   $0x19, Y9, Y9
+	VPXOR    Y3, Y9, Y9
+	VPALIGNR $0x0c, Y14, Y14, Y14
+	VPALIGNR $0x0c, Y9, Y9, Y9
+	VPALIGNR $0x08, Y12, Y12, Y12
+	VPALIGNR $0x08, Y13, Y13, Y13
+	VPALIGNR $0x04, Y4, Y4, Y4
+	VPALIGNR $0x04, Y1, Y1, Y1
+	CMPQ     R9, CX
 	JB       openAVX2Tail256LoopA
+	CMPQ     R9, $0x0a
+	JNE      openAVX2Tail256LoopB
+	MOVQ     BX, R9
+	SUBQ     SI, BX
+	MOVQ     BX, CX
+	MOVQ     224(BP), BX
 
-	CMPQ itr2, $10
-	JNE  openAVX2Tail256LoopB
-
-	MOVQ inl, itr2
-	SUBQ inp, inl
-	MOVQ inl, itr1
-	MOVQ tmpStoreAVX2, inl
-
-	// Hash the remainder of data (if any)
 openAVX2Tail256Hash:
-	ADDQ $16, itr1
-	CMPQ itr1, inl
-	JGT  openAVX2Tail256HashEnd
-	polyAdd (0(itr2))
-	polyMulAVX2
-	LEAQ 16(itr2), itr2
-	JMP  openAVX2Tail256Hash
-
-// Store 128 bytes safely, then go to store loop
+	ADDQ  $0x10, CX
+	CMPQ  CX, BX
+	JGT   openAVX2Tail256HashEnd
+	ADDQ  (R9), R10
+	ADCQ  8(R9), R11
+	ADCQ  $0x01, R12
+	MOVQ  (BP), DX
+	MOVQ  DX, R15
+	MULXQ R10, R13, R14
+	IMULQ R12, R15
+	MULXQ R11, AX, DX
+	ADDQ  AX, R14
+	ADCQ  DX, R15
+	MOVQ  8(BP), DX
+	MULXQ R10, R10, AX
+	ADDQ  R10, R14
+	MULXQ R11, R11, R8
+	ADCQ  R11, R15
+	ADCQ  $0x00, R8
+	IMULQ R12, DX
+	ADDQ  AX, R15
+	ADCQ  DX, R8
+	MOVQ  R13, R10
+	MOVQ  R14, R11
+	MOVQ  R15, R12
+	ANDQ  $0x03, R12
+	MOVQ  R15, R13
+	ANDQ  $-4, R13
+	MOVQ  R8, R14
+	SHRQ  $0x02, R8, R15
+	SHRQ  $0x02, R8
+	ADDQ  R13, R10
+	ADCQ  R14, R11
+	ADCQ  $0x00, R12
+	ADDQ  R15, R10
+	ADCQ  R8, R11
+	ADCQ  $0x00, R12
+	LEAQ  16(R9), R9
+	JMP   openAVX2Tail256Hash
+
 openAVX2Tail256HashEnd:
-	VPADDD     ·chacha20Constants<>(SB), AA0, AA0; VPADDD ·chacha20Constants<>(SB), AA1, AA1
-	VPADDD     state1StoreAVX2, BB0, BB0; VPADDD state1StoreAVX2, BB1, BB1
-	VPADDD     state2StoreAVX2, CC0, CC0; VPADDD state2StoreAVX2, CC1, CC1
-	VPADDD     TT1, DD0, DD0; VPADDD TT2, DD1, DD1
-	VPERM2I128 $0x02, AA0, BB0, AA2; VPERM2I128 $0x02, CC0, DD0, BB2; VPERM2I128 $0x13, AA0, BB0, CC2; VPERM2I128 $0x13, CC0, DD0, DD2
-	VPERM2I128 $0x02, AA1, BB1, AA0; VPERM2I128 $0x02, CC1, DD1, BB0; VPERM2I128 $0x13, AA1, BB1, CC0; VPERM2I128 $0x13, CC1, DD1, DD0
-
-	VPXOR   (0*32)(inp), AA2, AA2; VPXOR (1*32)(inp), BB2, BB2; VPXOR (2*32)(inp), CC2, CC2; VPXOR (3*32)(inp), DD2, DD2
-	VMOVDQU AA2, (0*32)(oup); VMOVDQU BB2, (1*32)(oup); VMOVDQU CC2, (2*32)(oup); VMOVDQU DD2, (3*32)(oup)
-	LEAQ    (4*32)(inp), inp
-	LEAQ    (4*32)(oup), oup
-	SUBQ    $4*32, inl
-
-	JMP openAVX2TailLoop
-
-// ----------------------------------------------------------------------------
-// Special optimization for the last 384 bytes of ciphertext
+	VPADDD     ·chacha20Constants<>+0(SB), Y0, Y0
+	VPADDD     ·chacha20Constants<>+0(SB), Y5, Y5
+	VPADDD     32(BP), Y14, Y14
+	VPADDD     32(BP), Y9, Y9
+	VPADDD     64(BP), Y12, Y12
+	VPADDD     64(BP), Y13, Y13
+	VPADDD     Y7, Y4, Y4
+	VPADDD     Y11, Y1, Y1
+	VPERM2I128 $0x02, Y0, Y14, Y6
+	VPERM2I128 $0x02, Y12, Y4, Y10
+	VPERM2I128 $0x13, Y0, Y14, Y8
+	VPERM2I128 $0x13, Y12, Y4, Y2
+	VPERM2I128 $0x02, Y5, Y9, Y0
+	VPERM2I128 $0x02, Y13, Y1, Y14
+	VPERM2I128 $0x13, Y5, Y9, Y12
+	VPERM2I128 $0x13, Y13, Y1, Y4
+	VPXOR      (SI), Y6, Y6
+	VPXOR      32(SI), Y10, Y10
+	VPXOR      64(SI), Y8, Y8
+	VPXOR      96(SI), Y2, Y2
+	VMOVDQU    Y6, (DI)
+	VMOVDQU    Y10, 32(DI)
+	VMOVDQU    Y8, 64(DI)
+	VMOVDQU    Y2, 96(DI)
+	LEAQ       128(SI), SI
+	LEAQ       128(DI), DI
+	SUBQ       $0x80, BX
+	JMP        openAVX2TailLoop
+
 openAVX2Tail384:
 	// Need to decrypt up to 384 bytes - prepare six blocks
-	VMOVDQA ·chacha20Constants<>(SB), AA0; VMOVDQA AA0, AA1; VMOVDQA AA0, AA2
-	VMOVDQA state1StoreAVX2, BB0; VMOVDQA BB0, BB1; VMOVDQA BB0, BB2
-	VMOVDQA state2StoreAVX2, CC0; VMOVDQA CC0, CC1; VMOVDQA CC0, CC2
-	VMOVDQA ctr3StoreAVX2, DD0
-	VPADDD  ·avx2IncMask<>(SB), DD0, DD0
-	VPADDD  ·avx2IncMask<>(SB), DD0, DD1
-	VPADDD  ·avx2IncMask<>(SB), DD1, DD2
-	VMOVDQA DD0, ctr0StoreAVX2
-	VMOVDQA DD1, ctr1StoreAVX2
-	VMOVDQA DD2, ctr2StoreAVX2
+	VMOVDQA ·chacha20Constants<>+0(SB), Y0
+	VMOVDQA Y0, Y5
+	VMOVDQA Y0, Y6
+	VMOVDQA 32(BP), Y14
+	VMOVDQA Y14, Y9
+	VMOVDQA Y14, Y10
+	VMOVDQA 64(BP), Y12
+	VMOVDQA Y12, Y13
+	VMOVDQA Y12, Y8
+	VMOVDQA 192(BP), Y4
+	VPADDD  ·avx2IncMask<>+0(SB), Y4, Y4
+	VPADDD  ·avx2IncMask<>+0(SB), Y4, Y1
+	VPADDD  ·avx2IncMask<>+0(SB), Y1, Y2
+	VMOVDQA Y4, 96(BP)
+	VMOVDQA Y1, 128(BP)
+	VMOVDQA Y2, 160(BP)
 
 	// Compute the number of iterations that will hash two blocks of data
-	MOVQ    inl, tmpStoreAVX2
-	MOVQ    inl, itr1
-	SUBQ    $256, itr1
-	SHRQ    $4, itr1
-	ADDQ    $6, itr1
-	MOVQ    $10, itr2
-	CMPQ    itr1, $10
-	CMOVQGT itr2, itr1
-	MOVQ    inp, inl
-	XORQ    itr2, itr2
-
-	// Perform ChaCha rounds, while hashing the remaining input
+	MOVQ    BX, 224(BP)
+	MOVQ    BX, CX
+	SUBQ    $0x00000100, CX
+	SHRQ    $0x04, CX
+	ADDQ    $0x06, CX
+	MOVQ    $0x0000000a, R9
+	CMPQ    CX, $0x0a
+	CMOVQGT R9, CX
+	MOVQ    SI, BX
+	XORQ    R9, R9
+
 openAVX2Tail384LoopB:
-	polyAdd(0(inl))
-	polyMulAVX2
-	LEAQ 16(inl), inl
+	ADDQ  (BX), R10
+	ADCQ  8(BX), R11
+	ADCQ  $0x01, R12
+	MOVQ  (BP), DX
+	MOVQ  DX, R15
+	MULXQ R10, R13, R14
+	IMULQ R12, R15
+	MULXQ R11, AX, DX
+	ADDQ  AX, R14
+	ADCQ  DX, R15
+	MOVQ  8(BP), DX
+	MULXQ R10, R10, AX
+	ADDQ  R10, R14
+	MULXQ R11, R11, R8
+	ADCQ  R11, R15
+	ADCQ  $0x00, R8
+	IMULQ R12, DX
+	ADDQ  AX, R15
+	ADCQ  DX, R8
+	MOVQ  R13, R10
+	MOVQ  R14, R11
+	MOVQ  R15, R12
+	ANDQ  $0x03, R12
+	MOVQ  R15, R13
+	ANDQ  $-4, R13
+	MOVQ  R8, R14
+	SHRQ  $0x02, R8, R15
+	SHRQ  $0x02, R8
+	ADDQ  R13, R10
+	ADCQ  R14, R11
+	ADCQ  $0x00, R12
+	ADDQ  R15, R10
+	ADCQ  R8, R11
+	ADCQ  $0x00, R12
+	LEAQ  16(BX), BX
 
 openAVX2Tail384LoopA:
-	chachaQR_AVX2(AA0, BB0, CC0, DD0, TT0); chachaQR_AVX2(AA1, BB1, CC1, DD1, TT0); chachaQR_AVX2(AA2, BB2, CC2, DD2, TT0)
-	VPALIGNR $4, BB0, BB0, BB0; VPALIGNR $4, BB1, BB1, BB1; VPALIGNR $4, BB2, BB2, BB2
-	VPALIGNR $8, CC0, CC0, CC0; VPALIGNR $8, CC1, CC1, CC1; VPALIGNR $8, CC2, CC2, CC2
-	VPALIGNR $12, DD0, DD0, DD0; VPALIGNR $12, DD1, DD1, DD1; VPALIGNR $12, DD2, DD2, DD2
-	polyAdd(0(inl))
-	polyMulAVX2
-	LEAQ     16(inl), inl
-	INCQ     itr2
-	chachaQR_AVX2(AA0, BB0, CC0, DD0, TT0); chachaQR_AVX2(AA1, BB1, CC1, DD1, TT0); chachaQR_AVX2(AA2, BB2, CC2, DD2, TT0)
-	VPALIGNR $12, BB0, BB0, BB0; VPALIGNR $12, BB1, BB1, BB1; VPALIGNR $12, BB2, BB2, BB2
-	VPALIGNR $8, CC0, CC0, CC0; VPALIGNR $8, CC1, CC1, CC1; VPALIGNR $8, CC2, CC2, CC2
-	VPALIGNR $4, DD0, DD0, DD0; VPALIGNR $4, DD1, DD1, DD1; VPALIGNR $4, DD2, DD2, DD2
-
-	CMPQ itr2, itr1
-	JB   openAVX2Tail384LoopB
-
-	CMPQ itr2, $10
-	JNE  openAVX2Tail384LoopA
-
-	MOVQ inl, itr2
-	SUBQ inp, inl
-	MOVQ inl, itr1
-	MOVQ tmpStoreAVX2, inl
+	VPADDD   Y14, Y0, Y0
+	VPXOR    Y0, Y4, Y4
+	VPSHUFB  ·rol16<>+0(SB), Y4, Y4
+	VPADDD   Y4, Y12, Y12
+	VPXOR    Y12, Y14, Y14
+	VPSLLD   $0x0c, Y14, Y3
+	VPSRLD   $0x14, Y14, Y14
+	VPXOR    Y3, Y14, Y14
+	VPADDD   Y14, Y0, Y0
+	VPXOR    Y0, Y4, Y4
+	VPSHUFB  ·rol8<>+0(SB), Y4, Y4
+	VPADDD   Y4, Y12, Y12
+	VPXOR    Y12, Y14, Y14
+	VPSLLD   $0x07, Y14, Y3
+	VPSRLD   $0x19, Y14, Y14
+	VPXOR    Y3, Y14, Y14
+	VPADDD   Y9, Y5, Y5
+	VPXOR    Y5, Y1, Y1
+	VPSHUFB  ·rol16<>+0(SB), Y1, Y1
+	VPADDD   Y1, Y13, Y13
+	VPXOR    Y13, Y9, Y9
+	VPSLLD   $0x0c, Y9, Y3
+	VPSRLD   $0x14, Y9, Y9
+	VPXOR    Y3, Y9, Y9
+	VPADDD   Y9, Y5, Y5
+	VPXOR    Y5, Y1, Y1
+	VPSHUFB  ·rol8<>+0(SB), Y1, Y1
+	VPADDD   Y1, Y13, Y13
+	VPXOR    Y13, Y9, Y9
+	VPSLLD   $0x07, Y9, Y3
+	VPSRLD   $0x19, Y9, Y9
+	VPXOR    Y3, Y9, Y9
+	VPADDD   Y10, Y6, Y6
+	VPXOR    Y6, Y2, Y2
+	VPSHUFB  ·rol16<>+0(SB), Y2, Y2
+	VPADDD   Y2, Y8, Y8
+	VPXOR    Y8, Y10, Y10
+	VPSLLD   $0x0c, Y10, Y3
+	VPSRLD   $0x14, Y10, Y10
+	VPXOR    Y3, Y10, Y10
+	VPADDD   Y10, Y6, Y6
+	VPXOR    Y6, Y2, Y2
+	VPSHUFB  ·rol8<>+0(SB), Y2, Y2
+	VPADDD   Y2, Y8, Y8
+	VPXOR    Y8, Y10, Y10
+	VPSLLD   $0x07, Y10, Y3
+	VPSRLD   $0x19, Y10, Y10
+	VPXOR    Y3, Y10, Y10
+	VPALIGNR $0x04, Y14, Y14, Y14
+	VPALIGNR $0x04, Y9, Y9, Y9
+	VPALIGNR $0x04, Y10, Y10, Y10
+	VPALIGNR $0x08, Y12, Y12, Y12
+	VPALIGNR $0x08, Y13, Y13, Y13
+	VPALIGNR $0x08, Y8, Y8, Y8
+	VPALIGNR $0x0c, Y4, Y4, Y4
+	VPALIGNR $0x0c, Y1, Y1, Y1
+	VPALIGNR $0x0c, Y2, Y2, Y2
+	ADDQ     (BX), R10
+	ADCQ     8(BX), R11
+	ADCQ     $0x01, R12
+	MOVQ     (BP), DX
+	MOVQ     DX, R15
+	MULXQ    R10, R13, R14
+	IMULQ    R12, R15
+	MULXQ    R11, AX, DX
+	ADDQ     AX, R14
+	ADCQ     DX, R15
+	MOVQ     8(BP), DX
+	MULXQ    R10, R10, AX
+	ADDQ     R10, R14
+	MULXQ    R11, R11, R8
+	ADCQ     R11, R15
+	ADCQ     $0x00, R8
+	IMULQ    R12, DX
+	ADDQ     AX, R15
+	ADCQ     DX, R8
+	MOVQ     R13, R10
+	MOVQ     R14, R11
+	MOVQ     R15, R12
+	ANDQ     $0x03, R12
+	MOVQ     R15, R13
+	ANDQ     $-4, R13
+	MOVQ     R8, R14
+	SHRQ     $0x02, R8, R15
+	SHRQ     $0x02, R8
+	ADDQ     R13, R10
+	ADCQ     R14, R11
+	ADCQ     $0x00, R12
+	ADDQ     R15, R10
+	ADCQ     R8, R11
+	ADCQ     $0x00, R12
+	LEAQ     16(BX), BX
+	INCQ     R9
+	VPADDD   Y14, Y0, Y0
+	VPXOR    Y0, Y4, Y4
+	VPSHUFB  ·rol16<>+0(SB), Y4, Y4
+	VPADDD   Y4, Y12, Y12
+	VPXOR    Y12, Y14, Y14
+	VPSLLD   $0x0c, Y14, Y3
+	VPSRLD   $0x14, Y14, Y14
+	VPXOR    Y3, Y14, Y14
+	VPADDD   Y14, Y0, Y0
+	VPXOR    Y0, Y4, Y4
+	VPSHUFB  ·rol8<>+0(SB), Y4, Y4
+	VPADDD   Y4, Y12, Y12
+	VPXOR    Y12, Y14, Y14
+	VPSLLD   $0x07, Y14, Y3
+	VPSRLD   $0x19, Y14, Y14
+	VPXOR    Y3, Y14, Y14
+	VPADDD   Y9, Y5, Y5
+	VPXOR    Y5, Y1, Y1
+	VPSHUFB  ·rol16<>+0(SB), Y1, Y1
+	VPADDD   Y1, Y13, Y13
+	VPXOR    Y13, Y9, Y9
+	VPSLLD   $0x0c, Y9, Y3
+	VPSRLD   $0x14, Y9, Y9
+	VPXOR    Y3, Y9, Y9
+	VPADDD   Y9, Y5, Y5
+	VPXOR    Y5, Y1, Y1
+	VPSHUFB  ·rol8<>+0(SB), Y1, Y1
+	VPADDD   Y1, Y13, Y13
+	VPXOR    Y13, Y9, Y9
+	VPSLLD   $0x07, Y9, Y3
+	VPSRLD   $0x19, Y9, Y9
+	VPXOR    Y3, Y9, Y9
+	VPADDD   Y10, Y6, Y6
+	VPXOR    Y6, Y2, Y2
+	VPSHUFB  ·rol16<>+0(SB), Y2, Y2
+	VPADDD   Y2, Y8, Y8
+	VPXOR    Y8, Y10, Y10
+	VPSLLD   $0x0c, Y10, Y3
+	VPSRLD   $0x14, Y10, Y10
+	VPXOR    Y3, Y10, Y10
+	VPADDD   Y10, Y6, Y6
+	VPXOR    Y6, Y2, Y2
+	VPSHUFB  ·rol8<>+0(SB), Y2, Y2
+	VPADDD   Y2, Y8, Y8
+	VPXOR    Y8, Y10, Y10
+	VPSLLD   $0x07, Y10, Y3
+	VPSRLD   $0x19, Y10, Y10
+	VPXOR    Y3, Y10, Y10
+	VPALIGNR $0x0c, Y14, Y14, Y14
+	VPALIGNR $0x0c, Y9, Y9, Y9
+	VPALIGNR $0x0c, Y10, Y10, Y10
+	VPALIGNR $0x08, Y12, Y12, Y12
+	VPALIGNR $0x08, Y13, Y13, Y13
+	VPALIGNR $0x08, Y8, Y8, Y8
+	VPALIGNR $0x04, Y4, Y4, Y4
+	VPALIGNR $0x04, Y1, Y1, Y1
+	VPALIGNR $0x04, Y2, Y2, Y2
+	CMPQ     R9, CX
+	JB       openAVX2Tail384LoopB
+	CMPQ     R9, $0x0a
+	JNE      openAVX2Tail384LoopA
+	MOVQ     BX, R9
+	SUBQ     SI, BX
+	MOVQ     BX, CX
+	MOVQ     224(BP), BX
 
 openAVX2Tail384Hash:
-	ADDQ $16, itr1
-	CMPQ itr1, inl
-	JGT  openAVX2Tail384HashEnd
-	polyAdd(0(itr2))
-	polyMulAVX2
-	LEAQ 16(itr2), itr2
-	JMP  openAVX2Tail384Hash
-
-// Store 256 bytes safely, then go to store loop
+	ADDQ  $0x10, CX
+	CMPQ  CX, BX
+	JGT   openAVX2Tail384HashEnd
+	ADDQ  (R9), R10
+	ADCQ  8(R9), R11
+	ADCQ  $0x01, R12
+	MOVQ  (BP), DX
+	MOVQ  DX, R15
+	MULXQ R10, R13, R14
+	IMULQ R12, R15
+	MULXQ R11, AX, DX
+	ADDQ  AX, R14
+	ADCQ  DX, R15
+	MOVQ  8(BP), DX
+	MULXQ R10, R10, AX
+	ADDQ  R10, R14
+	MULXQ R11, R11, R8
+	ADCQ  R11, R15
+	ADCQ  $0x00, R8
+	IMULQ R12, DX
+	ADDQ  AX, R15
+	ADCQ  DX, R8
+	MOVQ  R13, R10
+	MOVQ  R14, R11
+	MOVQ  R15, R12
+	ANDQ  $0x03, R12
+	MOVQ  R15, R13
+	ANDQ  $-4, R13
+	MOVQ  R8, R14
+	SHRQ  $0x02, R8, R15
+	SHRQ  $0x02, R8
+	ADDQ  R13, R10
+	ADCQ  R14, R11
+	ADCQ  $0x00, R12
+	ADDQ  R15, R10
+	ADCQ  R8, R11
+	ADCQ  $0x00, R12
+	LEAQ  16(R9), R9
+	JMP   openAVX2Tail384Hash
+
 openAVX2Tail384HashEnd:
-	VPADDD     ·chacha20Constants<>(SB), AA0, AA0; VPADDD ·chacha20Constants<>(SB), AA1, AA1; VPADDD ·chacha20Constants<>(SB), AA2, AA2
-	VPADDD     state1StoreAVX2, BB0, BB0; VPADDD state1StoreAVX2, BB1, BB1; VPADDD state1StoreAVX2, BB2, BB2
-	VPADDD     state2StoreAVX2, CC0, CC0; VPADDD state2StoreAVX2, CC1, CC1; VPADDD state2StoreAVX2, CC2, CC2
-	VPADDD     ctr0StoreAVX2, DD0, DD0; VPADDD ctr1StoreAVX2, DD1, DD1; VPADDD ctr2StoreAVX2, DD2, DD2
-	VPERM2I128 $0x02, AA0, BB0, TT0; VPERM2I128 $0x02, CC0, DD0, TT1; VPERM2I128 $0x13, AA0, BB0, TT2; VPERM2I128 $0x13, CC0, DD0, TT3
-	VPXOR      (0*32)(inp), TT0, TT0; VPXOR (1*32)(inp), TT1, TT1; VPXOR (2*32)(inp), TT2, TT2; VPXOR (3*32)(inp), TT3, TT3
-	VMOVDQU    TT0, (0*32)(oup); VMOVDQU TT1, (1*32)(oup); VMOVDQU TT2, (2*32)(oup); VMOVDQU TT3, (3*32)(oup)
-	VPERM2I128 $0x02, AA1, BB1, TT0; VPERM2I128 $0x02, CC1, DD1, TT1; VPERM2I128 $0x13, AA1, BB1, TT2; VPERM2I128 $0x13, CC1, DD1, TT3
-	VPXOR      (4*32)(inp), TT0, TT0; VPXOR (5*32)(inp), TT1, TT1; VPXOR (6*32)(inp), TT2, TT2; VPXOR (7*32)(inp), TT3, TT3
-	VMOVDQU    TT0, (4*32)(oup); VMOVDQU TT1, (5*32)(oup); VMOVDQU TT2, (6*32)(oup); VMOVDQU TT3, (7*32)(oup)
-	VPERM2I128 $0x02, AA2, BB2, AA0; VPERM2I128 $0x02, CC2, DD2, BB0; VPERM2I128 $0x13, AA2, BB2, CC0; VPERM2I128 $0x13, CC2, DD2, DD0
-	LEAQ       (8*32)(inp), inp
-	LEAQ       (8*32)(oup), oup
-	SUBQ       $8*32, inl
+	VPADDD     ·chacha20Constants<>+0(SB), Y0, Y0
+	VPADDD     ·chacha20Constants<>+0(SB), Y5, Y5
+	VPADDD     ·chacha20Constants<>+0(SB), Y6, Y6
+	VPADDD     32(BP), Y14, Y14
+	VPADDD     32(BP), Y9, Y9
+	VPADDD     32(BP), Y10, Y10
+	VPADDD     64(BP), Y12, Y12
+	VPADDD     64(BP), Y13, Y13
+	VPADDD     64(BP), Y8, Y8
+	VPADDD     96(BP), Y4, Y4
+	VPADDD     128(BP), Y1, Y1
+	VPADDD     160(BP), Y2, Y2
+	VPERM2I128 $0x02, Y0, Y14, Y3
+	VPERM2I128 $0x02, Y12, Y4, Y7
+	VPERM2I128 $0x13, Y0, Y14, Y11
+	VPERM2I128 $0x13, Y12, Y4, Y15
+	VPXOR      (SI), Y3, Y3
+	VPXOR      32(SI), Y7, Y7
+	VPXOR      64(SI), Y11, Y11
+	VPXOR      96(SI), Y15, Y15
+	VMOVDQU    Y3, (DI)
+	VMOVDQU    Y7, 32(DI)
+	VMOVDQU    Y11, 64(DI)
+	VMOVDQU    Y15, 96(DI)
+	VPERM2I128 $0x02, Y5, Y9, Y3
+	VPERM2I128 $0x02, Y13, Y1, Y7
+	VPERM2I128 $0x13, Y5, Y9, Y11
+	VPERM2I128 $0x13, Y13, Y1, Y15
+	VPXOR      128(SI), Y3, Y3
+	VPXOR      160(SI), Y7, Y7
+	VPXOR      192(SI), Y11, Y11
+	VPXOR      224(SI), Y15, Y15
+	VMOVDQU    Y3, 128(DI)
+	VMOVDQU    Y7, 160(DI)
+	VMOVDQU    Y11, 192(DI)
+	VMOVDQU    Y15, 224(DI)
+	VPERM2I128 $0x02, Y6, Y10, Y0
+	VPERM2I128 $0x02, Y8, Y2, Y14
+	VPERM2I128 $0x13, Y6, Y10, Y12
+	VPERM2I128 $0x13, Y8, Y2, Y4
+	LEAQ       256(SI), SI
+	LEAQ       256(DI), DI
+	SUBQ       $0x00000100, BX
 	JMP        openAVX2TailLoop
 
-// ----------------------------------------------------------------------------
-// Special optimization for the last 512 bytes of ciphertext
 openAVX2Tail512:
-	VMOVDQU ·chacha20Constants<>(SB), AA0; VMOVDQA AA0, AA1; VMOVDQA AA0, AA2; VMOVDQA AA0, AA3
-	VMOVDQA state1StoreAVX2, BB0; VMOVDQA BB0, BB1; VMOVDQA BB0, BB2; VMOVDQA BB0, BB3
-	VMOVDQA state2StoreAVX2, CC0; VMOVDQA CC0, CC1; VMOVDQA CC0, CC2; VMOVDQA CC0, CC3
-	VMOVDQA ctr3StoreAVX2, DD0; VPADDD ·avx2IncMask<>(SB), DD0, DD0; VPADDD ·avx2IncMask<>(SB), DD0, DD1; VPADDD ·avx2IncMask<>(SB), DD1, DD2; VPADDD ·avx2IncMask<>(SB), DD2, DD3
-	VMOVDQA DD0, ctr0StoreAVX2; VMOVDQA DD1, ctr1StoreAVX2; VMOVDQA DD2, ctr2StoreAVX2; VMOVDQA DD3, ctr3StoreAVX2
-	XORQ    itr1, itr1
-	MOVQ    inp, itr2
+	VMOVDQU ·chacha20Constants<>+0(SB), Y0
+	VMOVDQA Y0, Y5
+	VMOVDQA Y0, Y6
+	VMOVDQA Y0, Y7
+	VMOVDQA 32(BP), Y14
+	VMOVDQA Y14, Y9
+	VMOVDQA Y14, Y10
+	VMOVDQA Y14, Y11
+	VMOVDQA 64(BP), Y12
+	VMOVDQA Y12, Y13
+	VMOVDQA Y12, Y8
+	VMOVDQA Y12, Y15
+	VMOVDQA 192(BP), Y4
+	VPADDD  ·avx2IncMask<>+0(SB), Y4, Y4
+	VPADDD  ·avx2IncMask<>+0(SB), Y4, Y1
+	VPADDD  ·avx2IncMask<>+0(SB), Y1, Y2
+	VPADDD  ·avx2IncMask<>+0(SB), Y2, Y3
+	VMOVDQA Y4, 96(BP)
+	VMOVDQA Y1, 128(BP)
+	VMOVDQA Y2, 160(BP)
+	VMOVDQA Y3, 192(BP)
+	XORQ    CX, CX
+	MOVQ    SI, R9
 
 openAVX2Tail512LoopB:
-	polyAdd(0(itr2))
-	polyMulAVX2
-	LEAQ (2*8)(itr2), itr2
+	ADDQ  (R9), R10
+	ADCQ  8(R9), R11
+	ADCQ  $0x01, R12
+	MOVQ  (BP), DX
+	MOVQ  DX, R15
+	MULXQ R10, R13, R14
+	IMULQ R12, R15
+	MULXQ R11, AX, DX
+	ADDQ  AX, R14
+	ADCQ  DX, R15
+	MOVQ  8(BP), DX
+	MULXQ R10, R10, AX
+	ADDQ  R10, R14
+	MULXQ R11, R11, R8
+	ADCQ  R11, R15
+	ADCQ  $0x00, R8
+	IMULQ R12, DX
+	ADDQ  AX, R15
+	ADCQ  DX, R8
+	MOVQ  R13, R10
+	MOVQ  R14, R11
+	MOVQ  R15, R12
+	ANDQ  $0x03, R12
+	MOVQ  R15, R13
+	ANDQ  $-4, R13
+	MOVQ  R8, R14
+	SHRQ  $0x02, R8, R15
+	SHRQ  $0x02, R8
+	ADDQ  R13, R10
+	ADCQ  R14, R11
+	ADCQ  $0x00, R12
+	ADDQ  R15, R10
+	ADCQ  R8, R11
+	ADCQ  $0x00, R12
+	LEAQ  16(R9), R9
 
 openAVX2Tail512LoopA:
-	VPADDD   BB0, AA0, AA0; VPADDD BB1, AA1, AA1; VPADDD BB2, AA2, AA2; VPADDD BB3, AA3, AA3
-	VPXOR    AA0, DD0, DD0; VPXOR AA1, DD1, DD1; VPXOR AA2, DD2, DD2; VPXOR AA3, DD3, DD3
-	VPSHUFB  ·rol16<>(SB), DD0, DD0; VPSHUFB ·rol16<>(SB), DD1, DD1; VPSHUFB ·rol16<>(SB), DD2, DD2; VPSHUFB ·rol16<>(SB), DD3, DD3
-	VPADDD   DD0, CC0, CC0; VPADDD DD1, CC1, CC1; VPADDD DD2, CC2, CC2; VPADDD DD3, CC3, CC3
-	VPXOR    CC0, BB0, BB0; VPXOR CC1, BB1, BB1; VPXOR CC2, BB2, BB2; VPXOR CC3, BB3, BB3
-	VMOVDQA  CC3, tmpStoreAVX2
-	VPSLLD   $12, BB0, CC3; VPSRLD $20, BB0, BB0; VPXOR CC3, BB0, BB0
-	VPSLLD   $12, BB1, CC3; VPSRLD $20, BB1, BB1; VPXOR CC3, BB1, BB1
-	VPSLLD   $12, BB2, CC3; VPSRLD $20, BB2, BB2; VPXOR CC3, BB2, BB2
-	VPSLLD   $12, BB3, CC3; VPSRLD $20, BB3, BB3; VPXOR CC3, BB3, BB3
-	VMOVDQA  tmpStoreAVX2, CC3
-	polyAdd(0*8(itr2))
-	polyMulAVX2
-	VPADDD   BB0, AA0, AA0; VPADDD BB1, AA1, AA1; VPADDD BB2, AA2, AA2; VPADDD BB3, AA3, AA3
-	VPXOR    AA0, DD0, DD0; VPXOR AA1, DD1, DD1; VPXOR AA2, DD2, DD2; VPXOR AA3, DD3, DD3
-	VPSHUFB  ·rol8<>(SB), DD0, DD0; VPSHUFB ·rol8<>(SB), DD1, DD1; VPSHUFB ·rol8<>(SB), DD2, DD2; VPSHUFB ·rol8<>(SB), DD3, DD3
-	VPADDD   DD0, CC0, CC0; VPADDD DD1, CC1, CC1; VPADDD DD2, CC2, CC2; VPADDD DD3, CC3, CC3
-	VPXOR    CC0, BB0, BB0; VPXOR CC1, BB1, BB1; VPXOR CC2, BB2, BB2; VPXOR CC3, BB3, BB3
-	VMOVDQA  CC3, tmpStoreAVX2
-	VPSLLD   $7, BB0, CC3; VPSRLD $25, BB0, BB0; VPXOR CC3, BB0, BB0
-	VPSLLD   $7, BB1, CC3; VPSRLD $25, BB1, BB1; VPXOR CC3, BB1, BB1
-	VPSLLD   $7, BB2, CC3; VPSRLD $25, BB2, BB2; VPXOR CC3, BB2, BB2
-	VPSLLD   $7, BB3, CC3; VPSRLD $25, BB3, BB3; VPXOR CC3, BB3, BB3
-	VMOVDQA  tmpStoreAVX2, CC3
-	VPALIGNR $4, BB0, BB0, BB0; VPALIGNR $4, BB1, BB1, BB1; VPALIGNR $4, BB2, BB2, BB2; VPALIGNR $4, BB3, BB3, BB3
-	VPALIGNR $8, CC0, CC0, CC0; VPALIGNR $8, CC1, CC1, CC1; VPALIGNR $8, CC2, CC2, CC2; VPALIGNR $8, CC3, CC3, CC3
-	VPALIGNR $12, DD0, DD0, DD0; VPALIGNR $12, DD1, DD1, DD1; VPALIGNR $12, DD2, DD2, DD2; VPALIGNR $12, DD3, DD3, DD3
-	VPADDD   BB0, AA0, AA0; VPADDD BB1, AA1, AA1; VPADDD BB2, AA2, AA2; VPADDD BB3, AA3, AA3
-	VPXOR    AA0, DD0, DD0; VPXOR AA1, DD1, DD1; VPXOR AA2, DD2, DD2; VPXOR AA3, DD3, DD3
-	VPSHUFB  ·rol16<>(SB), DD0, DD0; VPSHUFB ·rol16<>(SB), DD1, DD1; VPSHUFB ·rol16<>(SB), DD2, DD2; VPSHUFB ·rol16<>(SB), DD3, DD3
-	VPADDD   DD0, CC0, CC0; VPADDD DD1, CC1, CC1; VPADDD DD2, CC2, CC2; VPADDD DD3, CC3, CC3
-	VPXOR    CC0, BB0, BB0; VPXOR CC1, BB1, BB1; VPXOR CC2, BB2, BB2; VPXOR CC3, BB3, BB3
-	polyAdd(2*8(itr2))
-	polyMulAVX2
-	LEAQ     (4*8)(itr2), itr2
-	VMOVDQA  CC3, tmpStoreAVX2
-	VPSLLD   $12, BB0, CC3; VPSRLD $20, BB0, BB0; VPXOR CC3, BB0, BB0
-	VPSLLD   $12, BB1, CC3; VPSRLD $20, BB1, BB1; VPXOR CC3, BB1, BB1
-	VPSLLD   $12, BB2, CC3; VPSRLD $20, BB2, BB2; VPXOR CC3, BB2, BB2
-	VPSLLD   $12, BB3, CC3; VPSRLD $20, BB3, BB3; VPXOR CC3, BB3, BB3
-	VMOVDQA  tmpStoreAVX2, CC3
-	VPADDD   BB0, AA0, AA0; VPADDD BB1, AA1, AA1; VPADDD BB2, AA2, AA2; VPADDD BB3, AA3, AA3
-	VPXOR    AA0, DD0, DD0; VPXOR AA1, DD1, DD1; VPXOR AA2, DD2, DD2; VPXOR AA3, DD3, DD3
-	VPSHUFB  ·rol8<>(SB), DD0, DD0; VPSHUFB ·rol8<>(SB), DD1, DD1; VPSHUFB ·rol8<>(SB), DD2, DD2; VPSHUFB ·rol8<>(SB), DD3, DD3
-	VPADDD   DD0, CC0, CC0; VPADDD DD1, CC1, CC1; VPADDD DD2, CC2, CC2; VPADDD DD3, CC3, CC3
-	VPXOR    CC0, BB0, BB0; VPXOR CC1, BB1, BB1; VPXOR CC2, BB2, BB2; VPXOR CC3, BB3, BB3
-	VMOVDQA  CC3, tmpStoreAVX2
-	VPSLLD   $7, BB0, CC3; VPSRLD $25, BB0, BB0; VPXOR CC3, BB0, BB0
-	VPSLLD   $7, BB1, CC3; VPSRLD $25, BB1, BB1; VPXOR CC3, BB1, BB1
-	VPSLLD   $7, BB2, CC3; VPSRLD $25, BB2, BB2; VPXOR CC3, BB2, BB2
-	VPSLLD   $7, BB3, CC3; VPSRLD $25, BB3, BB3; VPXOR CC3, BB3, BB3
-	VMOVDQA  tmpStoreAVX2, CC3
-	VPALIGNR $12, BB0, BB0, BB0; VPALIGNR $12, BB1, BB1, BB1; VPALIGNR $12, BB2, BB2, BB2; VPALIGNR $12, BB3, BB3, BB3
-	VPALIGNR $8, CC0, CC0, CC0; VPALIGNR $8, CC1, CC1, CC1; VPALIGNR $8, CC2, CC2, CC2; VPALIGNR $8, CC3, CC3, CC3
-	VPALIGNR $4, DD0, DD0, DD0; VPALIGNR $4, DD1, DD1, DD1; VPALIGNR $4, DD2, DD2, DD2; VPALIGNR $4, DD3, DD3, DD3
-	INCQ     itr1
-	CMPQ     itr1, $4
+	VPADDD   Y14, Y0, Y0
+	VPADDD   Y9, Y5, Y5
+	VPADDD   Y10, Y6, Y6
+	VPADDD   Y11, Y7, Y7
+	VPXOR    Y0, Y4, Y4
+	VPXOR    Y5, Y1, Y1
+	VPXOR    Y6, Y2, Y2
+	VPXOR    Y7, Y3, Y3
+	VPSHUFB  ·rol16<>+0(SB), Y4, Y4
+	VPSHUFB  ·rol16<>+0(SB), Y1, Y1
+	VPSHUFB  ·rol16<>+0(SB), Y2, Y2
+	VPSHUFB  ·rol16<>+0(SB), Y3, Y3
+	VPADDD   Y4, Y12, Y12
+	VPADDD   Y1, Y13, Y13
+	VPADDD   Y2, Y8, Y8
+	VPADDD   Y3, Y15, Y15
+	VPXOR    Y12, Y14, Y14
+	VPXOR    Y13, Y9, Y9
+	VPXOR    Y8, Y10, Y10
+	VPXOR    Y15, Y11, Y11
+	VMOVDQA  Y15, 224(BP)
+	VPSLLD   $0x0c, Y14, Y15
+	VPSRLD   $0x14, Y14, Y14
+	VPXOR    Y15, Y14, Y14
+	VPSLLD   $0x0c, Y9, Y15
+	VPSRLD   $0x14, Y9, Y9
+	VPXOR    Y15, Y9, Y9
+	VPSLLD   $0x0c, Y10, Y15
+	VPSRLD   $0x14, Y10, Y10
+	VPXOR    Y15, Y10, Y10
+	VPSLLD   $0x0c, Y11, Y15
+	VPSRLD   $0x14, Y11, Y11
+	VPXOR    Y15, Y11, Y11
+	VMOVDQA  224(BP), Y15
+	ADDQ     (R9), R10
+	ADCQ     8(R9), R11
+	ADCQ     $0x01, R12
+	MOVQ     (BP), DX
+	MOVQ     DX, R15
+	MULXQ    R10, R13, R14
+	IMULQ    R12, R15
+	MULXQ    R11, AX, DX
+	ADDQ     AX, R14
+	ADCQ     DX, R15
+	MOVQ     8(BP), DX
+	MULXQ    R10, R10, AX
+	ADDQ     R10, R14
+	MULXQ    R11, R11, R8
+	ADCQ     R11, R15
+	ADCQ     $0x00, R8
+	IMULQ    R12, DX
+	ADDQ     AX, R15
+	ADCQ     DX, R8
+	MOVQ     R13, R10
+	MOVQ     R14, R11
+	MOVQ     R15, R12
+	ANDQ     $0x03, R12
+	MOVQ     R15, R13
+	ANDQ     $-4, R13
+	MOVQ     R8, R14
+	SHRQ     $0x02, R8, R15
+	SHRQ     $0x02, R8
+	ADDQ     R13, R10
+	ADCQ     R14, R11
+	ADCQ     $0x00, R12
+	ADDQ     R15, R10
+	ADCQ     R8, R11
+	ADCQ     $0x00, R12
+	VPADDD   Y14, Y0, Y0
+	VPADDD   Y9, Y5, Y5
+	VPADDD   Y10, Y6, Y6
+	VPADDD   Y11, Y7, Y7
+	VPXOR    Y0, Y4, Y4
+	VPXOR    Y5, Y1, Y1
+	VPXOR    Y6, Y2, Y2
+	VPXOR    Y7, Y3, Y3
+	VPSHUFB  ·rol8<>+0(SB), Y4, Y4
+	VPSHUFB  ·rol8<>+0(SB), Y1, Y1
+	VPSHUFB  ·rol8<>+0(SB), Y2, Y2
+	VPSHUFB  ·rol8<>+0(SB), Y3, Y3
+	VPADDD   Y4, Y12, Y12
+	VPADDD   Y1, Y13, Y13
+	VPADDD   Y2, Y8, Y8
+	VPADDD   Y3, Y15, Y15
+	VPXOR    Y12, Y14, Y14
+	VPXOR    Y13, Y9, Y9
+	VPXOR    Y8, Y10, Y10
+	VPXOR    Y15, Y11, Y11
+	VMOVDQA  Y15, 224(BP)
+	VPSLLD   $0x07, Y14, Y15
+	VPSRLD   $0x19, Y14, Y14
+	VPXOR    Y15, Y14, Y14
+	VPSLLD   $0x07, Y9, Y15
+	VPSRLD   $0x19, Y9, Y9
+	VPXOR    Y15, Y9, Y9
+	VPSLLD   $0x07, Y10, Y15
+	VPSRLD   $0x19, Y10, Y10
+	VPXOR    Y15, Y10, Y10
+	VPSLLD   $0x07, Y11, Y15
+	VPSRLD   $0x19, Y11, Y11
+	VPXOR    Y15, Y11, Y11
+	VMOVDQA  224(BP), Y15
+	VPALIGNR $0x04, Y14, Y14, Y14
+	VPALIGNR $0x04, Y9, Y9, Y9
+	VPALIGNR $0x04, Y10, Y10, Y10
+	VPALIGNR $0x04, Y11, Y11, Y11
+	VPALIGNR $0x08, Y12, Y12, Y12
+	VPALIGNR $0x08, Y13, Y13, Y13
+	VPALIGNR $0x08, Y8, Y8, Y8
+	VPALIGNR $0x08, Y15, Y15, Y15
+	VPALIGNR $0x0c, Y4, Y4, Y4
+	VPALIGNR $0x0c, Y1, Y1, Y1
+	VPALIGNR $0x0c, Y2, Y2, Y2
+	VPALIGNR $0x0c, Y3, Y3, Y3
+	VPADDD   Y14, Y0, Y0
+	VPADDD   Y9, Y5, Y5
+	VPADDD   Y10, Y6, Y6
+	VPADDD   Y11, Y7, Y7
+	VPXOR    Y0, Y4, Y4
+	VPXOR    Y5, Y1, Y1
+	VPXOR    Y6, Y2, Y2
+	VPXOR    Y7, Y3, Y3
+	VPSHUFB  ·rol16<>+0(SB), Y4, Y4
+	VPSHUFB  ·rol16<>+0(SB), Y1, Y1
+	VPSHUFB  ·rol16<>+0(SB), Y2, Y2
+	VPSHUFB  ·rol16<>+0(SB), Y3, Y3
+	VPADDD   Y4, Y12, Y12
+	VPADDD   Y1, Y13, Y13
+	VPADDD   Y2, Y8, Y8
+	VPADDD   Y3, Y15, Y15
+	VPXOR    Y12, Y14, Y14
+	VPXOR    Y13, Y9, Y9
+	VPXOR    Y8, Y10, Y10
+	VPXOR    Y15, Y11, Y11
+	ADDQ     16(R9), R10
+	ADCQ     24(R9), R11
+	ADCQ     $0x01, R12
+	MOVQ     (BP), DX
+	MOVQ     DX, R15
+	MULXQ    R10, R13, R14
+	IMULQ    R12, R15
+	MULXQ    R11, AX, DX
+	ADDQ     AX, R14
+	ADCQ     DX, R15
+	MOVQ     8(BP), DX
+	MULXQ    R10, R10, AX
+	ADDQ     R10, R14
+	MULXQ    R11, R11, R8
+	ADCQ     R11, R15
+	ADCQ     $0x00, R8
+	IMULQ    R12, DX
+	ADDQ     AX, R15
+	ADCQ     DX, R8
+	MOVQ     R13, R10
+	MOVQ     R14, R11
+	MOVQ     R15, R12
+	ANDQ     $0x03, R12
+	MOVQ     R15, R13
+	ANDQ     $-4, R13
+	MOVQ     R8, R14
+	SHRQ     $0x02, R8, R15
+	SHRQ     $0x02, R8
+	ADDQ     R13, R10
+	ADCQ     R14, R11
+	ADCQ     $0x00, R12
+	ADDQ     R15, R10
+	ADCQ     R8, R11
+	ADCQ     $0x00, R12
+	LEAQ     32(R9), R9
+	VMOVDQA  Y15, 224(BP)
+	VPSLLD   $0x0c, Y14, Y15
+	VPSRLD   $0x14, Y14, Y14
+	VPXOR    Y15, Y14, Y14
+	VPSLLD   $0x0c, Y9, Y15
+	VPSRLD   $0x14, Y9, Y9
+	VPXOR    Y15, Y9, Y9
+	VPSLLD   $0x0c, Y10, Y15
+	VPSRLD   $0x14, Y10, Y10
+	VPXOR    Y15, Y10, Y10
+	VPSLLD   $0x0c, Y11, Y15
+	VPSRLD   $0x14, Y11, Y11
+	VPXOR    Y15, Y11, Y11
+	VMOVDQA  224(BP), Y15
+	VPADDD   Y14, Y0, Y0
+	VPADDD   Y9, Y5, Y5
+	VPADDD   Y10, Y6, Y6
+	VPADDD   Y11, Y7, Y7
+	VPXOR    Y0, Y4, Y4
+	VPXOR    Y5, Y1, Y1
+	VPXOR    Y6, Y2, Y2
+	VPXOR    Y7, Y3, Y3
+	VPSHUFB  ·rol8<>+0(SB), Y4, Y4
+	VPSHUFB  ·rol8<>+0(SB), Y1, Y1
+	VPSHUFB  ·rol8<>+0(SB), Y2, Y2
+	VPSHUFB  ·rol8<>+0(SB), Y3, Y3
+	VPADDD   Y4, Y12, Y12
+	VPADDD   Y1, Y13, Y13
+	VPADDD   Y2, Y8, Y8
+	VPADDD   Y3, Y15, Y15
+	VPXOR    Y12, Y14, Y14
+	VPXOR    Y13, Y9, Y9
+	VPXOR    Y8, Y10, Y10
+	VPXOR    Y15, Y11, Y11
+	VMOVDQA  Y15, 224(BP)
+	VPSLLD   $0x07, Y14, Y15
+	VPSRLD   $0x19, Y14, Y14
+	VPXOR    Y15, Y14, Y14
+	VPSLLD   $0x07, Y9, Y15
+	VPSRLD   $0x19, Y9, Y9
+	VPXOR    Y15, Y9, Y9
+	VPSLLD   $0x07, Y10, Y15
+	VPSRLD   $0x19, Y10, Y10
+	VPXOR    Y15, Y10, Y10
+	VPSLLD   $0x07, Y11, Y15
+	VPSRLD   $0x19, Y11, Y11
+	VPXOR    Y15, Y11, Y11
+	VMOVDQA  224(BP), Y15
+	VPALIGNR $0x0c, Y14, Y14, Y14
+	VPALIGNR $0x0c, Y9, Y9, Y9
+	VPALIGNR $0x0c, Y10, Y10, Y10
+	VPALIGNR $0x0c, Y11, Y11, Y11
+	VPALIGNR $0x08, Y12, Y12, Y12
+	VPALIGNR $0x08, Y13, Y13, Y13
+	VPALIGNR $0x08, Y8, Y8, Y8
+	VPALIGNR $0x08, Y15, Y15, Y15
+	VPALIGNR $0x04, Y4, Y4, Y4
+	VPALIGNR $0x04, Y1, Y1, Y1
+	VPALIGNR $0x04, Y2, Y2, Y2
+	VPALIGNR $0x04, Y3, Y3, Y3
+	INCQ     CX
+	CMPQ     CX, $0x04
 	JLT      openAVX2Tail512LoopB
-
-	CMPQ itr1, $10
-	JNE  openAVX2Tail512LoopA
-
-	MOVQ inl, itr1
-	SUBQ $384, itr1
-	ANDQ $-16, itr1
+	CMPQ     CX, $0x0a
+	JNE      openAVX2Tail512LoopA
+	MOVQ     BX, CX
+	SUBQ     $0x00000180, CX
+	ANDQ     $-16, CX
 
 openAVX2Tail512HashLoop:
-	TESTQ itr1, itr1
+	TESTQ CX, CX
 	JE    openAVX2Tail512HashEnd
-	polyAdd(0(itr2))
-	polyMulAVX2
-	LEAQ  16(itr2), itr2
-	SUBQ  $16, itr1
+	ADDQ  (R9), R10
+	ADCQ  8(R9), R11
+	ADCQ  $0x01, R12
+	MOVQ  (BP), DX
+	MOVQ  DX, R15
+	MULXQ R10, R13, R14
+	IMULQ R12, R15
+	MULXQ R11, AX, DX
+	ADDQ  AX, R14
+	ADCQ  DX, R15
+	MOVQ  8(BP), DX
+	MULXQ R10, R10, AX
+	ADDQ  R10, R14
+	MULXQ R11, R11, R8
+	ADCQ  R11, R15
+	ADCQ  $0x00, R8
+	IMULQ R12, DX
+	ADDQ  AX, R15
+	ADCQ  DX, R8
+	MOVQ  R13, R10
+	MOVQ  R14, R11
+	MOVQ  R15, R12
+	ANDQ  $0x03, R12
+	MOVQ  R15, R13
+	ANDQ  $-4, R13
+	MOVQ  R8, R14
+	SHRQ  $0x02, R8, R15
+	SHRQ  $0x02, R8
+	ADDQ  R13, R10
+	ADCQ  R14, R11
+	ADCQ  $0x00, R12
+	ADDQ  R15, R10
+	ADCQ  R8, R11
+	ADCQ  $0x00, R12
+	LEAQ  16(R9), R9
+	SUBQ  $0x10, CX
 	JMP   openAVX2Tail512HashLoop
 
 openAVX2Tail512HashEnd:
-	VPADDD     ·chacha20Constants<>(SB), AA0, AA0; VPADDD ·chacha20Constants<>(SB), AA1, AA1; VPADDD ·chacha20Constants<>(SB), AA2, AA2; VPADDD ·chacha20Constants<>(SB), AA3, AA3
-	VPADDD     state1StoreAVX2, BB0, BB0; VPADDD state1StoreAVX2, BB1, BB1; VPADDD state1StoreAVX2, BB2, BB2; VPADDD state1StoreAVX2, BB3, BB3
-	VPADDD     state2StoreAVX2, CC0, CC0; VPADDD state2StoreAVX2, CC1, CC1; VPADDD state2StoreAVX2, CC2, CC2; VPADDD state2StoreAVX2, CC3, CC3
-	VPADDD     ctr0StoreAVX2, DD0, DD0; VPADDD ctr1StoreAVX2, DD1, DD1; VPADDD ctr2StoreAVX2, DD2, DD2; VPADDD ctr3StoreAVX2, DD3, DD3
-	VMOVDQA    CC3, tmpStoreAVX2
-	VPERM2I128 $0x02, AA0, BB0, CC3; VPERM2I128 $0x13, AA0, BB0, BB0; VPERM2I128 $0x02, CC0, DD0, AA0; VPERM2I128 $0x13, CC0, DD0, CC0
-	VPXOR      (0*32)(inp), CC3, CC3; VPXOR (1*32)(inp), AA0, AA0; VPXOR (2*32)(inp), BB0, BB0; VPXOR (3*32)(inp), CC0, CC0
-	VMOVDQU    CC3, (0*32)(oup); VMOVDQU AA0, (1*32)(oup); VMOVDQU BB0, (2*32)(oup); VMOVDQU CC0, (3*32)(oup)
-	VPERM2I128 $0x02, AA1, BB1, AA0; VPERM2I128 $0x02, CC1, DD1, BB0; VPERM2I128 $0x13, AA1, BB1, CC0; VPERM2I128 $0x13, CC1, DD1, DD0
-	VPXOR      (4*32)(inp), AA0, AA0; VPXOR (5*32)(inp), BB0, BB0; VPXOR (6*32)(inp), CC0, CC0; VPXOR (7*32)(inp), DD0, DD0
-	VMOVDQU    AA0, (4*32)(oup); VMOVDQU BB0, (5*32)(oup); VMOVDQU CC0, (6*32)(oup); VMOVDQU DD0, (7*32)(oup)
-	VPERM2I128 $0x02, AA2, BB2, AA0; VPERM2I128 $0x02, CC2, DD2, BB0; VPERM2I128 $0x13, AA2, BB2, CC0; VPERM2I128 $0x13, CC2, DD2, DD0
-	VPXOR      (8*32)(inp), AA0, AA0; VPXOR (9*32)(inp), BB0, BB0; VPXOR (10*32)(inp), CC0, CC0; VPXOR (11*32)(inp), DD0, DD0
-	VMOVDQU    AA0, (8*32)(oup); VMOVDQU BB0, (9*32)(oup); VMOVDQU CC0, (10*32)(oup); VMOVDQU DD0, (11*32)(oup)
-	VPERM2I128 $0x02, AA3, BB3, AA0; VPERM2I128 $0x02, tmpStoreAVX2, DD3, BB0; VPERM2I128 $0x13, AA3, BB3, CC0; VPERM2I128 $0x13, tmpStoreAVX2, DD3, DD0
-
-	LEAQ (12*32)(inp), inp
-	LEAQ (12*32)(oup), oup
-	SUBQ $12*32, inl
-
-	JMP openAVX2TailLoop
-
-// ----------------------------------------------------------------------------
-// ----------------------------------------------------------------------------
-// func chacha20Poly1305Seal(dst, key, src, ad []byte)
-TEXT ·chacha20Poly1305Seal(SB), 0, $288-96
-	// For aligned stack access
+	VPADDD     ·chacha20Constants<>+0(SB), Y0, Y0
+	VPADDD     ·chacha20Constants<>+0(SB), Y5, Y5
+	VPADDD     ·chacha20Constants<>+0(SB), Y6, Y6
+	VPADDD     ·chacha20Constants<>+0(SB), Y7, Y7
+	VPADDD     32(BP), Y14, Y14
+	VPADDD     32(BP), Y9, Y9
+	VPADDD     32(BP), Y10, Y10
+	VPADDD     32(BP), Y11, Y11
+	VPADDD     64(BP), Y12, Y12
+	VPADDD     64(BP), Y13, Y13
+	VPADDD     64(BP), Y8, Y8
+	VPADDD     64(BP), Y15, Y15
+	VPADDD     96(BP), Y4, Y4
+	VPADDD     128(BP), Y1, Y1
+	VPADDD     160(BP), Y2, Y2
+	VPADDD     192(BP), Y3, Y3
+	VMOVDQA    Y15, 224(BP)
+	VPERM2I128 $0x02, Y0, Y14, Y15
+	VPERM2I128 $0x13, Y0, Y14, Y14
+	VPERM2I128 $0x02, Y12, Y4, Y0
+	VPERM2I128 $0x13, Y12, Y4, Y12
+	VPXOR      (SI), Y15, Y15
+	VPXOR      32(SI), Y0, Y0
+	VPXOR      64(SI), Y14, Y14
+	VPXOR      96(SI), Y12, Y12
+	VMOVDQU    Y15, (DI)
+	VMOVDQU    Y0, 32(DI)
+	VMOVDQU    Y14, 64(DI)
+	VMOVDQU    Y12, 96(DI)
+	VPERM2I128 $0x02, Y5, Y9, Y0
+	VPERM2I128 $0x02, Y13, Y1, Y14
+	VPERM2I128 $0x13, Y5, Y9, Y12
+	VPERM2I128 $0x13, Y13, Y1, Y4
+	VPXOR      128(SI), Y0, Y0
+	VPXOR      160(SI), Y14, Y14
+	VPXOR      192(SI), Y12, Y12
+	VPXOR      224(SI), Y4, Y4
+	VMOVDQU    Y0, 128(DI)
+	VMOVDQU    Y14, 160(DI)
+	VMOVDQU    Y12, 192(DI)
+	VMOVDQU    Y4, 224(DI)
+	VPERM2I128 $0x02, Y6, Y10, Y0
+	VPERM2I128 $0x02, Y8, Y2, Y14
+	VPERM2I128 $0x13, Y6, Y10, Y12
+	VPERM2I128 $0x13, Y8, Y2, Y4
+	VPXOR      256(SI), Y0, Y0
+	VPXOR      288(SI), Y14, Y14
+	VPXOR      320(SI), Y12, Y12
+	VPXOR      352(SI), Y4, Y4
+	VMOVDQU    Y0, 256(DI)
+	VMOVDQU    Y14, 288(DI)
+	VMOVDQU    Y12, 320(DI)
+	VMOVDQU    Y4, 352(DI)
+	VPERM2I128 $0x02, Y7, Y11, Y0
+	VPERM2I128 $0x02, 224(BP), Y3, Y14
+	VPERM2I128 $0x13, Y7, Y11, Y12
+	VPERM2I128 $0x13, 224(BP), Y3, Y4
+	LEAQ       384(SI), SI
+	LEAQ       384(DI), DI
+	SUBQ       $0x00000180, BX
+	JMP        openAVX2TailLoop
+
+DATA ·chacha20Constants<>+0(SB)/4, $0x61707865
+DATA ·chacha20Constants<>+4(SB)/4, $0x3320646e
+DATA ·chacha20Constants<>+8(SB)/4, $0x79622d32
+DATA ·chacha20Constants<>+12(SB)/4, $0x6b206574
+DATA ·chacha20Constants<>+16(SB)/4, $0x61707865
+DATA ·chacha20Constants<>+20(SB)/4, $0x3320646e
+DATA ·chacha20Constants<>+24(SB)/4, $0x79622d32
+DATA ·chacha20Constants<>+28(SB)/4, $0x6b206574
+GLOBL ·chacha20Constants<>(SB), RODATA|NOPTR, $32
+
+DATA ·polyClampMask<>+0(SB)/8, $0x0ffffffc0fffffff
+DATA ·polyClampMask<>+8(SB)/8, $0x0ffffffc0ffffffc
+DATA ·polyClampMask<>+16(SB)/8, $0xffffffffffffffff
+DATA ·polyClampMask<>+24(SB)/8, $0xffffffffffffffff
+GLOBL ·polyClampMask<>(SB), RODATA|NOPTR, $32
+
+DATA ·sseIncMask<>+0(SB)/8, $0x0000000000000001
+DATA ·sseIncMask<>+8(SB)/8, $0x0000000000000000
+GLOBL ·sseIncMask<>(SB), RODATA|NOPTR, $16
+
+DATA ·andMask<>+0(SB)/8, $0x00000000000000ff
+DATA ·andMask<>+8(SB)/8, $0x0000000000000000
+DATA ·andMask<>+16(SB)/8, $0x000000000000ffff
+DATA ·andMask<>+24(SB)/8, $0x0000000000000000
+DATA ·andMask<>+32(SB)/8, $0x0000000000ffffff
+DATA ·andMask<>+40(SB)/8, $0x0000000000000000
+DATA ·andMask<>+48(SB)/8, $0x00000000ffffffff
+DATA ·andMask<>+56(SB)/8, $0x0000000000000000
+DATA ·andMask<>+64(SB)/8, $0x000000ffffffffff
+DATA ·andMask<>+72(SB)/8, $0x0000000000000000
+DATA ·andMask<>+80(SB)/8, $0x0000ffffffffffff
+DATA ·andMask<>+88(SB)/8, $0x0000000000000000
+DATA ·andMask<>+96(SB)/8, $0x00ffffffffffffff
+DATA ·andMask<>+104(SB)/8, $0x0000000000000000
+DATA ·andMask<>+112(SB)/8, $0xffffffffffffffff
+DATA ·andMask<>+120(SB)/8, $0x0000000000000000
+DATA ·andMask<>+128(SB)/8, $0xffffffffffffffff
+DATA ·andMask<>+136(SB)/8, $0x00000000000000ff
+DATA ·andMask<>+144(SB)/8, $0xffffffffffffffff
+DATA ·andMask<>+152(SB)/8, $0x000000000000ffff
+DATA ·andMask<>+160(SB)/8, $0xffffffffffffffff
+DATA ·andMask<>+168(SB)/8, $0x0000000000ffffff
+DATA ·andMask<>+176(SB)/8, $0xffffffffffffffff
+DATA ·andMask<>+184(SB)/8, $0x00000000ffffffff
+DATA ·andMask<>+192(SB)/8, $0xffffffffffffffff
+DATA ·andMask<>+200(SB)/8, $0x000000ffffffffff
+DATA ·andMask<>+208(SB)/8, $0xffffffffffffffff
+DATA ·andMask<>+216(SB)/8, $0x0000ffffffffffff
+DATA ·andMask<>+224(SB)/8, $0xffffffffffffffff
+DATA ·andMask<>+232(SB)/8, $0x00ffffffffffffff
+GLOBL ·andMask<>(SB), RODATA|NOPTR, $240
+
+DATA ·avx2InitMask<>+0(SB)/8, $0x0000000000000000
+DATA ·avx2InitMask<>+8(SB)/8, $0x0000000000000000
+DATA ·avx2InitMask<>+16(SB)/8, $0x0000000000000001
+DATA ·avx2InitMask<>+24(SB)/8, $0x0000000000000000
+GLOBL ·avx2InitMask<>(SB), RODATA|NOPTR, $32
+
+DATA ·rol16<>+0(SB)/8, $0x0504070601000302
+DATA ·rol16<>+8(SB)/8, $0x0d0c0f0e09080b0a
+DATA ·rol16<>+16(SB)/8, $0x0504070601000302
+DATA ·rol16<>+24(SB)/8, $0x0d0c0f0e09080b0a
+GLOBL ·rol16<>(SB), RODATA|NOPTR, $32
+
+DATA ·rol8<>+0(SB)/8, $0x0605040702010003
+DATA ·rol8<>+8(SB)/8, $0x0e0d0c0f0a09080b
+DATA ·rol8<>+16(SB)/8, $0x0605040702010003
+DATA ·rol8<>+24(SB)/8, $0x0e0d0c0f0a09080b
+GLOBL ·rol8<>(SB), RODATA|NOPTR, $32
+
+DATA ·avx2IncMask<>+0(SB)/8, $0x0000000000000002
+DATA ·avx2IncMask<>+8(SB)/8, $0x0000000000000000
+DATA ·avx2IncMask<>+16(SB)/8, $0x0000000000000002
+DATA ·avx2IncMask<>+24(SB)/8, $0x0000000000000000
+GLOBL ·avx2IncMask<>(SB), RODATA|NOPTR, $32
+
+// func chacha20Poly1305Seal(dst []byte, key []uint32, src []byte, ad []byte)
+// Requires: AVX, AVX2, BMI2, CMOV, SSE2
+TEXT ·chacha20Poly1305Seal(SB), $288-96
 	MOVQ SP, BP
-	ADDQ $32, BP
+	ADDQ $0x20, BP
 	ANDQ $-32, BP
-	MOVQ dst+0(FP), oup
-	MOVQ key+24(FP), keyp
-	MOVQ src+48(FP), inp
-	MOVQ src_len+56(FP), inl
-	MOVQ ad+72(FP), adp
-
-	CMPB ·useAVX2(SB), $1
+	MOVQ dst_base+0(FP), DI
+	MOVQ key_base+24(FP), R8
+	MOVQ src_base+48(FP), SI
+	MOVQ src_len+56(FP), BX
+	MOVQ ad_base+72(FP), CX
+	CMPB ·useAVX2+0(SB), $0x01
 	JE   chacha20Poly1305Seal_AVX2
 
 	// Special optimization, for very short buffers
-	CMPQ inl, $128
-	JBE  sealSSE128 // About 15% faster
+	CMPQ BX, $0x80
+	JBE  sealSSE128
 
 	// In the seal case - prepare the poly key + 3 blocks of stream in the first iteration
-	MOVOU ·chacha20Constants<>(SB), A0
-	MOVOU (1*16)(keyp), B0
-	MOVOU (2*16)(keyp), C0
-	MOVOU (3*16)(keyp), D0
+	MOVOU ·chacha20Constants<>+0(SB), X0
+	MOVOU 16(R8), X3
+	MOVOU 32(R8), X6
+	MOVOU 48(R8), X9
 
 	// Store state on stack for future use
-	MOVO B0, state1Store
-	MOVO C0, state2Store
+	MOVO X3, 32(BP)
+	MOVO X6, 48(BP)
 
 	// Load state, increment counter blocks
-	MOVO A0, A1; MOVO B0, B1; MOVO C0, C1; MOVO D0, D1; PADDL ·sseIncMask<>(SB), D1
-	MOVO A1, A2; MOVO B1, B2; MOVO C1, C2; MOVO D1, D2; PADDL ·sseIncMask<>(SB), D2
-	MOVO A2, A3; MOVO B2, B3; MOVO C2, C3; MOVO D2, D3; PADDL ·sseIncMask<>(SB), D3
+	MOVO  X0, X1
+	MOVO  X3, X4
+	MOVO  X6, X7
+	MOVO  X9, X10
+	PADDL ·sseIncMask<>+0(SB), X10
+	MOVO  X1, X2
+	MOVO  X4, X5
+	MOVO  X7, X8
+	MOVO  X10, X11
+	PADDL ·sseIncMask<>+0(SB), X11
+	MOVO  X2, X12
+	MOVO  X5, X13
+	MOVO  X8, X14
+	MOVO  X11, X15
+	PADDL ·sseIncMask<>+0(SB), X15
 
 	// Store counters
-	MOVO D0, ctr0Store; MOVO D1, ctr1Store; MOVO D2, ctr2Store; MOVO D3, ctr3Store
-	MOVQ $10, itr2
+	MOVO X9, 80(BP)
+	MOVO X10, 96(BP)
+	MOVO X11, 112(BP)
+	MOVO X15, 128(BP)
+	MOVQ $0x0000000a, R9
 
 sealSSEIntroLoop:
-	MOVO         C3, tmpStore
-	chachaQR(A0, B0, C0, D0, C3); chachaQR(A1, B1, C1, D1, C3); chachaQR(A2, B2, C2, D2, C3)
-	MOVO         tmpStore, C3
-	MOVO         C1, tmpStore
-	chachaQR(A3, B3, C3, D3, C1)
-	MOVO         tmpStore, C1
-	shiftB0Left; shiftB1Left; shiftB2Left; shiftB3Left
-	shiftC0Left; shiftC1Left; shiftC2Left; shiftC3Left
-	shiftD0Left; shiftD1Left; shiftD2Left; shiftD3Left
-
-	MOVO          C3, tmpStore
-	chachaQR(A0, B0, C0, D0, C3); chachaQR(A1, B1, C1, D1, C3); chachaQR(A2, B2, C2, D2, C3)
-	MOVO          tmpStore, C3
-	MOVO          C1, tmpStore
-	chachaQR(A3, B3, C3, D3, C1)
-	MOVO          tmpStore, C1
-	shiftB0Right; shiftB1Right; shiftB2Right; shiftB3Right
-	shiftC0Right; shiftC1Right; shiftC2Right; shiftC3Right
-	shiftD0Right; shiftD1Right; shiftD2Right; shiftD3Right
-	DECQ          itr2
-	JNE           sealSSEIntroLoop
+	MOVO  X14, 64(BP)
+	PADDD X3, X0
+	PXOR  X0, X9
+	ROL16(X9, X14)
+	PADDD X9, X6
+	PXOR  X6, X3
+	MOVO  X3, X14
+	PSLLL $0x0c, X14
+	PSRLL $0x14, X3
+	PXOR  X14, X3
+	PADDD X3, X0
+	PXOR  X0, X9
+	ROL8(X9, X14)
+	PADDD X9, X6
+	PXOR  X6, X3
+	MOVO  X3, X14
+	PSLLL $0x07, X14
+	PSRLL $0x19, X3
+	PXOR  X14, X3
+	PADDD X4, X1
+	PXOR  X1, X10
+	ROL16(X10, X14)
+	PADDD X10, X7
+	PXOR  X7, X4
+	MOVO  X4, X14
+	PSLLL $0x0c, X14
+	PSRLL $0x14, X4
+	PXOR  X14, X4
+	PADDD X4, X1
+	PXOR  X1, X10
+	ROL8(X10, X14)
+	PADDD X10, X7
+	PXOR  X7, X4
+	MOVO  X4, X14
+	PSLLL $0x07, X14
+	PSRLL $0x19, X4
+	PXOR  X14, X4
+	PADDD X5, X2
+	PXOR  X2, X11
+	ROL16(X11, X14)
+	PADDD X11, X8
+	PXOR  X8, X5
+	MOVO  X5, X14
+	PSLLL $0x0c, X14
+	PSRLL $0x14, X5
+	PXOR  X14, X5
+	PADDD X5, X2
+	PXOR  X2, X11
+	ROL8(X11, X14)
+	PADDD X11, X8
+	PXOR  X8, X5
+	MOVO  X5, X14
+	PSLLL $0x07, X14
+	PSRLL $0x19, X5
+	PXOR  X14, X5
+	MOVO  64(BP), X14
+	MOVO  X7, 64(BP)
+	PADDD X13, X12
+	PXOR  X12, X15
+	ROL16(X15, X7)
+	PADDD X15, X14
+	PXOR  X14, X13
+	MOVO  X13, X7
+	PSLLL $0x0c, X7
+	PSRLL $0x14, X13
+	PXOR  X7, X13
+	PADDD X13, X12
+	PXOR  X12, X15
+	ROL8(X15, X7)
+	PADDD X15, X14
+	PXOR  X14, X13
+	MOVO  X13, X7
+	PSLLL $0x07, X7
+	PSRLL $0x19, X13
+	PXOR  X7, X13
+	MOVO  64(BP), X7
+	BYTE  $0x66
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xdb
+	BYTE  $0x04
+	BYTE  $0x66
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xe4
+	BYTE  $0x04
+	BYTE  $0x66
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xed
+	BYTE  $0x04
+	BYTE  $0x66
+	BYTE  $0x45
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xed
+	BYTE  $0x04
+	BYTE  $0x66
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xf6
+	BYTE  $0x08
+	BYTE  $0x66
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xff
+	BYTE  $0x08
+	BYTE  $0x66
+	BYTE  $0x45
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xc0
+	BYTE  $0x08
+	BYTE  $0x66
+	BYTE  $0x45
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xf6
+	BYTE  $0x08
+	BYTE  $0x66
+	BYTE  $0x45
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xc9
+	BYTE  $0x0c
+	BYTE  $0x66
+	BYTE  $0x45
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xd2
+	BYTE  $0x0c
+	BYTE  $0x66
+	BYTE  $0x45
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xdb
+	BYTE  $0x0c
+	BYTE  $0x66
+	BYTE  $0x45
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xff
+	BYTE  $0x0c
+	MOVO  X14, 64(BP)
+	PADDD X3, X0
+	PXOR  X0, X9
+	ROL16(X9, X14)
+	PADDD X9, X6
+	PXOR  X6, X3
+	MOVO  X3, X14
+	PSLLL $0x0c, X14
+	PSRLL $0x14, X3
+	PXOR  X14, X3
+	PADDD X3, X0
+	PXOR  X0, X9
+	ROL8(X9, X14)
+	PADDD X9, X6
+	PXOR  X6, X3
+	MOVO  X3, X14
+	PSLLL $0x07, X14
+	PSRLL $0x19, X3
+	PXOR  X14, X3
+	PADDD X4, X1
+	PXOR  X1, X10
+	ROL16(X10, X14)
+	PADDD X10, X7
+	PXOR  X7, X4
+	MOVO  X4, X14
+	PSLLL $0x0c, X14
+	PSRLL $0x14, X4
+	PXOR  X14, X4
+	PADDD X4, X1
+	PXOR  X1, X10
+	ROL8(X10, X14)
+	PADDD X10, X7
+	PXOR  X7, X4
+	MOVO  X4, X14
+	PSLLL $0x07, X14
+	PSRLL $0x19, X4
+	PXOR  X14, X4
+	PADDD X5, X2
+	PXOR  X2, X11
+	ROL16(X11, X14)
+	PADDD X11, X8
+	PXOR  X8, X5
+	MOVO  X5, X14
+	PSLLL $0x0c, X14
+	PSRLL $0x14, X5
+	PXOR  X14, X5
+	PADDD X5, X2
+	PXOR  X2, X11
+	ROL8(X11, X14)
+	PADDD X11, X8
+	PXOR  X8, X5
+	MOVO  X5, X14
+	PSLLL $0x07, X14
+	PSRLL $0x19, X5
+	PXOR  X14, X5
+	MOVO  64(BP), X14
+	MOVO  X7, 64(BP)
+	PADDD X13, X12
+	PXOR  X12, X15
+	ROL16(X15, X7)
+	PADDD X15, X14
+	PXOR  X14, X13
+	MOVO  X13, X7
+	PSLLL $0x0c, X7
+	PSRLL $0x14, X13
+	PXOR  X7, X13
+	PADDD X13, X12
+	PXOR  X12, X15
+	ROL8(X15, X7)
+	PADDD X15, X14
+	PXOR  X14, X13
+	MOVO  X13, X7
+	PSLLL $0x07, X7
+	PSRLL $0x19, X13
+	PXOR  X7, X13
+	MOVO  64(BP), X7
+	BYTE  $0x66
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xdb
+	BYTE  $0x0c
+	BYTE  $0x66
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xe4
+	BYTE  $0x0c
+	BYTE  $0x66
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xed
+	BYTE  $0x0c
+	BYTE  $0x66
+	BYTE  $0x45
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xed
+	BYTE  $0x0c
+	BYTE  $0x66
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xf6
+	BYTE  $0x08
+	BYTE  $0x66
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xff
+	BYTE  $0x08
+	BYTE  $0x66
+	BYTE  $0x45
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xc0
+	BYTE  $0x08
+	BYTE  $0x66
+	BYTE  $0x45
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xf6
+	BYTE  $0x08
+	BYTE  $0x66
+	BYTE  $0x45
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xc9
+	BYTE  $0x04
+	BYTE  $0x66
+	BYTE  $0x45
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xd2
+	BYTE  $0x04
+	BYTE  $0x66
+	BYTE  $0x45
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xdb
+	BYTE  $0x04
+	BYTE  $0x66
+	BYTE  $0x45
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xff
+	BYTE  $0x04
+	DECQ  R9
+	JNE   sealSSEIntroLoop
 
 	// Add in the state
-	PADDD ·chacha20Constants<>(SB), A0; PADDD ·chacha20Constants<>(SB), A1; PADDD ·chacha20Constants<>(SB), A2; PADDD ·chacha20Constants<>(SB), A3
-	PADDD state1Store, B0; PADDD state1Store, B1; PADDD state1Store, B2; PADDD state1Store, B3
-	PADDD state2Store, C1; PADDD state2Store, C2; PADDD state2Store, C3
-	PADDD ctr1Store, D1; PADDD ctr2Store, D2; PADDD ctr3Store, D3
+	PADDD ·chacha20Constants<>+0(SB), X0
+	PADDD ·chacha20Constants<>+0(SB), X1
+	PADDD ·chacha20Constants<>+0(SB), X2
+	PADDD ·chacha20Constants<>+0(SB), X12
+	PADDD 32(BP), X3
+	PADDD 32(BP), X4
+	PADDD 32(BP), X5
+	PADDD 32(BP), X13
+	PADDD 48(BP), X7
+	PADDD 48(BP), X8
+	PADDD 48(BP), X14
+	PADDD 96(BP), X10
+	PADDD 112(BP), X11
+	PADDD 128(BP), X15
 
 	// Clamp and store the key
-	PAND ·polyClampMask<>(SB), A0
-	MOVO A0, rStore
-	MOVO B0, sStore
+	PAND ·polyClampMask<>+0(SB), X0
+	MOVO X0, (BP)
+	MOVO X3, 16(BP)
 
 	// Hash AAD
-	MOVQ ad_len+80(FP), itr2
-	CALL polyHashADInternal<>(SB)
-
-	MOVOU (0*16)(inp), A0; MOVOU (1*16)(inp), B0; MOVOU (2*16)(inp), C0; MOVOU (3*16)(inp), D0
-	PXOR  A0, A1; PXOR B0, B1; PXOR C0, C1; PXOR D0, D1
-	MOVOU A1, (0*16)(oup); MOVOU B1, (1*16)(oup); MOVOU C1, (2*16)(oup); MOVOU D1, (3*16)(oup)
-	MOVOU (4*16)(inp), A0; MOVOU (5*16)(inp), B0; MOVOU (6*16)(inp), C0; MOVOU (7*16)(inp), D0
-	PXOR  A0, A2; PXOR B0, B2; PXOR C0, C2; PXOR D0, D2
-	MOVOU A2, (4*16)(oup); MOVOU B2, (5*16)(oup); MOVOU C2, (6*16)(oup); MOVOU D2, (7*16)(oup)
-
-	MOVQ $128, itr1
-	SUBQ $128, inl
-	LEAQ 128(inp), inp
-
-	MOVO A3, A1; MOVO B3, B1; MOVO C3, C1; MOVO D3, D1
-
-	CMPQ inl, $64
-	JBE  sealSSE128SealHash
-
-	MOVOU (0*16)(inp), A0; MOVOU (1*16)(inp), B0; MOVOU (2*16)(inp), C0; MOVOU (3*16)(inp), D0
-	PXOR  A0, A3; PXOR B0, B3; PXOR C0, C3; PXOR D0, D3
-	MOVOU A3, (8*16)(oup); MOVOU B3, (9*16)(oup); MOVOU C3, (10*16)(oup); MOVOU D3, (11*16)(oup)
-
-	ADDQ $64, itr1
-	SUBQ $64, inl
-	LEAQ 64(inp), inp
-
-	MOVQ $2, itr1
-	MOVQ $8, itr2
-
-	CMPQ inl, $64
-	JBE  sealSSETail64
-	CMPQ inl, $128
-	JBE  sealSSETail128
-	CMPQ inl, $192
-	JBE  sealSSETail192
+	MOVQ  ad_len+80(FP), R9
+	CALL  polyHashADInternal<>(SB)
+	MOVOU (SI), X0
+	MOVOU 16(SI), X3
+	MOVOU 32(SI), X6
+	MOVOU 48(SI), X9
+	PXOR  X0, X1
+	PXOR  X3, X4
+	PXOR  X6, X7
+	PXOR  X9, X10
+	MOVOU X1, (DI)
+	MOVOU X4, 16(DI)
+	MOVOU X7, 32(DI)
+	MOVOU X10, 48(DI)
+	MOVOU 64(SI), X0
+	MOVOU 80(SI), X3
+	MOVOU 96(SI), X6
+	MOVOU 112(SI), X9
+	PXOR  X0, X2
+	PXOR  X3, X5
+	PXOR  X6, X8
+	PXOR  X9, X11
+	MOVOU X2, 64(DI)
+	MOVOU X5, 80(DI)
+	MOVOU X8, 96(DI)
+	MOVOU X11, 112(DI)
+	MOVQ  $0x00000080, CX
+	SUBQ  $0x80, BX
+	LEAQ  128(SI), SI
+	MOVO  X12, X1
+	MOVO  X13, X4
+	MOVO  X14, X7
+	MOVO  X15, X10
+	CMPQ  BX, $0x40
+	JBE   sealSSE128SealHash
+	MOVOU (SI), X0
+	MOVOU 16(SI), X3
+	MOVOU 32(SI), X6
+	MOVOU 48(SI), X9
+	PXOR  X0, X12
+	PXOR  X3, X13
+	PXOR  X6, X14
+	PXOR  X9, X15
+	MOVOU X12, 128(DI)
+	MOVOU X13, 144(DI)
+	MOVOU X14, 160(DI)
+	MOVOU X15, 176(DI)
+	ADDQ  $0x40, CX
+	SUBQ  $0x40, BX
+	LEAQ  64(SI), SI
+	MOVQ  $0x00000002, CX
+	MOVQ  $0x00000008, R9
+	CMPQ  BX, $0x40
+	JBE   sealSSETail64
+	CMPQ  BX, $0x80
+	JBE   sealSSETail128
+	CMPQ  BX, $0xc0
+	JBE   sealSSETail192
 
 sealSSEMainLoop:
 	// Load state, increment counter blocks
-	MOVO ·chacha20Constants<>(SB), A0; MOVO state1Store, B0; MOVO state2Store, C0; MOVO ctr3Store, D0; PADDL ·sseIncMask<>(SB), D0
-	MOVO A0, A1; MOVO B0, B1; MOVO C0, C1; MOVO D0, D1; PADDL ·sseIncMask<>(SB), D1
-	MOVO A1, A2; MOVO B1, B2; MOVO C1, C2; MOVO D1, D2; PADDL ·sseIncMask<>(SB), D2
-	MOVO A2, A3; MOVO B2, B3; MOVO C2, C3; MOVO D2, D3; PADDL ·sseIncMask<>(SB), D3
+	MOVO  ·chacha20Constants<>+0(SB), X0
+	MOVO  32(BP), X3
+	MOVO  48(BP), X6
+	MOVO  128(BP), X9
+	PADDL ·sseIncMask<>+0(SB), X9
+	MOVO  X0, X1
+	MOVO  X3, X4
+	MOVO  X6, X7
+	MOVO  X9, X10
+	PADDL ·sseIncMask<>+0(SB), X10
+	MOVO  X1, X2
+	MOVO  X4, X5
+	MOVO  X7, X8
+	MOVO  X10, X11
+	PADDL ·sseIncMask<>+0(SB), X11
+	MOVO  X2, X12
+	MOVO  X5, X13
+	MOVO  X8, X14
+	MOVO  X11, X15
+	PADDL ·sseIncMask<>+0(SB), X15
 
 	// Store counters
-	MOVO D0, ctr0Store; MOVO D1, ctr1Store; MOVO D2, ctr2Store; MOVO D3, ctr3Store
+	MOVO X9, 80(BP)
+	MOVO X10, 96(BP)
+	MOVO X11, 112(BP)
+	MOVO X15, 128(BP)
 
 sealSSEInnerLoop:
-	MOVO          C3, tmpStore
-	chachaQR(A0, B0, C0, D0, C3); chachaQR(A1, B1, C1, D1, C3); chachaQR(A2, B2, C2, D2, C3)
-	MOVO          tmpStore, C3
-	MOVO          C1, tmpStore
-	chachaQR(A3, B3, C3, D3, C1)
-	MOVO          tmpStore, C1
-	polyAdd(0(oup))
-	shiftB0Left;  shiftB1Left; shiftB2Left; shiftB3Left
-	shiftC0Left;  shiftC1Left; shiftC2Left; shiftC3Left
-	shiftD0Left;  shiftD1Left; shiftD2Left; shiftD3Left
-	polyMulStage1
-	polyMulStage2
-	LEAQ          (2*8)(oup), oup
-	MOVO          C3, tmpStore
-	chachaQR(A0, B0, C0, D0, C3); chachaQR(A1, B1, C1, D1, C3); chachaQR(A2, B2, C2, D2, C3)
-	MOVO          tmpStore, C3
-	MOVO          C1, tmpStore
-	polyMulStage3
-	chachaQR(A3, B3, C3, D3, C1)
-	MOVO          tmpStore, C1
-	polyMulReduceStage
-	shiftB0Right; shiftB1Right; shiftB2Right; shiftB3Right
-	shiftC0Right; shiftC1Right; shiftC2Right; shiftC3Right
-	shiftD0Right; shiftD1Right; shiftD2Right; shiftD3Right
-	DECQ          itr2
-	JGE           sealSSEInnerLoop
-	polyAdd(0(oup))
-	polyMul
-	LEAQ          (2*8)(oup), oup
-	DECQ          itr1
-	JG            sealSSEInnerLoop
+	MOVO  X14, 64(BP)
+	PADDD X3, X0
+	PXOR  X0, X9
+	ROL16(X9, X14)
+	PADDD X9, X6
+	PXOR  X6, X3
+	MOVO  X3, X14
+	PSLLL $0x0c, X14
+	PSRLL $0x14, X3
+	PXOR  X14, X3
+	PADDD X3, X0
+	PXOR  X0, X9
+	ROL8(X9, X14)
+	PADDD X9, X6
+	PXOR  X6, X3
+	MOVO  X3, X14
+	PSLLL $0x07, X14
+	PSRLL $0x19, X3
+	PXOR  X14, X3
+	PADDD X4, X1
+	PXOR  X1, X10
+	ROL16(X10, X14)
+	PADDD X10, X7
+	PXOR  X7, X4
+	MOVO  X4, X14
+	PSLLL $0x0c, X14
+	PSRLL $0x14, X4
+	PXOR  X14, X4
+	PADDD X4, X1
+	PXOR  X1, X10
+	ROL8(X10, X14)
+	PADDD X10, X7
+	PXOR  X7, X4
+	MOVO  X4, X14
+	PSLLL $0x07, X14
+	PSRLL $0x19, X4
+	PXOR  X14, X4
+	PADDD X5, X2
+	PXOR  X2, X11
+	ROL16(X11, X14)
+	PADDD X11, X8
+	PXOR  X8, X5
+	MOVO  X5, X14
+	PSLLL $0x0c, X14
+	PSRLL $0x14, X5
+	PXOR  X14, X5
+	PADDD X5, X2
+	PXOR  X2, X11
+	ROL8(X11, X14)
+	PADDD X11, X8
+	PXOR  X8, X5
+	MOVO  X5, X14
+	PSLLL $0x07, X14
+	PSRLL $0x19, X5
+	PXOR  X14, X5
+	MOVO  64(BP), X14
+	MOVO  X7, 64(BP)
+	PADDD X13, X12
+	PXOR  X12, X15
+	ROL16(X15, X7)
+	PADDD X15, X14
+	PXOR  X14, X13
+	MOVO  X13, X7
+	PSLLL $0x0c, X7
+	PSRLL $0x14, X13
+	PXOR  X7, X13
+	PADDD X13, X12
+	PXOR  X12, X15
+	ROL8(X15, X7)
+	PADDD X15, X14
+	PXOR  X14, X13
+	MOVO  X13, X7
+	PSLLL $0x07, X7
+	PSRLL $0x19, X13
+	PXOR  X7, X13
+	MOVO  64(BP), X7
+	ADDQ  (DI), R10
+	ADCQ  8(DI), R11
+	ADCQ  $0x01, R12
+	BYTE  $0x66
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xdb
+	BYTE  $0x04
+	BYTE  $0x66
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xe4
+	BYTE  $0x04
+	BYTE  $0x66
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xed
+	BYTE  $0x04
+	BYTE  $0x66
+	BYTE  $0x45
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xed
+	BYTE  $0x04
+	BYTE  $0x66
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xf6
+	BYTE  $0x08
+	BYTE  $0x66
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xff
+	BYTE  $0x08
+	BYTE  $0x66
+	BYTE  $0x45
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xc0
+	BYTE  $0x08
+	BYTE  $0x66
+	BYTE  $0x45
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xf6
+	BYTE  $0x08
+	BYTE  $0x66
+	BYTE  $0x45
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xc9
+	BYTE  $0x0c
+	BYTE  $0x66
+	BYTE  $0x45
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xd2
+	BYTE  $0x0c
+	BYTE  $0x66
+	BYTE  $0x45
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xdb
+	BYTE  $0x0c
+	BYTE  $0x66
+	BYTE  $0x45
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xff
+	BYTE  $0x0c
+	MOVQ  (BP), AX
+	MOVQ  AX, R15
+	MULQ  R10
+	MOVQ  AX, R13
+	MOVQ  DX, R14
+	MOVQ  (BP), AX
+	MULQ  R11
+	IMULQ R12, R15
+	ADDQ  AX, R14
+	ADCQ  DX, R15
+	MOVQ  8(BP), AX
+	MOVQ  AX, R8
+	MULQ  R10
+	ADDQ  AX, R14
+	ADCQ  $0x00, DX
+	MOVQ  DX, R10
+	MOVQ  8(BP), AX
+	MULQ  R11
+	ADDQ  AX, R15
+	ADCQ  $0x00, DX
+	LEAQ  16(DI), DI
+	MOVO  X14, 64(BP)
+	PADDD X3, X0
+	PXOR  X0, X9
+	ROL16(X9, X14)
+	PADDD X9, X6
+	PXOR  X6, X3
+	MOVO  X3, X14
+	PSLLL $0x0c, X14
+	PSRLL $0x14, X3
+	PXOR  X14, X3
+	PADDD X3, X0
+	PXOR  X0, X9
+	ROL8(X9, X14)
+	PADDD X9, X6
+	PXOR  X6, X3
+	MOVO  X3, X14
+	PSLLL $0x07, X14
+	PSRLL $0x19, X3
+	PXOR  X14, X3
+	PADDD X4, X1
+	PXOR  X1, X10
+	ROL16(X10, X14)
+	PADDD X10, X7
+	PXOR  X7, X4
+	MOVO  X4, X14
+	PSLLL $0x0c, X14
+	PSRLL $0x14, X4
+	PXOR  X14, X4
+	PADDD X4, X1
+	PXOR  X1, X10
+	ROL8(X10, X14)
+	PADDD X10, X7
+	PXOR  X7, X4
+	MOVO  X4, X14
+	PSLLL $0x07, X14
+	PSRLL $0x19, X4
+	PXOR  X14, X4
+	PADDD X5, X2
+	PXOR  X2, X11
+	ROL16(X11, X14)
+	PADDD X11, X8
+	PXOR  X8, X5
+	MOVO  X5, X14
+	PSLLL $0x0c, X14
+	PSRLL $0x14, X5
+	PXOR  X14, X5
+	PADDD X5, X2
+	PXOR  X2, X11
+	ROL8(X11, X14)
+	PADDD X11, X8
+	PXOR  X8, X5
+	MOVO  X5, X14
+	PSLLL $0x07, X14
+	PSRLL $0x19, X5
+	PXOR  X14, X5
+	MOVO  64(BP), X14
+	MOVO  X7, 64(BP)
+	IMULQ R12, R8
+	ADDQ  R10, R15
+	ADCQ  DX, R8
+	PADDD X13, X12
+	PXOR  X12, X15
+	ROL16(X15, X7)
+	PADDD X15, X14
+	PXOR  X14, X13
+	MOVO  X13, X7
+	PSLLL $0x0c, X7
+	PSRLL $0x14, X13
+	PXOR  X7, X13
+	PADDD X13, X12
+	PXOR  X12, X15
+	ROL8(X15, X7)
+	PADDD X15, X14
+	PXOR  X14, X13
+	MOVO  X13, X7
+	PSLLL $0x07, X7
+	PSRLL $0x19, X13
+	PXOR  X7, X13
+	MOVO  64(BP), X7
+	MOVQ  R13, R10
+	MOVQ  R14, R11
+	MOVQ  R15, R12
+	ANDQ  $0x03, R12
+	MOVQ  R15, R13
+	ANDQ  $-4, R13
+	MOVQ  R8, R14
+	SHRQ  $0x02, R8, R15
+	SHRQ  $0x02, R8
+	ADDQ  R13, R10
+	ADCQ  R14, R11
+	ADCQ  $0x00, R12
+	ADDQ  R15, R10
+	ADCQ  R8, R11
+	ADCQ  $0x00, R12
+	BYTE  $0x66
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xdb
+	BYTE  $0x0c
+	BYTE  $0x66
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xe4
+	BYTE  $0x0c
+	BYTE  $0x66
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xed
+	BYTE  $0x0c
+	BYTE  $0x66
+	BYTE  $0x45
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xed
+	BYTE  $0x0c
+	BYTE  $0x66
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xf6
+	BYTE  $0x08
+	BYTE  $0x66
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xff
+	BYTE  $0x08
+	BYTE  $0x66
+	BYTE  $0x45
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xc0
+	BYTE  $0x08
+	BYTE  $0x66
+	BYTE  $0x45
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xf6
+	BYTE  $0x08
+	BYTE  $0x66
+	BYTE  $0x45
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xc9
+	BYTE  $0x04
+	BYTE  $0x66
+	BYTE  $0x45
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xd2
+	BYTE  $0x04
+	BYTE  $0x66
+	BYTE  $0x45
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xdb
+	BYTE  $0x04
+	BYTE  $0x66
+	BYTE  $0x45
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xff
+	BYTE  $0x04
+	DECQ  R9
+	JGE   sealSSEInnerLoop
+	ADDQ  (DI), R10
+	ADCQ  8(DI), R11
+	ADCQ  $0x01, R12
+	MOVQ  (BP), AX
+	MOVQ  AX, R15
+	MULQ  R10
+	MOVQ  AX, R13
+	MOVQ  DX, R14
+	MOVQ  (BP), AX
+	MULQ  R11
+	IMULQ R12, R15
+	ADDQ  AX, R14
+	ADCQ  DX, R15
+	MOVQ  8(BP), AX
+	MOVQ  AX, R8
+	MULQ  R10
+	ADDQ  AX, R14
+	ADCQ  $0x00, DX
+	MOVQ  DX, R10
+	MOVQ  8(BP), AX
+	MULQ  R11
+	ADDQ  AX, R15
+	ADCQ  $0x00, DX
+	IMULQ R12, R8
+	ADDQ  R10, R15
+	ADCQ  DX, R8
+	MOVQ  R13, R10
+	MOVQ  R14, R11
+	MOVQ  R15, R12
+	ANDQ  $0x03, R12
+	MOVQ  R15, R13
+	ANDQ  $-4, R13
+	MOVQ  R8, R14
+	SHRQ  $0x02, R8, R15
+	SHRQ  $0x02, R8
+	ADDQ  R13, R10
+	ADCQ  R14, R11
+	ADCQ  $0x00, R12
+	ADDQ  R15, R10
+	ADCQ  R8, R11
+	ADCQ  $0x00, R12
+	LEAQ  16(DI), DI
+	DECQ  CX
+	JG    sealSSEInnerLoop
 
 	// Add in the state
-	PADDD ·chacha20Constants<>(SB), A0; PADDD ·chacha20Constants<>(SB), A1; PADDD ·chacha20Constants<>(SB), A2; PADDD ·chacha20Constants<>(SB), A3
-	PADDD state1Store, B0; PADDD state1Store, B1; PADDD state1Store, B2; PADDD state1Store, B3
-	PADDD state2Store, C0; PADDD state2Store, C1; PADDD state2Store, C2; PADDD state2Store, C3
-	PADDD ctr0Store, D0; PADDD ctr1Store, D1; PADDD ctr2Store, D2; PADDD ctr3Store, D3
-	MOVO  D3, tmpStore
+	PADDD ·chacha20Constants<>+0(SB), X0
+	PADDD ·chacha20Constants<>+0(SB), X1
+	PADDD ·chacha20Constants<>+0(SB), X2
+	PADDD ·chacha20Constants<>+0(SB), X12
+	PADDD 32(BP), X3
+	PADDD 32(BP), X4
+	PADDD 32(BP), X5
+	PADDD 32(BP), X13
+	PADDD 48(BP), X6
+	PADDD 48(BP), X7
+	PADDD 48(BP), X8
+	PADDD 48(BP), X14
+	PADDD 80(BP), X9
+	PADDD 96(BP), X10
+	PADDD 112(BP), X11
+	PADDD 128(BP), X15
+	MOVO  X15, 64(BP)
 
 	// Load - xor - store
-	MOVOU (0*16)(inp), D3; PXOR D3, A0
-	MOVOU (1*16)(inp), D3; PXOR D3, B0
-	MOVOU (2*16)(inp), D3; PXOR D3, C0
-	MOVOU (3*16)(inp), D3; PXOR D3, D0
-	MOVOU A0, (0*16)(oup)
-	MOVOU B0, (1*16)(oup)
-	MOVOU C0, (2*16)(oup)
-	MOVOU D0, (3*16)(oup)
-	MOVO  tmpStore, D3
-
-	MOVOU (4*16)(inp), A0; MOVOU (5*16)(inp), B0; MOVOU (6*16)(inp), C0; MOVOU (7*16)(inp), D0
-	PXOR  A0, A1; PXOR B0, B1; PXOR C0, C1; PXOR D0, D1
-	MOVOU A1, (4*16)(oup); MOVOU B1, (5*16)(oup); MOVOU C1, (6*16)(oup); MOVOU D1, (7*16)(oup)
-	MOVOU (8*16)(inp), A0; MOVOU (9*16)(inp), B0; MOVOU (10*16)(inp), C0; MOVOU (11*16)(inp), D0
-	PXOR  A0, A2; PXOR B0, B2; PXOR C0, C2; PXOR D0, D2
-	MOVOU A2, (8*16)(oup); MOVOU B2, (9*16)(oup); MOVOU C2, (10*16)(oup); MOVOU D2, (11*16)(oup)
-	ADDQ  $192, inp
-	MOVQ  $192, itr1
-	SUBQ  $192, inl
-	MOVO  A3, A1
-	MOVO  B3, B1
-	MOVO  C3, C1
-	MOVO  D3, D1
-	CMPQ  inl, $64
+	MOVOU (SI), X15
+	PXOR  X15, X0
+	MOVOU 16(SI), X15
+	PXOR  X15, X3
+	MOVOU 32(SI), X15
+	PXOR  X15, X6
+	MOVOU 48(SI), X15
+	PXOR  X15, X9
+	MOVOU X0, (DI)
+	MOVOU X3, 16(DI)
+	MOVOU X6, 32(DI)
+	MOVOU X9, 48(DI)
+	MOVO  64(BP), X15
+	MOVOU 64(SI), X0
+	MOVOU 80(SI), X3
+	MOVOU 96(SI), X6
+	MOVOU 112(SI), X9
+	PXOR  X0, X1
+	PXOR  X3, X4
+	PXOR  X6, X7
+	PXOR  X9, X10
+	MOVOU X1, 64(DI)
+	MOVOU X4, 80(DI)
+	MOVOU X7, 96(DI)
+	MOVOU X10, 112(DI)
+	MOVOU 128(SI), X0
+	MOVOU 144(SI), X3
+	MOVOU 160(SI), X6
+	MOVOU 176(SI), X9
+	PXOR  X0, X2
+	PXOR  X3, X5
+	PXOR  X6, X8
+	PXOR  X9, X11
+	MOVOU X2, 128(DI)
+	MOVOU X5, 144(DI)
+	MOVOU X8, 160(DI)
+	MOVOU X11, 176(DI)
+	ADDQ  $0xc0, SI
+	MOVQ  $0x000000c0, CX
+	SUBQ  $0xc0, BX
+	MOVO  X12, X1
+	MOVO  X13, X4
+	MOVO  X14, X7
+	MOVO  X15, X10
+	CMPQ  BX, $0x40
 	JBE   sealSSE128SealHash
-	MOVOU (0*16)(inp), A0; MOVOU (1*16)(inp), B0; MOVOU (2*16)(inp), C0; MOVOU (3*16)(inp), D0
-	PXOR  A0, A3; PXOR B0, B3; PXOR C0, C3; PXOR D0, D3
-	MOVOU A3, (12*16)(oup); MOVOU B3, (13*16)(oup); MOVOU C3, (14*16)(oup); MOVOU D3, (15*16)(oup)
-	LEAQ  64(inp), inp
-	SUBQ  $64, inl
-	MOVQ  $6, itr1
-	MOVQ  $4, itr2
-	CMPQ  inl, $192
+	MOVOU (SI), X0
+	MOVOU 16(SI), X3
+	MOVOU 32(SI), X6
+	MOVOU 48(SI), X9
+	PXOR  X0, X12
+	PXOR  X3, X13
+	PXOR  X6, X14
+	PXOR  X9, X15
+	MOVOU X12, 192(DI)
+	MOVOU X13, 208(DI)
+	MOVOU X14, 224(DI)
+	MOVOU X15, 240(DI)
+	LEAQ  64(SI), SI
+	SUBQ  $0x40, BX
+	MOVQ  $0x00000006, CX
+	MOVQ  $0x00000004, R9
+	CMPQ  BX, $0xc0
 	JG    sealSSEMainLoop
-
-	MOVQ  inl, itr1
-	TESTQ inl, inl
+	MOVQ  BX, CX
+	TESTQ BX, BX
 	JE    sealSSE128SealHash
-	MOVQ  $6, itr1
-	CMPQ  inl, $64
+	MOVQ  $0x00000006, CX
+	CMPQ  BX, $0x40
 	JBE   sealSSETail64
-	CMPQ  inl, $128
+	CMPQ  BX, $0x80
 	JBE   sealSSETail128
 	JMP   sealSSETail192
 
-// ----------------------------------------------------------------------------
-// Special optimization for the last 64 bytes of plaintext
 sealSSETail64:
-	// Need to encrypt up to 64 bytes - prepare single block, hash 192 or 256 bytes
-	MOVO  ·chacha20Constants<>(SB), A1
-	MOVO  state1Store, B1
-	MOVO  state2Store, C1
-	MOVO  ctr3Store, D1
-	PADDL ·sseIncMask<>(SB), D1
-	MOVO  D1, ctr0Store
+	MOVO  ·chacha20Constants<>+0(SB), X1
+	MOVO  32(BP), X4
+	MOVO  48(BP), X7
+	MOVO  128(BP), X10
+	PADDL ·sseIncMask<>+0(SB), X10
+	MOVO  X10, 80(BP)
 
 sealSSETail64LoopA:
-	// Perform ChaCha rounds, while hashing the previously encrypted ciphertext
-	polyAdd(0(oup))
-	polyMul
-	LEAQ 16(oup), oup
+	ADDQ  (DI), R10
+	ADCQ  8(DI), R11
+	ADCQ  $0x01, R12
+	MOVQ  (BP), AX
+	MOVQ  AX, R15
+	MULQ  R10
+	MOVQ  AX, R13
+	MOVQ  DX, R14
+	MOVQ  (BP), AX
+	MULQ  R11
+	IMULQ R12, R15
+	ADDQ  AX, R14
+	ADCQ  DX, R15
+	MOVQ  8(BP), AX
+	MOVQ  AX, R8
+	MULQ  R10
+	ADDQ  AX, R14
+	ADCQ  $0x00, DX
+	MOVQ  DX, R10
+	MOVQ  8(BP), AX
+	MULQ  R11
+	ADDQ  AX, R15
+	ADCQ  $0x00, DX
+	IMULQ R12, R8
+	ADDQ  R10, R15
+	ADCQ  DX, R8
+	MOVQ  R13, R10
+	MOVQ  R14, R11
+	MOVQ  R15, R12
+	ANDQ  $0x03, R12
+	MOVQ  R15, R13
+	ANDQ  $-4, R13
+	MOVQ  R8, R14
+	SHRQ  $0x02, R8, R15
+	SHRQ  $0x02, R8
+	ADDQ  R13, R10
+	ADCQ  R14, R11
+	ADCQ  $0x00, R12
+	ADDQ  R15, R10
+	ADCQ  R8, R11
+	ADCQ  $0x00, R12
+	LEAQ  16(DI), DI
 
 sealSSETail64LoopB:
-	chachaQR(A1, B1, C1, D1, T1)
-	shiftB1Left;  shiftC1Left; shiftD1Left
-	chachaQR(A1, B1, C1, D1, T1)
-	shiftB1Right; shiftC1Right; shiftD1Right
-	polyAdd(0(oup))
-	polyMul
-	LEAQ          16(oup), oup
-
-	DECQ itr1
-	JG   sealSSETail64LoopA
-
-	DECQ  itr2
+	PADDD X4, X1
+	PXOR  X1, X10
+	ROL16(X10, X13)
+	PADDD X10, X7
+	PXOR  X7, X4
+	MOVO  X4, X13
+	PSLLL $0x0c, X13
+	PSRLL $0x14, X4
+	PXOR  X13, X4
+	PADDD X4, X1
+	PXOR  X1, X10
+	ROL8(X10, X13)
+	PADDD X10, X7
+	PXOR  X7, X4
+	MOVO  X4, X13
+	PSLLL $0x07, X13
+	PSRLL $0x19, X4
+	PXOR  X13, X4
+	BYTE  $0x66
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xe4
+	BYTE  $0x04
+	BYTE  $0x66
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xff
+	BYTE  $0x08
+	BYTE  $0x66
+	BYTE  $0x45
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xd2
+	BYTE  $0x0c
+	PADDD X4, X1
+	PXOR  X1, X10
+	ROL16(X10, X13)
+	PADDD X10, X7
+	PXOR  X7, X4
+	MOVO  X4, X13
+	PSLLL $0x0c, X13
+	PSRLL $0x14, X4
+	PXOR  X13, X4
+	PADDD X4, X1
+	PXOR  X1, X10
+	ROL8(X10, X13)
+	PADDD X10, X7
+	PXOR  X7, X4
+	MOVO  X4, X13
+	PSLLL $0x07, X13
+	PSRLL $0x19, X4
+	PXOR  X13, X4
+	BYTE  $0x66
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xe4
+	BYTE  $0x0c
+	BYTE  $0x66
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xff
+	BYTE  $0x08
+	BYTE  $0x66
+	BYTE  $0x45
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xd2
+	BYTE  $0x04
+	ADDQ  (DI), R10
+	ADCQ  8(DI), R11
+	ADCQ  $0x01, R12
+	MOVQ  (BP), AX
+	MOVQ  AX, R15
+	MULQ  R10
+	MOVQ  AX, R13
+	MOVQ  DX, R14
+	MOVQ  (BP), AX
+	MULQ  R11
+	IMULQ R12, R15
+	ADDQ  AX, R14
+	ADCQ  DX, R15
+	MOVQ  8(BP), AX
+	MOVQ  AX, R8
+	MULQ  R10
+	ADDQ  AX, R14
+	ADCQ  $0x00, DX
+	MOVQ  DX, R10
+	MOVQ  8(BP), AX
+	MULQ  R11
+	ADDQ  AX, R15
+	ADCQ  $0x00, DX
+	IMULQ R12, R8
+	ADDQ  R10, R15
+	ADCQ  DX, R8
+	MOVQ  R13, R10
+	MOVQ  R14, R11
+	MOVQ  R15, R12
+	ANDQ  $0x03, R12
+	MOVQ  R15, R13
+	ANDQ  $-4, R13
+	MOVQ  R8, R14
+	SHRQ  $0x02, R8, R15
+	SHRQ  $0x02, R8
+	ADDQ  R13, R10
+	ADCQ  R14, R11
+	ADCQ  $0x00, R12
+	ADDQ  R15, R10
+	ADCQ  R8, R11
+	ADCQ  $0x00, R12
+	LEAQ  16(DI), DI
+	DECQ  CX
+	JG    sealSSETail64LoopA
+	DECQ  R9
 	JGE   sealSSETail64LoopB
-	PADDL ·chacha20Constants<>(SB), A1
-	PADDL state1Store, B1
-	PADDL state2Store, C1
-	PADDL ctr0Store, D1
+	PADDL ·chacha20Constants<>+0(SB), X1
+	PADDL 32(BP), X4
+	PADDL 48(BP), X7
+	PADDL 80(BP), X10
+	JMP   sealSSE128Seal
 
-	JMP sealSSE128Seal
-
-// ----------------------------------------------------------------------------
-// Special optimization for the last 128 bytes of plaintext
 sealSSETail128:
-	// Need to encrypt up to 128 bytes - prepare two blocks, hash 192 or 256 bytes
-	MOVO ·chacha20Constants<>(SB), A0; MOVO state1Store, B0; MOVO state2Store, C0; MOVO ctr3Store, D0; PADDL ·sseIncMask<>(SB), D0; MOVO D0, ctr0Store
-	MOVO A0, A1; MOVO B0, B1; MOVO C0, C1; MOVO D0, D1; PADDL ·sseIncMask<>(SB), D1; MOVO D1, ctr1Store
+	MOVO  ·chacha20Constants<>+0(SB), X0
+	MOVO  32(BP), X3
+	MOVO  48(BP), X6
+	MOVO  128(BP), X9
+	PADDL ·sseIncMask<>+0(SB), X9
+	MOVO  X9, 80(BP)
+	MOVO  X0, X1
+	MOVO  X3, X4
+	MOVO  X6, X7
+	MOVO  X9, X10
+	PADDL ·sseIncMask<>+0(SB), X10
+	MOVO  X10, 96(BP)
 
 sealSSETail128LoopA:
-	// Perform ChaCha rounds, while hashing the previously encrypted ciphertext
-	polyAdd(0(oup))
-	polyMul
-	LEAQ 16(oup), oup
+	ADDQ  (DI), R10
+	ADCQ  8(DI), R11
+	ADCQ  $0x01, R12
+	MOVQ  (BP), AX
+	MOVQ  AX, R15
+	MULQ  R10
+	MOVQ  AX, R13
+	MOVQ  DX, R14
+	MOVQ  (BP), AX
+	MULQ  R11
+	IMULQ R12, R15
+	ADDQ  AX, R14
+	ADCQ  DX, R15
+	MOVQ  8(BP), AX
+	MOVQ  AX, R8
+	MULQ  R10
+	ADDQ  AX, R14
+	ADCQ  $0x00, DX
+	MOVQ  DX, R10
+	MOVQ  8(BP), AX
+	MULQ  R11
+	ADDQ  AX, R15
+	ADCQ  $0x00, DX
+	IMULQ R12, R8
+	ADDQ  R10, R15
+	ADCQ  DX, R8
+	MOVQ  R13, R10
+	MOVQ  R14, R11
+	MOVQ  R15, R12
+	ANDQ  $0x03, R12
+	MOVQ  R15, R13
+	ANDQ  $-4, R13
+	MOVQ  R8, R14
+	SHRQ  $0x02, R8, R15
+	SHRQ  $0x02, R8
+	ADDQ  R13, R10
+	ADCQ  R14, R11
+	ADCQ  $0x00, R12
+	ADDQ  R15, R10
+	ADCQ  R8, R11
+	ADCQ  $0x00, R12
+	LEAQ  16(DI), DI
 
 sealSSETail128LoopB:
-	chachaQR(A0, B0, C0, D0, T0); chachaQR(A1, B1, C1, D1, T0)
-	shiftB0Left;  shiftC0Left; shiftD0Left
-	shiftB1Left;  shiftC1Left; shiftD1Left
-	polyAdd(0(oup))
-	polyMul
-	LEAQ          16(oup), oup
-	chachaQR(A0, B0, C0, D0, T0); chachaQR(A1, B1, C1, D1, T0)
-	shiftB0Right; shiftC0Right; shiftD0Right
-	shiftB1Right; shiftC1Right; shiftD1Right
-
-	DECQ itr1
-	JG   sealSSETail128LoopA
-
-	DECQ itr2
-	JGE  sealSSETail128LoopB
-
-	PADDL ·chacha20Constants<>(SB), A0; PADDL ·chacha20Constants<>(SB), A1
-	PADDL state1Store, B0; PADDL state1Store, B1
-	PADDL state2Store, C0; PADDL state2Store, C1
-	PADDL ctr0Store, D0; PADDL ctr1Store, D1
-
-	MOVOU (0*16)(inp), T0; MOVOU (1*16)(inp), T1; MOVOU (2*16)(inp), T2; MOVOU (3*16)(inp), T3
-	PXOR  T0, A0; PXOR T1, B0; PXOR T2, C0; PXOR T3, D0
-	MOVOU A0, (0*16)(oup); MOVOU B0, (1*16)(oup); MOVOU C0, (2*16)(oup); MOVOU D0, (3*16)(oup)
-
-	MOVQ $64, itr1
-	LEAQ 64(inp), inp
-	SUBQ $64, inl
-
-	JMP sealSSE128SealHash
-
-// ----------------------------------------------------------------------------
-// Special optimization for the last 192 bytes of plaintext
+	PADDD X3, X0
+	PXOR  X0, X9
+	ROL16(X9, X12)
+	PADDD X9, X6
+	PXOR  X6, X3
+	MOVO  X3, X12
+	PSLLL $0x0c, X12
+	PSRLL $0x14, X3
+	PXOR  X12, X3
+	PADDD X3, X0
+	PXOR  X0, X9
+	ROL8(X9, X12)
+	PADDD X9, X6
+	PXOR  X6, X3
+	MOVO  X3, X12
+	PSLLL $0x07, X12
+	PSRLL $0x19, X3
+	PXOR  X12, X3
+	PADDD X4, X1
+	PXOR  X1, X10
+	ROL16(X10, X12)
+	PADDD X10, X7
+	PXOR  X7, X4
+	MOVO  X4, X12
+	PSLLL $0x0c, X12
+	PSRLL $0x14, X4
+	PXOR  X12, X4
+	PADDD X4, X1
+	PXOR  X1, X10
+	ROL8(X10, X12)
+	PADDD X10, X7
+	PXOR  X7, X4
+	MOVO  X4, X12
+	PSLLL $0x07, X12
+	PSRLL $0x19, X4
+	PXOR  X12, X4
+	BYTE  $0x66
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xdb
+	BYTE  $0x04
+	BYTE  $0x66
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xf6
+	BYTE  $0x08
+	BYTE  $0x66
+	BYTE  $0x45
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xc9
+	BYTE  $0x0c
+	BYTE  $0x66
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xe4
+	BYTE  $0x04
+	BYTE  $0x66
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xff
+	BYTE  $0x08
+	BYTE  $0x66
+	BYTE  $0x45
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xd2
+	BYTE  $0x0c
+	ADDQ  (DI), R10
+	ADCQ  8(DI), R11
+	ADCQ  $0x01, R12
+	MOVQ  (BP), AX
+	MOVQ  AX, R15
+	MULQ  R10
+	MOVQ  AX, R13
+	MOVQ  DX, R14
+	MOVQ  (BP), AX
+	MULQ  R11
+	IMULQ R12, R15
+	ADDQ  AX, R14
+	ADCQ  DX, R15
+	MOVQ  8(BP), AX
+	MOVQ  AX, R8
+	MULQ  R10
+	ADDQ  AX, R14
+	ADCQ  $0x00, DX
+	MOVQ  DX, R10
+	MOVQ  8(BP), AX
+	MULQ  R11
+	ADDQ  AX, R15
+	ADCQ  $0x00, DX
+	IMULQ R12, R8
+	ADDQ  R10, R15
+	ADCQ  DX, R8
+	MOVQ  R13, R10
+	MOVQ  R14, R11
+	MOVQ  R15, R12
+	ANDQ  $0x03, R12
+	MOVQ  R15, R13
+	ANDQ  $-4, R13
+	MOVQ  R8, R14
+	SHRQ  $0x02, R8, R15
+	SHRQ  $0x02, R8
+	ADDQ  R13, R10
+	ADCQ  R14, R11
+	ADCQ  $0x00, R12
+	ADDQ  R15, R10
+	ADCQ  R8, R11
+	ADCQ  $0x00, R12
+	LEAQ  16(DI), DI
+	PADDD X3, X0
+	PXOR  X0, X9
+	ROL16(X9, X12)
+	PADDD X9, X6
+	PXOR  X6, X3
+	MOVO  X3, X12
+	PSLLL $0x0c, X12
+	PSRLL $0x14, X3
+	PXOR  X12, X3
+	PADDD X3, X0
+	PXOR  X0, X9
+	ROL8(X9, X12)
+	PADDD X9, X6
+	PXOR  X6, X3
+	MOVO  X3, X12
+	PSLLL $0x07, X12
+	PSRLL $0x19, X3
+	PXOR  X12, X3
+	PADDD X4, X1
+	PXOR  X1, X10
+	ROL16(X10, X12)
+	PADDD X10, X7
+	PXOR  X7, X4
+	MOVO  X4, X12
+	PSLLL $0x0c, X12
+	PSRLL $0x14, X4
+	PXOR  X12, X4
+	PADDD X4, X1
+	PXOR  X1, X10
+	ROL8(X10, X12)
+	PADDD X10, X7
+	PXOR  X7, X4
+	MOVO  X4, X12
+	PSLLL $0x07, X12
+	PSRLL $0x19, X4
+	PXOR  X12, X4
+	BYTE  $0x66
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xdb
+	BYTE  $0x0c
+	BYTE  $0x66
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xf6
+	BYTE  $0x08
+	BYTE  $0x66
+	BYTE  $0x45
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xc9
+	BYTE  $0x04
+	BYTE  $0x66
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xe4
+	BYTE  $0x0c
+	BYTE  $0x66
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xff
+	BYTE  $0x08
+	BYTE  $0x66
+	BYTE  $0x45
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xd2
+	BYTE  $0x04
+	DECQ  CX
+	JG    sealSSETail128LoopA
+	DECQ  R9
+	JGE   sealSSETail128LoopB
+	PADDL ·chacha20Constants<>+0(SB), X0
+	PADDL ·chacha20Constants<>+0(SB), X1
+	PADDL 32(BP), X3
+	PADDL 32(BP), X4
+	PADDL 48(BP), X6
+	PADDL 48(BP), X7
+	PADDL 80(BP), X9
+	PADDL 96(BP), X10
+	MOVOU (SI), X12
+	MOVOU 16(SI), X13
+	MOVOU 32(SI), X14
+	MOVOU 48(SI), X15
+	PXOR  X12, X0
+	PXOR  X13, X3
+	PXOR  X14, X6
+	PXOR  X15, X9
+	MOVOU X0, (DI)
+	MOVOU X3, 16(DI)
+	MOVOU X6, 32(DI)
+	MOVOU X9, 48(DI)
+	MOVQ  $0x00000040, CX
+	LEAQ  64(SI), SI
+	SUBQ  $0x40, BX
+	JMP   sealSSE128SealHash
+
 sealSSETail192:
-	// Need to encrypt up to 192 bytes - prepare three blocks, hash 192 or 256 bytes
-	MOVO ·chacha20Constants<>(SB), A0; MOVO state1Store, B0; MOVO state2Store, C0; MOVO ctr3Store, D0; PADDL ·sseIncMask<>(SB), D0; MOVO D0, ctr0Store
-	MOVO A0, A1; MOVO B0, B1; MOVO C0, C1; MOVO D0, D1; PADDL ·sseIncMask<>(SB), D1; MOVO D1, ctr1Store
-	MOVO A1, A2; MOVO B1, B2; MOVO C1, C2; MOVO D1, D2; PADDL ·sseIncMask<>(SB), D2; MOVO D2, ctr2Store
+	MOVO  ·chacha20Constants<>+0(SB), X0
+	MOVO  32(BP), X3
+	MOVO  48(BP), X6
+	MOVO  128(BP), X9
+	PADDL ·sseIncMask<>+0(SB), X9
+	MOVO  X9, 80(BP)
+	MOVO  X0, X1
+	MOVO  X3, X4
+	MOVO  X6, X7
+	MOVO  X9, X10
+	PADDL ·sseIncMask<>+0(SB), X10
+	MOVO  X10, 96(BP)
+	MOVO  X1, X2
+	MOVO  X4, X5
+	MOVO  X7, X8
+	MOVO  X10, X11
+	PADDL ·sseIncMask<>+0(SB), X11
+	MOVO  X11, 112(BP)
 
 sealSSETail192LoopA:
-	// Perform ChaCha rounds, while hashing the previously encrypted ciphertext
-	polyAdd(0(oup))
-	polyMul
-	LEAQ 16(oup), oup
+	ADDQ  (DI), R10
+	ADCQ  8(DI), R11
+	ADCQ  $0x01, R12
+	MOVQ  (BP), AX
+	MOVQ  AX, R15
+	MULQ  R10
+	MOVQ  AX, R13
+	MOVQ  DX, R14
+	MOVQ  (BP), AX
+	MULQ  R11
+	IMULQ R12, R15
+	ADDQ  AX, R14
+	ADCQ  DX, R15
+	MOVQ  8(BP), AX
+	MOVQ  AX, R8
+	MULQ  R10
+	ADDQ  AX, R14
+	ADCQ  $0x00, DX
+	MOVQ  DX, R10
+	MOVQ  8(BP), AX
+	MULQ  R11
+	ADDQ  AX, R15
+	ADCQ  $0x00, DX
+	IMULQ R12, R8
+	ADDQ  R10, R15
+	ADCQ  DX, R8
+	MOVQ  R13, R10
+	MOVQ  R14, R11
+	MOVQ  R15, R12
+	ANDQ  $0x03, R12
+	MOVQ  R15, R13
+	ANDQ  $-4, R13
+	MOVQ  R8, R14
+	SHRQ  $0x02, R8, R15
+	SHRQ  $0x02, R8
+	ADDQ  R13, R10
+	ADCQ  R14, R11
+	ADCQ  $0x00, R12
+	ADDQ  R15, R10
+	ADCQ  R8, R11
+	ADCQ  $0x00, R12
+	LEAQ  16(DI), DI
 
 sealSSETail192LoopB:
-	chachaQR(A0, B0, C0, D0, T0); chachaQR(A1, B1, C1, D1, T0); chachaQR(A2, B2, C2, D2, T0)
-	shiftB0Left; shiftC0Left; shiftD0Left
-	shiftB1Left; shiftC1Left; shiftD1Left
-	shiftB2Left; shiftC2Left; shiftD2Left
-
-	polyAdd(0(oup))
-	polyMul
-	LEAQ 16(oup), oup
-
-	chachaQR(A0, B0, C0, D0, T0); chachaQR(A1, B1, C1, D1, T0); chachaQR(A2, B2, C2, D2, T0)
-	shiftB0Right; shiftC0Right; shiftD0Right
-	shiftB1Right; shiftC1Right; shiftD1Right
-	shiftB2Right; shiftC2Right; shiftD2Right
-
-	DECQ itr1
-	JG   sealSSETail192LoopA
-
-	DECQ itr2
-	JGE  sealSSETail192LoopB
-
-	PADDL ·chacha20Constants<>(SB), A0; PADDL ·chacha20Constants<>(SB), A1; PADDL ·chacha20Constants<>(SB), A2
-	PADDL state1Store, B0; PADDL state1Store, B1; PADDL state1Store, B2
-	PADDL state2Store, C0; PADDL state2Store, C1; PADDL state2Store, C2
-	PADDL ctr0Store, D0; PADDL ctr1Store, D1; PADDL ctr2Store, D2
-
-	MOVOU (0*16)(inp), T0; MOVOU (1*16)(inp), T1; MOVOU (2*16)(inp), T2; MOVOU (3*16)(inp), T3
-	PXOR  T0, A0; PXOR T1, B0; PXOR T2, C0; PXOR T3, D0
-	MOVOU A0, (0*16)(oup); MOVOU B0, (1*16)(oup); MOVOU C0, (2*16)(oup); MOVOU D0, (3*16)(oup)
-	MOVOU (4*16)(inp), T0; MOVOU (5*16)(inp), T1; MOVOU (6*16)(inp), T2; MOVOU (7*16)(inp), T3
-	PXOR  T0, A1; PXOR T1, B1; PXOR T2, C1; PXOR T3, D1
-	MOVOU A1, (4*16)(oup); MOVOU B1, (5*16)(oup); MOVOU C1, (6*16)(oup); MOVOU D1, (7*16)(oup)
-
-	MOVO A2, A1
-	MOVO B2, B1
-	MOVO C2, C1
-	MOVO D2, D1
-	MOVQ $128, itr1
-	LEAQ 128(inp), inp
-	SUBQ $128, inl
-
-	JMP sealSSE128SealHash
-
-// ----------------------------------------------------------------------------
-// Special seal optimization for buffers smaller than 129 bytes
+	PADDD X3, X0
+	PXOR  X0, X9
+	ROL16(X9, X12)
+	PADDD X9, X6
+	PXOR  X6, X3
+	MOVO  X3, X12
+	PSLLL $0x0c, X12
+	PSRLL $0x14, X3
+	PXOR  X12, X3
+	PADDD X3, X0
+	PXOR  X0, X9
+	ROL8(X9, X12)
+	PADDD X9, X6
+	PXOR  X6, X3
+	MOVO  X3, X12
+	PSLLL $0x07, X12
+	PSRLL $0x19, X3
+	PXOR  X12, X3
+	PADDD X4, X1
+	PXOR  X1, X10
+	ROL16(X10, X12)
+	PADDD X10, X7
+	PXOR  X7, X4
+	MOVO  X4, X12
+	PSLLL $0x0c, X12
+	PSRLL $0x14, X4
+	PXOR  X12, X4
+	PADDD X4, X1
+	PXOR  X1, X10
+	ROL8(X10, X12)
+	PADDD X10, X7
+	PXOR  X7, X4
+	MOVO  X4, X12
+	PSLLL $0x07, X12
+	PSRLL $0x19, X4
+	PXOR  X12, X4
+	PADDD X5, X2
+	PXOR  X2, X11
+	ROL16(X11, X12)
+	PADDD X11, X8
+	PXOR  X8, X5
+	MOVO  X5, X12
+	PSLLL $0x0c, X12
+	PSRLL $0x14, X5
+	PXOR  X12, X5
+	PADDD X5, X2
+	PXOR  X2, X11
+	ROL8(X11, X12)
+	PADDD X11, X8
+	PXOR  X8, X5
+	MOVO  X5, X12
+	PSLLL $0x07, X12
+	PSRLL $0x19, X5
+	PXOR  X12, X5
+	BYTE  $0x66
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xdb
+	BYTE  $0x04
+	BYTE  $0x66
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xf6
+	BYTE  $0x08
+	BYTE  $0x66
+	BYTE  $0x45
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xc9
+	BYTE  $0x0c
+	BYTE  $0x66
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xe4
+	BYTE  $0x04
+	BYTE  $0x66
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xff
+	BYTE  $0x08
+	BYTE  $0x66
+	BYTE  $0x45
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xd2
+	BYTE  $0x0c
+	BYTE  $0x66
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xed
+	BYTE  $0x04
+	BYTE  $0x66
+	BYTE  $0x45
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xc0
+	BYTE  $0x08
+	BYTE  $0x66
+	BYTE  $0x45
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xdb
+	BYTE  $0x0c
+	ADDQ  (DI), R10
+	ADCQ  8(DI), R11
+	ADCQ  $0x01, R12
+	MOVQ  (BP), AX
+	MOVQ  AX, R15
+	MULQ  R10
+	MOVQ  AX, R13
+	MOVQ  DX, R14
+	MOVQ  (BP), AX
+	MULQ  R11
+	IMULQ R12, R15
+	ADDQ  AX, R14
+	ADCQ  DX, R15
+	MOVQ  8(BP), AX
+	MOVQ  AX, R8
+	MULQ  R10
+	ADDQ  AX, R14
+	ADCQ  $0x00, DX
+	MOVQ  DX, R10
+	MOVQ  8(BP), AX
+	MULQ  R11
+	ADDQ  AX, R15
+	ADCQ  $0x00, DX
+	IMULQ R12, R8
+	ADDQ  R10, R15
+	ADCQ  DX, R8
+	MOVQ  R13, R10
+	MOVQ  R14, R11
+	MOVQ  R15, R12
+	ANDQ  $0x03, R12
+	MOVQ  R15, R13
+	ANDQ  $-4, R13
+	MOVQ  R8, R14
+	SHRQ  $0x02, R8, R15
+	SHRQ  $0x02, R8
+	ADDQ  R13, R10
+	ADCQ  R14, R11
+	ADCQ  $0x00, R12
+	ADDQ  R15, R10
+	ADCQ  R8, R11
+	ADCQ  $0x00, R12
+	LEAQ  16(DI), DI
+	PADDD X3, X0
+	PXOR  X0, X9
+	ROL16(X9, X12)
+	PADDD X9, X6
+	PXOR  X6, X3
+	MOVO  X3, X12
+	PSLLL $0x0c, X12
+	PSRLL $0x14, X3
+	PXOR  X12, X3
+	PADDD X3, X0
+	PXOR  X0, X9
+	ROL8(X9, X12)
+	PADDD X9, X6
+	PXOR  X6, X3
+	MOVO  X3, X12
+	PSLLL $0x07, X12
+	PSRLL $0x19, X3
+	PXOR  X12, X3
+	PADDD X4, X1
+	PXOR  X1, X10
+	ROL16(X10, X12)
+	PADDD X10, X7
+	PXOR  X7, X4
+	MOVO  X4, X12
+	PSLLL $0x0c, X12
+	PSRLL $0x14, X4
+	PXOR  X12, X4
+	PADDD X4, X1
+	PXOR  X1, X10
+	ROL8(X10, X12)
+	PADDD X10, X7
+	PXOR  X7, X4
+	MOVO  X4, X12
+	PSLLL $0x07, X12
+	PSRLL $0x19, X4
+	PXOR  X12, X4
+	PADDD X5, X2
+	PXOR  X2, X11
+	ROL16(X11, X12)
+	PADDD X11, X8
+	PXOR  X8, X5
+	MOVO  X5, X12
+	PSLLL $0x0c, X12
+	PSRLL $0x14, X5
+	PXOR  X12, X5
+	PADDD X5, X2
+	PXOR  X2, X11
+	ROL8(X11, X12)
+	PADDD X11, X8
+	PXOR  X8, X5
+	MOVO  X5, X12
+	PSLLL $0x07, X12
+	PSRLL $0x19, X5
+	PXOR  X12, X5
+	BYTE  $0x66
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xdb
+	BYTE  $0x0c
+	BYTE  $0x66
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xf6
+	BYTE  $0x08
+	BYTE  $0x66
+	BYTE  $0x45
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xc9
+	BYTE  $0x04
+	BYTE  $0x66
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xe4
+	BYTE  $0x0c
+	BYTE  $0x66
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xff
+	BYTE  $0x08
+	BYTE  $0x66
+	BYTE  $0x45
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xd2
+	BYTE  $0x04
+	BYTE  $0x66
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xed
+	BYTE  $0x0c
+	BYTE  $0x66
+	BYTE  $0x45
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xc0
+	BYTE  $0x08
+	BYTE  $0x66
+	BYTE  $0x45
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xdb
+	BYTE  $0x04
+	DECQ  CX
+	JG    sealSSETail192LoopA
+	DECQ  R9
+	JGE   sealSSETail192LoopB
+	PADDL ·chacha20Constants<>+0(SB), X0
+	PADDL ·chacha20Constants<>+0(SB), X1
+	PADDL ·chacha20Constants<>+0(SB), X2
+	PADDL 32(BP), X3
+	PADDL 32(BP), X4
+	PADDL 32(BP), X5
+	PADDL 48(BP), X6
+	PADDL 48(BP), X7
+	PADDL 48(BP), X8
+	PADDL 80(BP), X9
+	PADDL 96(BP), X10
+	PADDL 112(BP), X11
+	MOVOU (SI), X12
+	MOVOU 16(SI), X13
+	MOVOU 32(SI), X14
+	MOVOU 48(SI), X15
+	PXOR  X12, X0
+	PXOR  X13, X3
+	PXOR  X14, X6
+	PXOR  X15, X9
+	MOVOU X0, (DI)
+	MOVOU X3, 16(DI)
+	MOVOU X6, 32(DI)
+	MOVOU X9, 48(DI)
+	MOVOU 64(SI), X12
+	MOVOU 80(SI), X13
+	MOVOU 96(SI), X14
+	MOVOU 112(SI), X15
+	PXOR  X12, X1
+	PXOR  X13, X4
+	PXOR  X14, X7
+	PXOR  X15, X10
+	MOVOU X1, 64(DI)
+	MOVOU X4, 80(DI)
+	MOVOU X7, 96(DI)
+	MOVOU X10, 112(DI)
+	MOVO  X2, X1
+	MOVO  X5, X4
+	MOVO  X8, X7
+	MOVO  X11, X10
+	MOVQ  $0x00000080, CX
+	LEAQ  128(SI), SI
+	SUBQ  $0x80, BX
+	JMP   sealSSE128SealHash
+
 sealSSE128:
-	// For up to 128 bytes of ciphertext and 64 bytes for the poly key, we require to process three blocks
-	MOVOU ·chacha20Constants<>(SB), A0; MOVOU (1*16)(keyp), B0; MOVOU (2*16)(keyp), C0; MOVOU (3*16)(keyp), D0
-	MOVO  A0, A1; MOVO B0, B1; MOVO C0, C1; MOVO D0, D1; PADDL ·sseIncMask<>(SB), D1
-	MOVO  A1, A2; MOVO B1, B2; MOVO C1, C2; MOVO D1, D2; PADDL ·sseIncMask<>(SB), D2
-	MOVO  B0, T1; MOVO C0, T2; MOVO D1, T3
-	MOVQ  $10, itr2
+	MOVOU ·chacha20Constants<>+0(SB), X0
+	MOVOU 16(R8), X3
+	MOVOU 32(R8), X6
+	MOVOU 48(R8), X9
+	MOVO  X0, X1
+	MOVO  X3, X4
+	MOVO  X6, X7
+	MOVO  X9, X10
+	PADDL ·sseIncMask<>+0(SB), X10
+	MOVO  X1, X2
+	MOVO  X4, X5
+	MOVO  X7, X8
+	MOVO  X10, X11
+	PADDL ·sseIncMask<>+0(SB), X11
+	MOVO  X3, X13
+	MOVO  X6, X14
+	MOVO  X10, X15
+	MOVQ  $0x0000000a, R9
 
 sealSSE128InnerCipherLoop:
-	chachaQR(A0, B0, C0, D0, T0); chachaQR(A1, B1, C1, D1, T0); chachaQR(A2, B2, C2, D2, T0)
-	shiftB0Left;  shiftB1Left; shiftB2Left
-	shiftC0Left;  shiftC1Left; shiftC2Left
-	shiftD0Left;  shiftD1Left; shiftD2Left
-	chachaQR(A0, B0, C0, D0, T0); chachaQR(A1, B1, C1, D1, T0); chachaQR(A2, B2, C2, D2, T0)
-	shiftB0Right; shiftB1Right; shiftB2Right
-	shiftC0Right; shiftC1Right; shiftC2Right
-	shiftD0Right; shiftD1Right; shiftD2Right
-	DECQ          itr2
-	JNE           sealSSE128InnerCipherLoop
+	PADDD X3, X0
+	PXOR  X0, X9
+	ROL16(X9, X12)
+	PADDD X9, X6
+	PXOR  X6, X3
+	MOVO  X3, X12
+	PSLLL $0x0c, X12
+	PSRLL $0x14, X3
+	PXOR  X12, X3
+	PADDD X3, X0
+	PXOR  X0, X9
+	ROL8(X9, X12)
+	PADDD X9, X6
+	PXOR  X6, X3
+	MOVO  X3, X12
+	PSLLL $0x07, X12
+	PSRLL $0x19, X3
+	PXOR  X12, X3
+	PADDD X4, X1
+	PXOR  X1, X10
+	ROL16(X10, X12)
+	PADDD X10, X7
+	PXOR  X7, X4
+	MOVO  X4, X12
+	PSLLL $0x0c, X12
+	PSRLL $0x14, X4
+	PXOR  X12, X4
+	PADDD X4, X1
+	PXOR  X1, X10
+	ROL8(X10, X12)
+	PADDD X10, X7
+	PXOR  X7, X4
+	MOVO  X4, X12
+	PSLLL $0x07, X12
+	PSRLL $0x19, X4
+	PXOR  X12, X4
+	PADDD X5, X2
+	PXOR  X2, X11
+	ROL16(X11, X12)
+	PADDD X11, X8
+	PXOR  X8, X5
+	MOVO  X5, X12
+	PSLLL $0x0c, X12
+	PSRLL $0x14, X5
+	PXOR  X12, X5
+	PADDD X5, X2
+	PXOR  X2, X11
+	ROL8(X11, X12)
+	PADDD X11, X8
+	PXOR  X8, X5
+	MOVO  X5, X12
+	PSLLL $0x07, X12
+	PSRLL $0x19, X5
+	PXOR  X12, X5
+	BYTE  $0x66
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xdb
+	BYTE  $0x04
+	BYTE  $0x66
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xe4
+	BYTE  $0x04
+	BYTE  $0x66
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xed
+	BYTE  $0x04
+	BYTE  $0x66
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xf6
+	BYTE  $0x08
+	BYTE  $0x66
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xff
+	BYTE  $0x08
+	BYTE  $0x66
+	BYTE  $0x45
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xc0
+	BYTE  $0x08
+	BYTE  $0x66
+	BYTE  $0x45
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xc9
+	BYTE  $0x0c
+	BYTE  $0x66
+	BYTE  $0x45
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xd2
+	BYTE  $0x0c
+	BYTE  $0x66
+	BYTE  $0x45
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xdb
+	BYTE  $0x0c
+	PADDD X3, X0
+	PXOR  X0, X9
+	ROL16(X9, X12)
+	PADDD X9, X6
+	PXOR  X6, X3
+	MOVO  X3, X12
+	PSLLL $0x0c, X12
+	PSRLL $0x14, X3
+	PXOR  X12, X3
+	PADDD X3, X0
+	PXOR  X0, X9
+	ROL8(X9, X12)
+	PADDD X9, X6
+	PXOR  X6, X3
+	MOVO  X3, X12
+	PSLLL $0x07, X12
+	PSRLL $0x19, X3
+	PXOR  X12, X3
+	PADDD X4, X1
+	PXOR  X1, X10
+	ROL16(X10, X12)
+	PADDD X10, X7
+	PXOR  X7, X4
+	MOVO  X4, X12
+	PSLLL $0x0c, X12
+	PSRLL $0x14, X4
+	PXOR  X12, X4
+	PADDD X4, X1
+	PXOR  X1, X10
+	ROL8(X10, X12)
+	PADDD X10, X7
+	PXOR  X7, X4
+	MOVO  X4, X12
+	PSLLL $0x07, X12
+	PSRLL $0x19, X4
+	PXOR  X12, X4
+	PADDD X5, X2
+	PXOR  X2, X11
+	ROL16(X11, X12)
+	PADDD X11, X8
+	PXOR  X8, X5
+	MOVO  X5, X12
+	PSLLL $0x0c, X12
+	PSRLL $0x14, X5
+	PXOR  X12, X5
+	PADDD X5, X2
+	PXOR  X2, X11
+	ROL8(X11, X12)
+	PADDD X11, X8
+	PXOR  X8, X5
+	MOVO  X5, X12
+	PSLLL $0x07, X12
+	PSRLL $0x19, X5
+	PXOR  X12, X5
+	BYTE  $0x66
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xdb
+	BYTE  $0x0c
+	BYTE  $0x66
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xe4
+	BYTE  $0x0c
+	BYTE  $0x66
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xed
+	BYTE  $0x0c
+	BYTE  $0x66
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xf6
+	BYTE  $0x08
+	BYTE  $0x66
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xff
+	BYTE  $0x08
+	BYTE  $0x66
+	BYTE  $0x45
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xc0
+	BYTE  $0x08
+	BYTE  $0x66
+	BYTE  $0x45
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xc9
+	BYTE  $0x04
+	BYTE  $0x66
+	BYTE  $0x45
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xd2
+	BYTE  $0x04
+	BYTE  $0x66
+	BYTE  $0x45
+	BYTE  $0x0f
+	BYTE  $0x3a
+	BYTE  $0x0f
+	BYTE  $0xdb
+	BYTE  $0x04
+	DECQ  R9
+	JNE   sealSSE128InnerCipherLoop
 
 	// A0|B0 hold the Poly1305 32-byte key, C0,D0 can be discarded
-	PADDL ·chacha20Constants<>(SB), A0; PADDL ·chacha20Constants<>(SB), A1; PADDL ·chacha20Constants<>(SB), A2
-	PADDL T1, B0; PADDL T1, B1; PADDL T1, B2
-	PADDL T2, C1; PADDL T2, C2
-	PADDL T3, D1; PADDL ·sseIncMask<>(SB), T3; PADDL T3, D2
-	PAND  ·polyClampMask<>(SB), A0
-	MOVOU A0, rStore
-	MOVOU B0, sStore
+	PADDL ·chacha20Constants<>+0(SB), X0
+	PADDL ·chacha20Constants<>+0(SB), X1
+	PADDL ·chacha20Constants<>+0(SB), X2
+	PADDL X13, X3
+	PADDL X13, X4
+	PADDL X13, X5
+	PADDL X14, X7
+	PADDL X14, X8
+	PADDL X15, X10
+	PADDL ·sseIncMask<>+0(SB), X15
+	PADDL X15, X11
+	PAND  ·polyClampMask<>+0(SB), X0
+	MOVOU X0, (BP)
+	MOVOU X3, 16(BP)
 
 	// Hash
-	MOVQ ad_len+80(FP), itr2
+	MOVQ ad_len+80(FP), R9
 	CALL polyHashADInternal<>(SB)
-	XORQ itr1, itr1
+	XORQ CX, CX
 
 sealSSE128SealHash:
-	// itr1 holds the number of bytes encrypted but not yet hashed
-	CMPQ itr1, $16
-	JB   sealSSE128Seal
-	polyAdd(0(oup))
-	polyMul
-
-	SUBQ $16, itr1
-	ADDQ $16, oup
-
-	JMP sealSSE128SealHash
+	CMPQ  CX, $0x10
+	JB    sealSSE128Seal
+	ADDQ  (DI), R10
+	ADCQ  8(DI), R11
+	ADCQ  $0x01, R12
+	MOVQ  (BP), AX
+	MOVQ  AX, R15
+	MULQ  R10
+	MOVQ  AX, R13
+	MOVQ  DX, R14
+	MOVQ  (BP), AX
+	MULQ  R11
+	IMULQ R12, R15
+	ADDQ  AX, R14
+	ADCQ  DX, R15
+	MOVQ  8(BP), AX
+	MOVQ  AX, R8
+	MULQ  R10
+	ADDQ  AX, R14
+	ADCQ  $0x00, DX
+	MOVQ  DX, R10
+	MOVQ  8(BP), AX
+	MULQ  R11
+	ADDQ  AX, R15
+	ADCQ  $0x00, DX
+	IMULQ R12, R8
+	ADDQ  R10, R15
+	ADCQ  DX, R8
+	MOVQ  R13, R10
+	MOVQ  R14, R11
+	MOVQ  R15, R12
+	ANDQ  $0x03, R12
+	MOVQ  R15, R13
+	ANDQ  $-4, R13
+	MOVQ  R8, R14
+	SHRQ  $0x02, R8, R15
+	SHRQ  $0x02, R8
+	ADDQ  R13, R10
+	ADCQ  R14, R11
+	ADCQ  $0x00, R12
+	ADDQ  R15, R10
+	ADCQ  R8, R11
+	ADCQ  $0x00, R12
+	SUBQ  $0x10, CX
+	ADDQ  $0x10, DI
+	JMP   sealSSE128SealHash
 
 sealSSE128Seal:
-	CMPQ inl, $16
+	CMPQ BX, $0x10
 	JB   sealSSETail
-	SUBQ $16, inl
+	SUBQ $0x10, BX
 
 	// Load for decryption
-	MOVOU (inp), T0
-	PXOR  T0, A1
-	MOVOU A1, (oup)
-	LEAQ  (1*16)(inp), inp
-	LEAQ  (1*16)(oup), oup
+	MOVOU (SI), X12
+	PXOR  X12, X1
+	MOVOU X1, (DI)
+	LEAQ  16(SI), SI
+	LEAQ  16(DI), DI
 
 	// Extract for hashing
-	MOVQ   A1, t0
-	PSRLDQ $8, A1
-	MOVQ A1, t1
-	ADDQ   t0, acc0; ADCQ t1, acc1; ADCQ $1, acc2
-	polyMul
+	MOVQ   X1, R13
+	PSRLDQ $0x08, X1
+	MOVQ   X1, R14
+	ADDQ   R13, R10
+	ADCQ   R14, R11
+	ADCQ   $0x01, R12
+	MOVQ   (BP), AX
+	MOVQ   AX, R15
+	MULQ   R10
+	MOVQ   AX, R13
+	MOVQ   DX, R14
+	MOVQ   (BP), AX
+	MULQ   R11
+	IMULQ  R12, R15
+	ADDQ   AX, R14
+	ADCQ   DX, R15
+	MOVQ   8(BP), AX
+	MOVQ   AX, R8
+	MULQ   R10
+	ADDQ   AX, R14
+	ADCQ   $0x00, DX
+	MOVQ   DX, R10
+	MOVQ   8(BP), AX
+	MULQ   R11
+	ADDQ   AX, R15
+	ADCQ   $0x00, DX
+	IMULQ  R12, R8
+	ADDQ   R10, R15
+	ADCQ   DX, R8
+	MOVQ   R13, R10
+	MOVQ   R14, R11
+	MOVQ   R15, R12
+	ANDQ   $0x03, R12
+	MOVQ   R15, R13
+	ANDQ   $-4, R13
+	MOVQ   R8, R14
+	SHRQ   $0x02, R8, R15
+	SHRQ   $0x02, R8
+	ADDQ   R13, R10
+	ADCQ   R14, R11
+	ADCQ   $0x00, R12
+	ADDQ   R15, R10
+	ADCQ   R8, R11
+	ADCQ   $0x00, R12
 
 	// Shift the stream "left"
-	MOVO B1, A1
-	MOVO C1, B1
-	MOVO D1, C1
-	MOVO A2, D1
-	MOVO B2, A2
-	MOVO C2, B2
-	MOVO D2, C2
+	MOVO X4, X1
+	MOVO X7, X4
+	MOVO X10, X7
+	MOVO X2, X10
+	MOVO X5, X2
+	MOVO X8, X5
+	MOVO X11, X8
 	JMP  sealSSE128Seal
 
 sealSSETail:
-	TESTQ inl, inl
+	TESTQ BX, BX
 	JE    sealSSEFinalize
 
 	// We can only load the PT one byte at a time to avoid read after end of buffer
-	MOVQ inl, itr2
-	SHLQ $4, itr2
-	LEAQ ·andMask<>(SB), t0
-	MOVQ inl, itr1
-	LEAQ -1(inp)(inl*1), inp
-	XORQ t2, t2
-	XORQ t3, t3
+	MOVQ BX, R9
+	SHLQ $0x04, R9
+	LEAQ ·andMask<>+0(SB), R13
+	MOVQ BX, CX
+	LEAQ -1(SI)(BX*1), SI
+	XORQ R15, R15
+	XORQ R8, R8
 	XORQ AX, AX
 
 sealSSETailLoadLoop:
-	SHLQ $8, t2, t3
-	SHLQ $8, t2
-	MOVB (inp), AX
-	XORQ AX, t2
-	LEAQ   -1(inp), inp
-	DECQ   itr1
+	SHLQ   $0x08, R15, R8
+	SHLQ   $0x08, R15
+	MOVB   (SI), AX
+	XORQ   AX, R15
+	LEAQ   -1(SI), SI
+	DECQ   CX
 	JNE    sealSSETailLoadLoop
-	MOVQ t2, 0+tmpStore
-	MOVQ t3, 8+tmpStore
-	PXOR 0+tmpStore, A1
-	MOVOU  A1, (oup)
-	MOVOU  -16(t0)(itr2*1), T0
-	PAND   T0, A1
-	MOVQ   A1, t0
-	PSRLDQ $8, A1
-	MOVQ   A1, t1
-	ADDQ   t0, acc0; ADCQ t1, acc1; ADCQ $1, acc2
-	polyMul
-
-	ADDQ inl, oup
+	MOVQ   R15, 64(BP)
+	MOVQ   R8, 72(BP)
+	PXOR   64(BP), X1
+	MOVOU  X1, (DI)
+	MOVOU  -16(R13)(R9*1), X12
+	PAND   X12, X1
+	MOVQ   X1, R13
+	PSRLDQ $0x08, X1
+	MOVQ   X1, R14
+	ADDQ   R13, R10
+	ADCQ   R14, R11
+	ADCQ   $0x01, R12
+	MOVQ   (BP), AX
+	MOVQ   AX, R15
+	MULQ   R10
+	MOVQ   AX, R13
+	MOVQ   DX, R14
+	MOVQ   (BP), AX
+	MULQ   R11
+	IMULQ  R12, R15
+	ADDQ   AX, R14
+	ADCQ   DX, R15
+	MOVQ   8(BP), AX
+	MOVQ   AX, R8
+	MULQ   R10
+	ADDQ   AX, R14
+	ADCQ   $0x00, DX
+	MOVQ   DX, R10
+	MOVQ   8(BP), AX
+	MULQ   R11
+	ADDQ   AX, R15
+	ADCQ   $0x00, DX
+	IMULQ  R12, R8
+	ADDQ   R10, R15
+	ADCQ   DX, R8
+	MOVQ   R13, R10
+	MOVQ   R14, R11
+	MOVQ   R15, R12
+	ANDQ   $0x03, R12
+	MOVQ   R15, R13
+	ANDQ   $-4, R13
+	MOVQ   R8, R14
+	SHRQ   $0x02, R8, R15
+	SHRQ   $0x02, R8
+	ADDQ   R13, R10
+	ADCQ   R14, R11
+	ADCQ   $0x00, R12
+	ADDQ   R15, R10
+	ADCQ   R8, R11
+	ADCQ   $0x00, R12
+	ADDQ   BX, DI
 
 sealSSEFinalize:
 	// Hash in the buffer lengths
-	ADDQ ad_len+80(FP), acc0
-	ADCQ src_len+56(FP), acc1
-	ADCQ $1, acc2
-	polyMul
+	ADDQ  ad_len+80(FP), R10
+	ADCQ  src_len+56(FP), R11
+	ADCQ  $0x01, R12
+	MOVQ  (BP), AX
+	MOVQ  AX, R15
+	MULQ  R10
+	MOVQ  AX, R13
+	MOVQ  DX, R14
+	MOVQ  (BP), AX
+	MULQ  R11
+	IMULQ R12, R15
+	ADDQ  AX, R14
+	ADCQ  DX, R15
+	MOVQ  8(BP), AX
+	MOVQ  AX, R8
+	MULQ  R10
+	ADDQ  AX, R14
+	ADCQ  $0x00, DX
+	MOVQ  DX, R10
+	MOVQ  8(BP), AX
+	MULQ  R11
+	ADDQ  AX, R15
+	ADCQ  $0x00, DX
+	IMULQ R12, R8
+	ADDQ  R10, R15
+	ADCQ  DX, R8
+	MOVQ  R13, R10
+	MOVQ  R14, R11
+	MOVQ  R15, R12
+	ANDQ  $0x03, R12
+	MOVQ  R15, R13
+	ANDQ  $-4, R13
+	MOVQ  R8, R14
+	SHRQ  $0x02, R8, R15
+	SHRQ  $0x02, R8
+	ADDQ  R13, R10
+	ADCQ  R14, R11
+	ADCQ  $0x00, R12
+	ADDQ  R15, R10
+	ADCQ  R8, R11
+	ADCQ  $0x00, R12
 
 	// Final reduce
-	MOVQ    acc0, t0
-	MOVQ    acc1, t1
-	MOVQ    acc2, t2
-	SUBQ    $-5, acc0
-	SBBQ    $-1, acc1
-	SBBQ    $3, acc2
-	CMOVQCS t0, acc0
-	CMOVQCS t1, acc1
-	CMOVQCS t2, acc2
+	MOVQ    R10, R13
+	MOVQ    R11, R14
+	MOVQ    R12, R15
+	SUBQ    $-5, R10
+	SBBQ    $-1, R11
+	SBBQ    $0x03, R12
+	CMOVQCS R13, R10
+	CMOVQCS R14, R11
+	CMOVQCS R15, R12
 
 	// Add in the "s" part of the key
-	ADDQ 0+sStore, acc0
-	ADCQ 8+sStore, acc1
+	ADDQ 16(BP), R10
+	ADCQ 24(BP), R11
 
 	// Finally store the tag at the end of the message
-	MOVQ acc0, (0*8)(oup)
-	MOVQ acc1, (1*8)(oup)
+	MOVQ R10, (DI)
+	MOVQ R11, 8(DI)
 	RET
 
-// ----------------------------------------------------------------------------
-// ------------------------- AVX2 Code ----------------------------------------
 chacha20Poly1305Seal_AVX2:
 	VZEROUPPER
-	VMOVDQU ·chacha20Constants<>(SB), AA0
-	BYTE    $0xc4; BYTE $0x42; BYTE $0x7d; BYTE $0x5a; BYTE $0x70; BYTE $0x10 // broadcasti128 16(r8), ymm14
-	BYTE    $0xc4; BYTE $0x42; BYTE $0x7d; BYTE $0x5a; BYTE $0x60; BYTE $0x20 // broadcasti128 32(r8), ymm12
-	BYTE    $0xc4; BYTE $0xc2; BYTE $0x7d; BYTE $0x5a; BYTE $0x60; BYTE $0x30 // broadcasti128 48(r8), ymm4
-	VPADDD  ·avx2InitMask<>(SB), DD0, DD0
+	VMOVDQU ·chacha20Constants<>+0(SB), Y0
+	BYTE    $0xc4
+	BYTE    $0x42
+	BYTE    $0x7d
+	BYTE    $0x5a
+	BYTE    $0x70
+	BYTE    $0x10
+	BYTE    $0xc4
+	BYTE    $0x42
+	BYTE    $0x7d
+	BYTE    $0x5a
+	BYTE    $0x60
+	BYTE    $0x20
+	BYTE    $0xc4
+	BYTE    $0xc2
+	BYTE    $0x7d
+	BYTE    $0x5a
+	BYTE    $0x60
+	BYTE    $0x30
+	VPADDD  ·avx2InitMask<>+0(SB), Y4, Y4
 
 	// Special optimizations, for very short buffers
-	CMPQ inl, $192
-	JBE  seal192AVX2 // 33% faster
-	CMPQ inl, $320
-	JBE  seal320AVX2 // 17% faster
+	CMPQ BX, $0x000000c0
+	JBE  seal192AVX2
+	CMPQ BX, $0x00000140
+	JBE  seal320AVX2
 
 	// For the general key prepare the key first - as a byproduct we have 64 bytes of cipher stream
-	VMOVDQA AA0, AA1; VMOVDQA AA0, AA2; VMOVDQA AA0, AA3
-	VMOVDQA BB0, BB1; VMOVDQA BB0, BB2; VMOVDQA BB0, BB3; VMOVDQA BB0, state1StoreAVX2
-	VMOVDQA CC0, CC1; VMOVDQA CC0, CC2; VMOVDQA CC0, CC3; VMOVDQA CC0, state2StoreAVX2
-	VPADDD  ·avx2IncMask<>(SB), DD0, DD1; VMOVDQA DD0, ctr0StoreAVX2
-	VPADDD  ·avx2IncMask<>(SB), DD1, DD2; VMOVDQA DD1, ctr1StoreAVX2
-	VPADDD  ·avx2IncMask<>(SB), DD2, DD3; VMOVDQA DD2, ctr2StoreAVX2
-	VMOVDQA DD3, ctr3StoreAVX2
-	MOVQ    $10, itr2
+	VMOVDQA Y0, Y5
+	VMOVDQA Y0, Y6
+	VMOVDQA Y0, Y7
+	VMOVDQA Y14, Y9
+	VMOVDQA Y14, Y10
+	VMOVDQA Y14, Y11
+	VMOVDQA Y14, 32(BP)
+	VMOVDQA Y12, Y13
+	VMOVDQA Y12, Y8
+	VMOVDQA Y12, Y15
+	VMOVDQA Y12, 64(BP)
+	VPADDD  ·avx2IncMask<>+0(SB), Y4, Y1
+	VMOVDQA Y4, 96(BP)
+	VPADDD  ·avx2IncMask<>+0(SB), Y1, Y2
+	VMOVDQA Y1, 128(BP)
+	VPADDD  ·avx2IncMask<>+0(SB), Y2, Y3
+	VMOVDQA Y2, 160(BP)
+	VMOVDQA Y3, 192(BP)
+	MOVQ    $0x0000000a, R9
 
 sealAVX2IntroLoop:
-	VMOVDQA CC3, tmpStoreAVX2
-	chachaQR_AVX2(AA0, BB0, CC0, DD0, CC3); chachaQR_AVX2(AA1, BB1, CC1, DD1, CC3); chachaQR_AVX2(AA2, BB2, CC2, DD2, CC3)
-	VMOVDQA tmpStoreAVX2, CC3
-	VMOVDQA CC1, tmpStoreAVX2
-	chachaQR_AVX2(AA3, BB3, CC3, DD3, CC1)
-	VMOVDQA tmpStoreAVX2, CC1
-
-	VPALIGNR $4, BB0, BB0, BB0; VPALIGNR $8, CC0, CC0, CC0; VPALIGNR $12, DD0, DD0, DD0
-	VPALIGNR $4, BB1, BB1, BB1; VPALIGNR $8, CC1, CC1, CC1; VPALIGNR $12, DD1, DD1, DD1
-	VPALIGNR $4, BB2, BB2, BB2; VPALIGNR $8, CC2, CC2, CC2; VPALIGNR $12, DD2, DD2, DD2
-	VPALIGNR $4, BB3, BB3, BB3; VPALIGNR $8, CC3, CC3, CC3; VPALIGNR $12, DD3, DD3, DD3
-
-	VMOVDQA CC3, tmpStoreAVX2
-	chachaQR_AVX2(AA0, BB0, CC0, DD0, CC3); chachaQR_AVX2(AA1, BB1, CC1, DD1, CC3); chachaQR_AVX2(AA2, BB2, CC2, DD2, CC3)
-	VMOVDQA tmpStoreAVX2, CC3
-	VMOVDQA CC1, tmpStoreAVX2
-	chachaQR_AVX2(AA3, BB3, CC3, DD3, CC1)
-	VMOVDQA tmpStoreAVX2, CC1
-
-	VPALIGNR $12, BB0, BB0, BB0; VPALIGNR $8, CC0, CC0, CC0; VPALIGNR $4, DD0, DD0, DD0
-	VPALIGNR $12, BB1, BB1, BB1; VPALIGNR $8, CC1, CC1, CC1; VPALIGNR $4, DD1, DD1, DD1
-	VPALIGNR $12, BB2, BB2, BB2; VPALIGNR $8, CC2, CC2, CC2; VPALIGNR $4, DD2, DD2, DD2
-	VPALIGNR $12, BB3, BB3, BB3; VPALIGNR $8, CC3, CC3, CC3; VPALIGNR $4, DD3, DD3, DD3
-	DECQ     itr2
-	JNE      sealAVX2IntroLoop
-
-	VPADDD ·chacha20Constants<>(SB), AA0, AA0; VPADDD ·chacha20Constants<>(SB), AA1, AA1; VPADDD ·chacha20Constants<>(SB), AA2, AA2; VPADDD ·chacha20Constants<>(SB), AA3, AA3
-	VPADDD state1StoreAVX2, BB0, BB0; VPADDD state1StoreAVX2, BB1, BB1; VPADDD state1StoreAVX2, BB2, BB2; VPADDD state1StoreAVX2, BB3, BB3
-	VPADDD state2StoreAVX2, CC0, CC0; VPADDD state2StoreAVX2, CC1, CC1; VPADDD state2StoreAVX2, CC2, CC2; VPADDD state2StoreAVX2, CC3, CC3
-	VPADDD ctr0StoreAVX2, DD0, DD0; VPADDD ctr1StoreAVX2, DD1, DD1; VPADDD ctr2StoreAVX2, DD2, DD2; VPADDD ctr3StoreAVX2, DD3, DD3
-
-	VPERM2I128 $0x13, CC0, DD0, CC0 // Stream bytes 96 - 127
-	VPERM2I128 $0x02, AA0, BB0, DD0 // The Poly1305 key
-	VPERM2I128 $0x13, AA0, BB0, AA0 // Stream bytes 64 - 95
+	VMOVDQA    Y15, 224(BP)
+	VPADDD     Y14, Y0, Y0
+	VPXOR      Y0, Y4, Y4
+	VPSHUFB    ·rol16<>+0(SB), Y4, Y4
+	VPADDD     Y4, Y12, Y12
+	VPXOR      Y12, Y14, Y14
+	VPSLLD     $0x0c, Y14, Y15
+	VPSRLD     $0x14, Y14, Y14
+	VPXOR      Y15, Y14, Y14
+	VPADDD     Y14, Y0, Y0
+	VPXOR      Y0, Y4, Y4
+	VPSHUFB    ·rol8<>+0(SB), Y4, Y4
+	VPADDD     Y4, Y12, Y12
+	VPXOR      Y12, Y14, Y14
+	VPSLLD     $0x07, Y14, Y15
+	VPSRLD     $0x19, Y14, Y14
+	VPXOR      Y15, Y14, Y14
+	VPADDD     Y9, Y5, Y5
+	VPXOR      Y5, Y1, Y1
+	VPSHUFB    ·rol16<>+0(SB), Y1, Y1
+	VPADDD     Y1, Y13, Y13
+	VPXOR      Y13, Y9, Y9
+	VPSLLD     $0x0c, Y9, Y15
+	VPSRLD     $0x14, Y9, Y9
+	VPXOR      Y15, Y9, Y9
+	VPADDD     Y9, Y5, Y5
+	VPXOR      Y5, Y1, Y1
+	VPSHUFB    ·rol8<>+0(SB), Y1, Y1
+	VPADDD     Y1, Y13, Y13
+	VPXOR      Y13, Y9, Y9
+	VPSLLD     $0x07, Y9, Y15
+	VPSRLD     $0x19, Y9, Y9
+	VPXOR      Y15, Y9, Y9
+	VPADDD     Y10, Y6, Y6
+	VPXOR      Y6, Y2, Y2
+	VPSHUFB    ·rol16<>+0(SB), Y2, Y2
+	VPADDD     Y2, Y8, Y8
+	VPXOR      Y8, Y10, Y10
+	VPSLLD     $0x0c, Y10, Y15
+	VPSRLD     $0x14, Y10, Y10
+	VPXOR      Y15, Y10, Y10
+	VPADDD     Y10, Y6, Y6
+	VPXOR      Y6, Y2, Y2
+	VPSHUFB    ·rol8<>+0(SB), Y2, Y2
+	VPADDD     Y2, Y8, Y8
+	VPXOR      Y8, Y10, Y10
+	VPSLLD     $0x07, Y10, Y15
+	VPSRLD     $0x19, Y10, Y10
+	VPXOR      Y15, Y10, Y10
+	VMOVDQA    224(BP), Y15
+	VMOVDQA    Y13, 224(BP)
+	VPADDD     Y11, Y7, Y7
+	VPXOR      Y7, Y3, Y3
+	VPSHUFB    ·rol16<>+0(SB), Y3, Y3
+	VPADDD     Y3, Y15, Y15
+	VPXOR      Y15, Y11, Y11
+	VPSLLD     $0x0c, Y11, Y13
+	VPSRLD     $0x14, Y11, Y11
+	VPXOR      Y13, Y11, Y11
+	VPADDD     Y11, Y7, Y7
+	VPXOR      Y7, Y3, Y3
+	VPSHUFB    ·rol8<>+0(SB), Y3, Y3
+	VPADDD     Y3, Y15, Y15
+	VPXOR      Y15, Y11, Y11
+	VPSLLD     $0x07, Y11, Y13
+	VPSRLD     $0x19, Y11, Y11
+	VPXOR      Y13, Y11, Y11
+	VMOVDQA    224(BP), Y13
+	VPALIGNR   $0x04, Y14, Y14, Y14
+	VPALIGNR   $0x08, Y12, Y12, Y12
+	VPALIGNR   $0x0c, Y4, Y4, Y4
+	VPALIGNR   $0x04, Y9, Y9, Y9
+	VPALIGNR   $0x08, Y13, Y13, Y13
+	VPALIGNR   $0x0c, Y1, Y1, Y1
+	VPALIGNR   $0x04, Y10, Y10, Y10
+	VPALIGNR   $0x08, Y8, Y8, Y8
+	VPALIGNR   $0x0c, Y2, Y2, Y2
+	VPALIGNR   $0x04, Y11, Y11, Y11
+	VPALIGNR   $0x08, Y15, Y15, Y15
+	VPALIGNR   $0x0c, Y3, Y3, Y3
+	VMOVDQA    Y15, 224(BP)
+	VPADDD     Y14, Y0, Y0
+	VPXOR      Y0, Y4, Y4
+	VPSHUFB    ·rol16<>+0(SB), Y4, Y4
+	VPADDD     Y4, Y12, Y12
+	VPXOR      Y12, Y14, Y14
+	VPSLLD     $0x0c, Y14, Y15
+	VPSRLD     $0x14, Y14, Y14
+	VPXOR      Y15, Y14, Y14
+	VPADDD     Y14, Y0, Y0
+	VPXOR      Y0, Y4, Y4
+	VPSHUFB    ·rol8<>+0(SB), Y4, Y4
+	VPADDD     Y4, Y12, Y12
+	VPXOR      Y12, Y14, Y14
+	VPSLLD     $0x07, Y14, Y15
+	VPSRLD     $0x19, Y14, Y14
+	VPXOR      Y15, Y14, Y14
+	VPADDD     Y9, Y5, Y5
+	VPXOR      Y5, Y1, Y1
+	VPSHUFB    ·rol16<>+0(SB), Y1, Y1
+	VPADDD     Y1, Y13, Y13
+	VPXOR      Y13, Y9, Y9
+	VPSLLD     $0x0c, Y9, Y15
+	VPSRLD     $0x14, Y9, Y9
+	VPXOR      Y15, Y9, Y9
+	VPADDD     Y9, Y5, Y5
+	VPXOR      Y5, Y1, Y1
+	VPSHUFB    ·rol8<>+0(SB), Y1, Y1
+	VPADDD     Y1, Y13, Y13
+	VPXOR      Y13, Y9, Y9
+	VPSLLD     $0x07, Y9, Y15
+	VPSRLD     $0x19, Y9, Y9
+	VPXOR      Y15, Y9, Y9
+	VPADDD     Y10, Y6, Y6
+	VPXOR      Y6, Y2, Y2
+	VPSHUFB    ·rol16<>+0(SB), Y2, Y2
+	VPADDD     Y2, Y8, Y8
+	VPXOR      Y8, Y10, Y10
+	VPSLLD     $0x0c, Y10, Y15
+	VPSRLD     $0x14, Y10, Y10
+	VPXOR      Y15, Y10, Y10
+	VPADDD     Y10, Y6, Y6
+	VPXOR      Y6, Y2, Y2
+	VPSHUFB    ·rol8<>+0(SB), Y2, Y2
+	VPADDD     Y2, Y8, Y8
+	VPXOR      Y8, Y10, Y10
+	VPSLLD     $0x07, Y10, Y15
+	VPSRLD     $0x19, Y10, Y10
+	VPXOR      Y15, Y10, Y10
+	VMOVDQA    224(BP), Y15
+	VMOVDQA    Y13, 224(BP)
+	VPADDD     Y11, Y7, Y7
+	VPXOR      Y7, Y3, Y3
+	VPSHUFB    ·rol16<>+0(SB), Y3, Y3
+	VPADDD     Y3, Y15, Y15
+	VPXOR      Y15, Y11, Y11
+	VPSLLD     $0x0c, Y11, Y13
+	VPSRLD     $0x14, Y11, Y11
+	VPXOR      Y13, Y11, Y11
+	VPADDD     Y11, Y7, Y7
+	VPXOR      Y7, Y3, Y3
+	VPSHUFB    ·rol8<>+0(SB), Y3, Y3
+	VPADDD     Y3, Y15, Y15
+	VPXOR      Y15, Y11, Y11
+	VPSLLD     $0x07, Y11, Y13
+	VPSRLD     $0x19, Y11, Y11
+	VPXOR      Y13, Y11, Y11
+	VMOVDQA    224(BP), Y13
+	VPALIGNR   $0x0c, Y14, Y14, Y14
+	VPALIGNR   $0x08, Y12, Y12, Y12
+	VPALIGNR   $0x04, Y4, Y4, Y4
+	VPALIGNR   $0x0c, Y9, Y9, Y9
+	VPALIGNR   $0x08, Y13, Y13, Y13
+	VPALIGNR   $0x04, Y1, Y1, Y1
+	VPALIGNR   $0x0c, Y10, Y10, Y10
+	VPALIGNR   $0x08, Y8, Y8, Y8
+	VPALIGNR   $0x04, Y2, Y2, Y2
+	VPALIGNR   $0x0c, Y11, Y11, Y11
+	VPALIGNR   $0x08, Y15, Y15, Y15
+	VPALIGNR   $0x04, Y3, Y3, Y3
+	DECQ       R9
+	JNE        sealAVX2IntroLoop
+	VPADDD     ·chacha20Constants<>+0(SB), Y0, Y0
+	VPADDD     ·chacha20Constants<>+0(SB), Y5, Y5
+	VPADDD     ·chacha20Constants<>+0(SB), Y6, Y6
+	VPADDD     ·chacha20Constants<>+0(SB), Y7, Y7
+	VPADDD     32(BP), Y14, Y14
+	VPADDD     32(BP), Y9, Y9
+	VPADDD     32(BP), Y10, Y10
+	VPADDD     32(BP), Y11, Y11
+	VPADDD     64(BP), Y12, Y12
+	VPADDD     64(BP), Y13, Y13
+	VPADDD     64(BP), Y8, Y8
+	VPADDD     64(BP), Y15, Y15
+	VPADDD     96(BP), Y4, Y4
+	VPADDD     128(BP), Y1, Y1
+	VPADDD     160(BP), Y2, Y2
+	VPADDD     192(BP), Y3, Y3
+	VPERM2I128 $0x13, Y12, Y4, Y12
+	VPERM2I128 $0x02, Y0, Y14, Y4
+	VPERM2I128 $0x13, Y0, Y14, Y0
 
 	// Clamp and store poly key
-	VPAND   ·polyClampMask<>(SB), DD0, DD0
-	VMOVDQA DD0, rsStoreAVX2
+	VPAND   ·polyClampMask<>+0(SB), Y4, Y4
+	VMOVDQA Y4, (BP)
 
 	// Hash AD
-	MOVQ ad_len+80(FP), itr2
+	MOVQ ad_len+80(FP), R9
 	CALL polyHashADInternal<>(SB)
 
 	// Can store at least 320 bytes
-	VPXOR   (0*32)(inp), AA0, AA0
-	VPXOR   (1*32)(inp), CC0, CC0
-	VMOVDQU AA0, (0*32)(oup)
-	VMOVDQU CC0, (1*32)(oup)
-
-	VPERM2I128 $0x02, AA1, BB1, AA0; VPERM2I128 $0x02, CC1, DD1, BB0; VPERM2I128 $0x13, AA1, BB1, CC0; VPERM2I128 $0x13, CC1, DD1, DD0
-	VPXOR      (2*32)(inp), AA0, AA0; VPXOR (3*32)(inp), BB0, BB0; VPXOR (4*32)(inp), CC0, CC0; VPXOR (5*32)(inp), DD0, DD0
-	VMOVDQU    AA0, (2*32)(oup); VMOVDQU BB0, (3*32)(oup); VMOVDQU CC0, (4*32)(oup); VMOVDQU DD0, (5*32)(oup)
-	VPERM2I128 $0x02, AA2, BB2, AA0; VPERM2I128 $0x02, CC2, DD2, BB0; VPERM2I128 $0x13, AA2, BB2, CC0; VPERM2I128 $0x13, CC2, DD2, DD0
-	VPXOR      (6*32)(inp), AA0, AA0; VPXOR (7*32)(inp), BB0, BB0; VPXOR (8*32)(inp), CC0, CC0; VPXOR (9*32)(inp), DD0, DD0
-	VMOVDQU    AA0, (6*32)(oup); VMOVDQU BB0, (7*32)(oup); VMOVDQU CC0, (8*32)(oup); VMOVDQU DD0, (9*32)(oup)
-
-	MOVQ $320, itr1
-	SUBQ $320, inl
-	LEAQ 320(inp), inp
-
-	VPERM2I128 $0x02, AA3, BB3, AA0; VPERM2I128 $0x02, CC3, DD3, BB0; VPERM2I128 $0x13, AA3, BB3, CC0; VPERM2I128 $0x13, CC3, DD3, DD0
-	CMPQ       inl, $128
+	VPXOR      (SI), Y0, Y0
+	VPXOR      32(SI), Y12, Y12
+	VMOVDQU    Y0, (DI)
+	VMOVDQU    Y12, 32(DI)
+	VPERM2I128 $0x02, Y5, Y9, Y0
+	VPERM2I128 $0x02, Y13, Y1, Y14
+	VPERM2I128 $0x13, Y5, Y9, Y12
+	VPERM2I128 $0x13, Y13, Y1, Y4
+	VPXOR      64(SI), Y0, Y0
+	VPXOR      96(SI), Y14, Y14
+	VPXOR      128(SI), Y12, Y12
+	VPXOR      160(SI), Y4, Y4
+	VMOVDQU    Y0, 64(DI)
+	VMOVDQU    Y14, 96(DI)
+	VMOVDQU    Y12, 128(DI)
+	VMOVDQU    Y4, 160(DI)
+	VPERM2I128 $0x02, Y6, Y10, Y0
+	VPERM2I128 $0x02, Y8, Y2, Y14
+	VPERM2I128 $0x13, Y6, Y10, Y12
+	VPERM2I128 $0x13, Y8, Y2, Y4
+	VPXOR      192(SI), Y0, Y0
+	VPXOR      224(SI), Y14, Y14
+	VPXOR      256(SI), Y12, Y12
+	VPXOR      288(SI), Y4, Y4
+	VMOVDQU    Y0, 192(DI)
+	VMOVDQU    Y14, 224(DI)
+	VMOVDQU    Y12, 256(DI)
+	VMOVDQU    Y4, 288(DI)
+	MOVQ       $0x00000140, CX
+	SUBQ       $0x00000140, BX
+	LEAQ       320(SI), SI
+	VPERM2I128 $0x02, Y7, Y11, Y0
+	VPERM2I128 $0x02, Y15, Y3, Y14
+	VPERM2I128 $0x13, Y7, Y11, Y12
+	VPERM2I128 $0x13, Y15, Y3, Y4
+	CMPQ       BX, $0x80
 	JBE        sealAVX2SealHash
-
-	VPXOR   (0*32)(inp), AA0, AA0; VPXOR (1*32)(inp), BB0, BB0; VPXOR (2*32)(inp), CC0, CC0; VPXOR (3*32)(inp), DD0, DD0
-	VMOVDQU AA0, (10*32)(oup); VMOVDQU BB0, (11*32)(oup); VMOVDQU CC0, (12*32)(oup); VMOVDQU DD0, (13*32)(oup)
-	SUBQ    $128, inl
-	LEAQ    128(inp), inp
-
-	MOVQ $8, itr1
-	MOVQ $2, itr2
-
-	CMPQ inl, $128
-	JBE  sealAVX2Tail128
-	CMPQ inl, $256
-	JBE  sealAVX2Tail256
-	CMPQ inl, $384
-	JBE  sealAVX2Tail384
-	CMPQ inl, $512
-	JBE  sealAVX2Tail512
+	VPXOR      (SI), Y0, Y0
+	VPXOR      32(SI), Y14, Y14
+	VPXOR      64(SI), Y12, Y12
+	VPXOR      96(SI), Y4, Y4
+	VMOVDQU    Y0, 320(DI)
+	VMOVDQU    Y14, 352(DI)
+	VMOVDQU    Y12, 384(DI)
+	VMOVDQU    Y4, 416(DI)
+	SUBQ       $0x80, BX
+	LEAQ       128(SI), SI
+	MOVQ       $0x00000008, CX
+	MOVQ       $0x00000002, R9
+	CMPQ       BX, $0x80
+	JBE        sealAVX2Tail128
+	CMPQ       BX, $0x00000100
+	JBE        sealAVX2Tail256
+	CMPQ       BX, $0x00000180
+	JBE        sealAVX2Tail384
+	CMPQ       BX, $0x00000200
+	JBE        sealAVX2Tail512
 
 	// We have 448 bytes to hash, but main loop hashes 512 bytes at a time - perform some rounds, before the main loop
-	VMOVDQA ·chacha20Constants<>(SB), AA0; VMOVDQA AA0, AA1; VMOVDQA AA0, AA2; VMOVDQA AA0, AA3
-	VMOVDQA state1StoreAVX2, BB0; VMOVDQA BB0, BB1; VMOVDQA BB0, BB2; VMOVDQA BB0, BB3
-	VMOVDQA state2StoreAVX2, CC0; VMOVDQA CC0, CC1; VMOVDQA CC0, CC2; VMOVDQA CC0, CC3
-	VMOVDQA ctr3StoreAVX2, DD0
-	VPADDD  ·avx2IncMask<>(SB), DD0, DD0; VPADDD ·avx2IncMask<>(SB), DD0, DD1; VPADDD ·avx2IncMask<>(SB), DD1, DD2; VPADDD ·avx2IncMask<>(SB), DD2, DD3
-	VMOVDQA DD0, ctr0StoreAVX2; VMOVDQA DD1, ctr1StoreAVX2; VMOVDQA DD2, ctr2StoreAVX2; VMOVDQA DD3, ctr3StoreAVX2
-
-	VMOVDQA CC3, tmpStoreAVX2
-	chachaQR_AVX2(AA0, BB0, CC0, DD0, CC3); chachaQR_AVX2(AA1, BB1, CC1, DD1, CC3); chachaQR_AVX2(AA2, BB2, CC2, DD2, CC3)
-	VMOVDQA tmpStoreAVX2, CC3
-	VMOVDQA CC1, tmpStoreAVX2
-	chachaQR_AVX2(AA3, BB3, CC3, DD3, CC1)
-	VMOVDQA tmpStoreAVX2, CC1
-
-	VPALIGNR $4, BB0, BB0, BB0; VPALIGNR $8, CC0, CC0, CC0; VPALIGNR $12, DD0, DD0, DD0
-	VPALIGNR $4, BB1, BB1, BB1; VPALIGNR $8, CC1, CC1, CC1; VPALIGNR $12, DD1, DD1, DD1
-	VPALIGNR $4, BB2, BB2, BB2; VPALIGNR $8, CC2, CC2, CC2; VPALIGNR $12, DD2, DD2, DD2
-	VPALIGNR $4, BB3, BB3, BB3; VPALIGNR $8, CC3, CC3, CC3; VPALIGNR $12, DD3, DD3, DD3
-
-	VMOVDQA CC3, tmpStoreAVX2
-	chachaQR_AVX2(AA0, BB0, CC0, DD0, CC3); chachaQR_AVX2(AA1, BB1, CC1, DD1, CC3); chachaQR_AVX2(AA2, BB2, CC2, DD2, CC3)
-	VMOVDQA tmpStoreAVX2, CC3
-	VMOVDQA CC1, tmpStoreAVX2
-	chachaQR_AVX2(AA3, BB3, CC3, DD3, CC1)
-	VMOVDQA tmpStoreAVX2, CC1
-
-	VPALIGNR $12, BB0, BB0, BB0; VPALIGNR $8, CC0, CC0, CC0; VPALIGNR $4, DD0, DD0, DD0
-	VPALIGNR $12, BB1, BB1, BB1; VPALIGNR $8, CC1, CC1, CC1; VPALIGNR $4, DD1, DD1, DD1
-	VPALIGNR $12, BB2, BB2, BB2; VPALIGNR $8, CC2, CC2, CC2; VPALIGNR $4, DD2, DD2, DD2
-	VPALIGNR $12, BB3, BB3, BB3; VPALIGNR $8, CC3, CC3, CC3; VPALIGNR $4, DD3, DD3, DD3
-	VPADDD   BB0, AA0, AA0; VPADDD BB1, AA1, AA1; VPADDD BB2, AA2, AA2; VPADDD BB3, AA3, AA3
-	VPXOR    AA0, DD0, DD0; VPXOR AA1, DD1, DD1; VPXOR AA2, DD2, DD2; VPXOR AA3, DD3, DD3
-	VPSHUFB  ·rol16<>(SB), DD0, DD0; VPSHUFB ·rol16<>(SB), DD1, DD1; VPSHUFB ·rol16<>(SB), DD2, DD2; VPSHUFB ·rol16<>(SB), DD3, DD3
-	VPADDD   DD0, CC0, CC0; VPADDD DD1, CC1, CC1; VPADDD DD2, CC2, CC2; VPADDD DD3, CC3, CC3
-	VPXOR    CC0, BB0, BB0; VPXOR CC1, BB1, BB1; VPXOR CC2, BB2, BB2; VPXOR CC3, BB3, BB3
-	VMOVDQA  CC3, tmpStoreAVX2
-	VPSLLD   $12, BB0, CC3; VPSRLD $20, BB0, BB0; VPXOR CC3, BB0, BB0
-	VPSLLD   $12, BB1, CC3; VPSRLD $20, BB1, BB1; VPXOR CC3, BB1, BB1
-	VPSLLD   $12, BB2, CC3; VPSRLD $20, BB2, BB2; VPXOR CC3, BB2, BB2
-	VPSLLD   $12, BB3, CC3; VPSRLD $20, BB3, BB3; VPXOR CC3, BB3, BB3
-	VMOVDQA  tmpStoreAVX2, CC3
-
-	SUBQ $16, oup                  // Adjust the pointer
-	MOVQ $9, itr1
-	JMP  sealAVX2InternalLoopStart
+	VMOVDQA  ·chacha20Constants<>+0(SB), Y0
+	VMOVDQA  Y0, Y5
+	VMOVDQA  Y0, Y6
+	VMOVDQA  Y0, Y7
+	VMOVDQA  32(BP), Y14
+	VMOVDQA  Y14, Y9
+	VMOVDQA  Y14, Y10
+	VMOVDQA  Y14, Y11
+	VMOVDQA  64(BP), Y12
+	VMOVDQA  Y12, Y13
+	VMOVDQA  Y12, Y8
+	VMOVDQA  Y12, Y15
+	VMOVDQA  192(BP), Y4
+	VPADDD   ·avx2IncMask<>+0(SB), Y4, Y4
+	VPADDD   ·avx2IncMask<>+0(SB), Y4, Y1
+	VPADDD   ·avx2IncMask<>+0(SB), Y1, Y2
+	VPADDD   ·avx2IncMask<>+0(SB), Y2, Y3
+	VMOVDQA  Y4, 96(BP)
+	VMOVDQA  Y1, 128(BP)
+	VMOVDQA  Y2, 160(BP)
+	VMOVDQA  Y3, 192(BP)
+	VMOVDQA  Y15, 224(BP)
+	VPADDD   Y14, Y0, Y0
+	VPXOR    Y0, Y4, Y4
+	VPSHUFB  ·rol16<>+0(SB), Y4, Y4
+	VPADDD   Y4, Y12, Y12
+	VPXOR    Y12, Y14, Y14
+	VPSLLD   $0x0c, Y14, Y15
+	VPSRLD   $0x14, Y14, Y14
+	VPXOR    Y15, Y14, Y14
+	VPADDD   Y14, Y0, Y0
+	VPXOR    Y0, Y4, Y4
+	VPSHUFB  ·rol8<>+0(SB), Y4, Y4
+	VPADDD   Y4, Y12, Y12
+	VPXOR    Y12, Y14, Y14
+	VPSLLD   $0x07, Y14, Y15
+	VPSRLD   $0x19, Y14, Y14
+	VPXOR    Y15, Y14, Y14
+	VPADDD   Y9, Y5, Y5
+	VPXOR    Y5, Y1, Y1
+	VPSHUFB  ·rol16<>+0(SB), Y1, Y1
+	VPADDD   Y1, Y13, Y13
+	VPXOR    Y13, Y9, Y9
+	VPSLLD   $0x0c, Y9, Y15
+	VPSRLD   $0x14, Y9, Y9
+	VPXOR    Y15, Y9, Y9
+	VPADDD   Y9, Y5, Y5
+	VPXOR    Y5, Y1, Y1
+	VPSHUFB  ·rol8<>+0(SB), Y1, Y1
+	VPADDD   Y1, Y13, Y13
+	VPXOR    Y13, Y9, Y9
+	VPSLLD   $0x07, Y9, Y15
+	VPSRLD   $0x19, Y9, Y9
+	VPXOR    Y15, Y9, Y9
+	VPADDD   Y10, Y6, Y6
+	VPXOR    Y6, Y2, Y2
+	VPSHUFB  ·rol16<>+0(SB), Y2, Y2
+	VPADDD   Y2, Y8, Y8
+	VPXOR    Y8, Y10, Y10
+	VPSLLD   $0x0c, Y10, Y15
+	VPSRLD   $0x14, Y10, Y10
+	VPXOR    Y15, Y10, Y10
+	VPADDD   Y10, Y6, Y6
+	VPXOR    Y6, Y2, Y2
+	VPSHUFB  ·rol8<>+0(SB), Y2, Y2
+	VPADDD   Y2, Y8, Y8
+	VPXOR    Y8, Y10, Y10
+	VPSLLD   $0x07, Y10, Y15
+	VPSRLD   $0x19, Y10, Y10
+	VPXOR    Y15, Y10, Y10
+	VMOVDQA  224(BP), Y15
+	VMOVDQA  Y13, 224(BP)
+	VPADDD   Y11, Y7, Y7
+	VPXOR    Y7, Y3, Y3
+	VPSHUFB  ·rol16<>+0(SB), Y3, Y3
+	VPADDD   Y3, Y15, Y15
+	VPXOR    Y15, Y11, Y11
+	VPSLLD   $0x0c, Y11, Y13
+	VPSRLD   $0x14, Y11, Y11
+	VPXOR    Y13, Y11, Y11
+	VPADDD   Y11, Y7, Y7
+	VPXOR    Y7, Y3, Y3
+	VPSHUFB  ·rol8<>+0(SB), Y3, Y3
+	VPADDD   Y3, Y15, Y15
+	VPXOR    Y15, Y11, Y11
+	VPSLLD   $0x07, Y11, Y13
+	VPSRLD   $0x19, Y11, Y11
+	VPXOR    Y13, Y11, Y11
+	VMOVDQA  224(BP), Y13
+	VPALIGNR $0x04, Y14, Y14, Y14
+	VPALIGNR $0x08, Y12, Y12, Y12
+	VPALIGNR $0x0c, Y4, Y4, Y4
+	VPALIGNR $0x04, Y9, Y9, Y9
+	VPALIGNR $0x08, Y13, Y13, Y13
+	VPALIGNR $0x0c, Y1, Y1, Y1
+	VPALIGNR $0x04, Y10, Y10, Y10
+	VPALIGNR $0x08, Y8, Y8, Y8
+	VPALIGNR $0x0c, Y2, Y2, Y2
+	VPALIGNR $0x04, Y11, Y11, Y11
+	VPALIGNR $0x08, Y15, Y15, Y15
+	VPALIGNR $0x0c, Y3, Y3, Y3
+	VMOVDQA  Y15, 224(BP)
+	VPADDD   Y14, Y0, Y0
+	VPXOR    Y0, Y4, Y4
+	VPSHUFB  ·rol16<>+0(SB), Y4, Y4
+	VPADDD   Y4, Y12, Y12
+	VPXOR    Y12, Y14, Y14
+	VPSLLD   $0x0c, Y14, Y15
+	VPSRLD   $0x14, Y14, Y14
+	VPXOR    Y15, Y14, Y14
+	VPADDD   Y14, Y0, Y0
+	VPXOR    Y0, Y4, Y4
+	VPSHUFB  ·rol8<>+0(SB), Y4, Y4
+	VPADDD   Y4, Y12, Y12
+	VPXOR    Y12, Y14, Y14
+	VPSLLD   $0x07, Y14, Y15
+	VPSRLD   $0x19, Y14, Y14
+	VPXOR    Y15, Y14, Y14
+	VPADDD   Y9, Y5, Y5
+	VPXOR    Y5, Y1, Y1
+	VPSHUFB  ·rol16<>+0(SB), Y1, Y1
+	VPADDD   Y1, Y13, Y13
+	VPXOR    Y13, Y9, Y9
+	VPSLLD   $0x0c, Y9, Y15
+	VPSRLD   $0x14, Y9, Y9
+	VPXOR    Y15, Y9, Y9
+	VPADDD   Y9, Y5, Y5
+	VPXOR    Y5, Y1, Y1
+	VPSHUFB  ·rol8<>+0(SB), Y1, Y1
+	VPADDD   Y1, Y13, Y13
+	VPXOR    Y13, Y9, Y9
+	VPSLLD   $0x07, Y9, Y15
+	VPSRLD   $0x19, Y9, Y9
+	VPXOR    Y15, Y9, Y9
+	VPADDD   Y10, Y6, Y6
+	VPXOR    Y6, Y2, Y2
+	VPSHUFB  ·rol16<>+0(SB), Y2, Y2
+	VPADDD   Y2, Y8, Y8
+	VPXOR    Y8, Y10, Y10
+	VPSLLD   $0x0c, Y10, Y15
+	VPSRLD   $0x14, Y10, Y10
+	VPXOR    Y15, Y10, Y10
+	VPADDD   Y10, Y6, Y6
+	VPXOR    Y6, Y2, Y2
+	VPSHUFB  ·rol8<>+0(SB), Y2, Y2
+	VPADDD   Y2, Y8, Y8
+	VPXOR    Y8, Y10, Y10
+	VPSLLD   $0x07, Y10, Y15
+	VPSRLD   $0x19, Y10, Y10
+	VPXOR    Y15, Y10, Y10
+	VMOVDQA  224(BP), Y15
+	VMOVDQA  Y13, 224(BP)
+	VPADDD   Y11, Y7, Y7
+	VPXOR    Y7, Y3, Y3
+	VPSHUFB  ·rol16<>+0(SB), Y3, Y3
+	VPADDD   Y3, Y15, Y15
+	VPXOR    Y15, Y11, Y11
+	VPSLLD   $0x0c, Y11, Y13
+	VPSRLD   $0x14, Y11, Y11
+	VPXOR    Y13, Y11, Y11
+	VPADDD   Y11, Y7, Y7
+	VPXOR    Y7, Y3, Y3
+	VPSHUFB  ·rol8<>+0(SB), Y3, Y3
+	VPADDD   Y3, Y15, Y15
+	VPXOR    Y15, Y11, Y11
+	VPSLLD   $0x07, Y11, Y13
+	VPSRLD   $0x19, Y11, Y11
+	VPXOR    Y13, Y11, Y11
+	VMOVDQA  224(BP), Y13
+	VPALIGNR $0x0c, Y14, Y14, Y14
+	VPALIGNR $0x08, Y12, Y12, Y12
+	VPALIGNR $0x04, Y4, Y4, Y4
+	VPALIGNR $0x0c, Y9, Y9, Y9
+	VPALIGNR $0x08, Y13, Y13, Y13
+	VPALIGNR $0x04, Y1, Y1, Y1
+	VPALIGNR $0x0c, Y10, Y10, Y10
+	VPALIGNR $0x08, Y8, Y8, Y8
+	VPALIGNR $0x04, Y2, Y2, Y2
+	VPALIGNR $0x0c, Y11, Y11, Y11
+	VPALIGNR $0x08, Y15, Y15, Y15
+	VPALIGNR $0x04, Y3, Y3, Y3
+	VPADDD   Y14, Y0, Y0
+	VPADDD   Y9, Y5, Y5
+	VPADDD   Y10, Y6, Y6
+	VPADDD   Y11, Y7, Y7
+	VPXOR    Y0, Y4, Y4
+	VPXOR    Y5, Y1, Y1
+	VPXOR    Y6, Y2, Y2
+	VPXOR    Y7, Y3, Y3
+	VPSHUFB  ·rol16<>+0(SB), Y4, Y4
+	VPSHUFB  ·rol16<>+0(SB), Y1, Y1
+	VPSHUFB  ·rol16<>+0(SB), Y2, Y2
+	VPSHUFB  ·rol16<>+0(SB), Y3, Y3
+	VPADDD   Y4, Y12, Y12
+	VPADDD   Y1, Y13, Y13
+	VPADDD   Y2, Y8, Y8
+	VPADDD   Y3, Y15, Y15
+	VPXOR    Y12, Y14, Y14
+	VPXOR    Y13, Y9, Y9
+	VPXOR    Y8, Y10, Y10
+	VPXOR    Y15, Y11, Y11
+	VMOVDQA  Y15, 224(BP)
+	VPSLLD   $0x0c, Y14, Y15
+	VPSRLD   $0x14, Y14, Y14
+	VPXOR    Y15, Y14, Y14
+	VPSLLD   $0x0c, Y9, Y15
+	VPSRLD   $0x14, Y9, Y9
+	VPXOR    Y15, Y9, Y9
+	VPSLLD   $0x0c, Y10, Y15
+	VPSRLD   $0x14, Y10, Y10
+	VPXOR    Y15, Y10, Y10
+	VPSLLD   $0x0c, Y11, Y15
+	VPSRLD   $0x14, Y11, Y11
+	VPXOR    Y15, Y11, Y11
+	VMOVDQA  224(BP), Y15
+	SUBQ     $0x10, DI
+	MOVQ     $0x00000009, CX
+	JMP      sealAVX2InternalLoopStart
 
 sealAVX2MainLoop:
-	// Load state, increment counter blocks, store the incremented counters
-	VMOVDQU ·chacha20Constants<>(SB), AA0; VMOVDQA AA0, AA1; VMOVDQA AA0, AA2; VMOVDQA AA0, AA3
-	VMOVDQA state1StoreAVX2, BB0; VMOVDQA BB0, BB1; VMOVDQA BB0, BB2; VMOVDQA BB0, BB3
-	VMOVDQA state2StoreAVX2, CC0; VMOVDQA CC0, CC1; VMOVDQA CC0, CC2; VMOVDQA CC0, CC3
-	VMOVDQA ctr3StoreAVX2, DD0; VPADDD ·avx2IncMask<>(SB), DD0, DD0; VPADDD ·avx2IncMask<>(SB), DD0, DD1; VPADDD ·avx2IncMask<>(SB), DD1, DD2; VPADDD ·avx2IncMask<>(SB), DD2, DD3
-	VMOVDQA DD0, ctr0StoreAVX2; VMOVDQA DD1, ctr1StoreAVX2; VMOVDQA DD2, ctr2StoreAVX2; VMOVDQA DD3, ctr3StoreAVX2
-	MOVQ    $10, itr1
+	VMOVDQU ·chacha20Constants<>+0(SB), Y0
+	VMOVDQA Y0, Y5
+	VMOVDQA Y0, Y6
+	VMOVDQA Y0, Y7
+	VMOVDQA 32(BP), Y14
+	VMOVDQA Y14, Y9
+	VMOVDQA Y14, Y10
+	VMOVDQA Y14, Y11
+	VMOVDQA 64(BP), Y12
+	VMOVDQA Y12, Y13
+	VMOVDQA Y12, Y8
+	VMOVDQA Y12, Y15
+	VMOVDQA 192(BP), Y4
+	VPADDD  ·avx2IncMask<>+0(SB), Y4, Y4
+	VPADDD  ·avx2IncMask<>+0(SB), Y4, Y1
+	VPADDD  ·avx2IncMask<>+0(SB), Y1, Y2
+	VPADDD  ·avx2IncMask<>+0(SB), Y2, Y3
+	VMOVDQA Y4, 96(BP)
+	VMOVDQA Y1, 128(BP)
+	VMOVDQA Y2, 160(BP)
+	VMOVDQA Y3, 192(BP)
+	MOVQ    $0x0000000a, CX
 
 sealAVX2InternalLoop:
-	polyAdd(0*8(oup))
-	VPADDD  BB0, AA0, AA0; VPADDD BB1, AA1, AA1; VPADDD BB2, AA2, AA2; VPADDD BB3, AA3, AA3
-	polyMulStage1_AVX2
-	VPXOR   AA0, DD0, DD0; VPXOR AA1, DD1, DD1; VPXOR AA2, DD2, DD2; VPXOR AA3, DD3, DD3
-	VPSHUFB ·rol16<>(SB), DD0, DD0; VPSHUFB ·rol16<>(SB), DD1, DD1; VPSHUFB ·rol16<>(SB), DD2, DD2; VPSHUFB ·rol16<>(SB), DD3, DD3
-	polyMulStage2_AVX2
-	VPADDD  DD0, CC0, CC0; VPADDD DD1, CC1, CC1; VPADDD DD2, CC2, CC2; VPADDD DD3, CC3, CC3
-	VPXOR   CC0, BB0, BB0; VPXOR CC1, BB1, BB1; VPXOR CC2, BB2, BB2; VPXOR CC3, BB3, BB3
-	polyMulStage3_AVX2
-	VMOVDQA CC3, tmpStoreAVX2
-	VPSLLD  $12, BB0, CC3; VPSRLD $20, BB0, BB0; VPXOR CC3, BB0, BB0
-	VPSLLD  $12, BB1, CC3; VPSRLD $20, BB1, BB1; VPXOR CC3, BB1, BB1
-	VPSLLD  $12, BB2, CC3; VPSRLD $20, BB2, BB2; VPXOR CC3, BB2, BB2
-	VPSLLD  $12, BB3, CC3; VPSRLD $20, BB3, BB3; VPXOR CC3, BB3, BB3
-	VMOVDQA tmpStoreAVX2, CC3
-	polyMulReduceStage
+	ADDQ    (DI), R10
+	ADCQ    8(DI), R11
+	ADCQ    $0x01, R12
+	VPADDD  Y14, Y0, Y0
+	VPADDD  Y9, Y5, Y5
+	VPADDD  Y10, Y6, Y6
+	VPADDD  Y11, Y7, Y7
+	MOVQ    (BP), DX
+	MOVQ    DX, R15
+	MULXQ   R10, R13, R14
+	IMULQ   R12, R15
+	MULXQ   R11, AX, DX
+	ADDQ    AX, R14
+	ADCQ    DX, R15
+	VPXOR   Y0, Y4, Y4
+	VPXOR   Y5, Y1, Y1
+	VPXOR   Y6, Y2, Y2
+	VPXOR   Y7, Y3, Y3
+	VPSHUFB ·rol16<>+0(SB), Y4, Y4
+	VPSHUFB ·rol16<>+0(SB), Y1, Y1
+	VPSHUFB ·rol16<>+0(SB), Y2, Y2
+	VPSHUFB ·rol16<>+0(SB), Y3, Y3
+	MOVQ    8(BP), DX
+	MULXQ   R10, R10, AX
+	ADDQ    R10, R14
+	MULXQ   R11, R11, R8
+	ADCQ    R11, R15
+	ADCQ    $0x00, R8
+	VPADDD  Y4, Y12, Y12
+	VPADDD  Y1, Y13, Y13
+	VPADDD  Y2, Y8, Y8
+	VPADDD  Y3, Y15, Y15
+	VPXOR   Y12, Y14, Y14
+	VPXOR   Y13, Y9, Y9
+	VPXOR   Y8, Y10, Y10
+	VPXOR   Y15, Y11, Y11
+	IMULQ   R12, DX
+	ADDQ    AX, R15
+	ADCQ    DX, R8
+	VMOVDQA Y15, 224(BP)
+	VPSLLD  $0x0c, Y14, Y15
+	VPSRLD  $0x14, Y14, Y14
+	VPXOR   Y15, Y14, Y14
+	VPSLLD  $0x0c, Y9, Y15
+	VPSRLD  $0x14, Y9, Y9
+	VPXOR   Y15, Y9, Y9
+	VPSLLD  $0x0c, Y10, Y15
+	VPSRLD  $0x14, Y10, Y10
+	VPXOR   Y15, Y10, Y10
+	VPSLLD  $0x0c, Y11, Y15
+	VPSRLD  $0x14, Y11, Y11
+	VPXOR   Y15, Y11, Y11
+	VMOVDQA 224(BP), Y15
+	MOVQ    R13, R10
+	MOVQ    R14, R11
+	MOVQ    R15, R12
+	ANDQ    $0x03, R12
+	MOVQ    R15, R13
+	ANDQ    $-4, R13
+	MOVQ    R8, R14
+	SHRQ    $0x02, R8, R15
+	SHRQ    $0x02, R8
+	ADDQ    R13, R10
+	ADCQ    R14, R11
+	ADCQ    $0x00, R12
+	ADDQ    R15, R10
+	ADCQ    R8, R11
+	ADCQ    $0x00, R12
 
 sealAVX2InternalLoopStart:
-	VPADDD   BB0, AA0, AA0; VPADDD BB1, AA1, AA1; VPADDD BB2, AA2, AA2; VPADDD BB3, AA3, AA3
-	VPXOR    AA0, DD0, DD0; VPXOR AA1, DD1, DD1; VPXOR AA2, DD2, DD2; VPXOR AA3, DD3, DD3
-	VPSHUFB  ·rol8<>(SB), DD0, DD0; VPSHUFB ·rol8<>(SB), DD1, DD1; VPSHUFB ·rol8<>(SB), DD2, DD2; VPSHUFB ·rol8<>(SB), DD3, DD3
-	polyAdd(2*8(oup))
-	VPADDD   DD0, CC0, CC0; VPADDD DD1, CC1, CC1; VPADDD DD2, CC2, CC2; VPADDD DD3, CC3, CC3
-	polyMulStage1_AVX2
-	VPXOR    CC0, BB0, BB0; VPXOR CC1, BB1, BB1; VPXOR CC2, BB2, BB2; VPXOR CC3, BB3, BB3
-	VMOVDQA  CC3, tmpStoreAVX2
-	VPSLLD   $7, BB0, CC3; VPSRLD $25, BB0, BB0; VPXOR CC3, BB0, BB0
-	VPSLLD   $7, BB1, CC3; VPSRLD $25, BB1, BB1; VPXOR CC3, BB1, BB1
-	VPSLLD   $7, BB2, CC3; VPSRLD $25, BB2, BB2; VPXOR CC3, BB2, BB2
-	VPSLLD   $7, BB3, CC3; VPSRLD $25, BB3, BB3; VPXOR CC3, BB3, BB3
-	VMOVDQA  tmpStoreAVX2, CC3
-	polyMulStage2_AVX2
-	VPALIGNR $4, BB0, BB0, BB0; VPALIGNR $4, BB1, BB1, BB1; VPALIGNR $4, BB2, BB2, BB2; VPALIGNR $4, BB3, BB3, BB3
-	VPALIGNR $8, CC0, CC0, CC0; VPALIGNR $8, CC1, CC1, CC1; VPALIGNR $8, CC2, CC2, CC2; VPALIGNR $8, CC3, CC3, CC3
-	VPALIGNR $12, DD0, DD0, DD0; VPALIGNR $12, DD1, DD1, DD1; VPALIGNR $12, DD2, DD2, DD2; VPALIGNR $12, DD3, DD3, DD3
-	VPADDD   BB0, AA0, AA0; VPADDD BB1, AA1, AA1; VPADDD BB2, AA2, AA2; VPADDD BB3, AA3, AA3
-	polyMulStage3_AVX2
-	VPXOR    AA0, DD0, DD0; VPXOR AA1, DD1, DD1; VPXOR AA2, DD2, DD2; VPXOR AA3, DD3, DD3
-	VPSHUFB  ·rol16<>(SB), DD0, DD0; VPSHUFB ·rol16<>(SB), DD1, DD1; VPSHUFB ·rol16<>(SB), DD2, DD2; VPSHUFB ·rol16<>(SB), DD3, DD3
-	polyMulReduceStage
-	VPADDD   DD0, CC0, CC0; VPADDD DD1, CC1, CC1; VPADDD DD2, CC2, CC2; VPADDD DD3, CC3, CC3
-	VPXOR    CC0, BB0, BB0; VPXOR CC1, BB1, BB1; VPXOR CC2, BB2, BB2; VPXOR CC3, BB3, BB3
-	polyAdd(4*8(oup))
-	LEAQ     (6*8)(oup), oup
-	VMOVDQA  CC3, tmpStoreAVX2
-	VPSLLD   $12, BB0, CC3; VPSRLD $20, BB0, BB0; VPXOR CC3, BB0, BB0
-	VPSLLD   $12, BB1, CC3; VPSRLD $20, BB1, BB1; VPXOR CC3, BB1, BB1
-	VPSLLD   $12, BB2, CC3; VPSRLD $20, BB2, BB2; VPXOR CC3, BB2, BB2
-	VPSLLD   $12, BB3, CC3; VPSRLD $20, BB3, BB3; VPXOR CC3, BB3, BB3
-	VMOVDQA  tmpStoreAVX2, CC3
-	polyMulStage1_AVX2
-	VPADDD   BB0, AA0, AA0; VPADDD BB1, AA1, AA1; VPADDD BB2, AA2, AA2; VPADDD BB3, AA3, AA3
-	VPXOR    AA0, DD0, DD0; VPXOR AA1, DD1, DD1; VPXOR AA2, DD2, DD2; VPXOR AA3, DD3, DD3
-	polyMulStage2_AVX2
-	VPSHUFB  ·rol8<>(SB), DD0, DD0; VPSHUFB ·rol8<>(SB), DD1, DD1; VPSHUFB ·rol8<>(SB), DD2, DD2; VPSHUFB ·rol8<>(SB), DD3, DD3
-	VPADDD   DD0, CC0, CC0; VPADDD DD1, CC1, CC1; VPADDD DD2, CC2, CC2; VPADDD DD3, CC3, CC3
-	polyMulStage3_AVX2
-	VPXOR    CC0, BB0, BB0; VPXOR CC1, BB1, BB1; VPXOR CC2, BB2, BB2; VPXOR CC3, BB3, BB3
-	VMOVDQA  CC3, tmpStoreAVX2
-	VPSLLD   $7, BB0, CC3; VPSRLD $25, BB0, BB0; VPXOR CC3, BB0, BB0
-	VPSLLD   $7, BB1, CC3; VPSRLD $25, BB1, BB1; VPXOR CC3, BB1, BB1
-	VPSLLD   $7, BB2, CC3; VPSRLD $25, BB2, BB2; VPXOR CC3, BB2, BB2
-	VPSLLD   $7, BB3, CC3; VPSRLD $25, BB3, BB3; VPXOR CC3, BB3, BB3
-	VMOVDQA  tmpStoreAVX2, CC3
-	polyMulReduceStage
-	VPALIGNR $12, BB0, BB0, BB0; VPALIGNR $12, BB1, BB1, BB1; VPALIGNR $12, BB2, BB2, BB2; VPALIGNR $12, BB3, BB3, BB3
-	VPALIGNR $8, CC0, CC0, CC0; VPALIGNR $8, CC1, CC1, CC1; VPALIGNR $8, CC2, CC2, CC2; VPALIGNR $8, CC3, CC3, CC3
-	VPALIGNR $4, DD0, DD0, DD0; VPALIGNR $4, DD1, DD1, DD1; VPALIGNR $4, DD2, DD2, DD2; VPALIGNR $4, DD3, DD3, DD3
-	DECQ     itr1
+	VPADDD   Y14, Y0, Y0
+	VPADDD   Y9, Y5, Y5
+	VPADDD   Y10, Y6, Y6
+	VPADDD   Y11, Y7, Y7
+	VPXOR    Y0, Y4, Y4
+	VPXOR    Y5, Y1, Y1
+	VPXOR    Y6, Y2, Y2
+	VPXOR    Y7, Y3, Y3
+	VPSHUFB  ·rol8<>+0(SB), Y4, Y4
+	VPSHUFB  ·rol8<>+0(SB), Y1, Y1
+	VPSHUFB  ·rol8<>+0(SB), Y2, Y2
+	VPSHUFB  ·rol8<>+0(SB), Y3, Y3
+	ADDQ     16(DI), R10
+	ADCQ     24(DI), R11
+	ADCQ     $0x01, R12
+	VPADDD   Y4, Y12, Y12
+	VPADDD   Y1, Y13, Y13
+	VPADDD   Y2, Y8, Y8
+	VPADDD   Y3, Y15, Y15
+	MOVQ     (BP), DX
+	MOVQ     DX, R15
+	MULXQ    R10, R13, R14
+	IMULQ    R12, R15
+	MULXQ    R11, AX, DX
+	ADDQ     AX, R14
+	ADCQ     DX, R15
+	VPXOR    Y12, Y14, Y14
+	VPXOR    Y13, Y9, Y9
+	VPXOR    Y8, Y10, Y10
+	VPXOR    Y15, Y11, Y11
+	VMOVDQA  Y15, 224(BP)
+	VPSLLD   $0x07, Y14, Y15
+	VPSRLD   $0x19, Y14, Y14
+	VPXOR    Y15, Y14, Y14
+	VPSLLD   $0x07, Y9, Y15
+	VPSRLD   $0x19, Y9, Y9
+	VPXOR    Y15, Y9, Y9
+	VPSLLD   $0x07, Y10, Y15
+	VPSRLD   $0x19, Y10, Y10
+	VPXOR    Y15, Y10, Y10
+	VPSLLD   $0x07, Y11, Y15
+	VPSRLD   $0x19, Y11, Y11
+	VPXOR    Y15, Y11, Y11
+	VMOVDQA  224(BP), Y15
+	MOVQ     8(BP), DX
+	MULXQ    R10, R10, AX
+	ADDQ     R10, R14
+	MULXQ    R11, R11, R8
+	ADCQ     R11, R15
+	ADCQ     $0x00, R8
+	VPALIGNR $0x04, Y14, Y14, Y14
+	VPALIGNR $0x04, Y9, Y9, Y9
+	VPALIGNR $0x04, Y10, Y10, Y10
+	VPALIGNR $0x04, Y11, Y11, Y11
+	VPALIGNR $0x08, Y12, Y12, Y12
+	VPALIGNR $0x08, Y13, Y13, Y13
+	VPALIGNR $0x08, Y8, Y8, Y8
+	VPALIGNR $0x08, Y15, Y15, Y15
+	VPALIGNR $0x0c, Y4, Y4, Y4
+	VPALIGNR $0x0c, Y1, Y1, Y1
+	VPALIGNR $0x0c, Y2, Y2, Y2
+	VPALIGNR $0x0c, Y3, Y3, Y3
+	VPADDD   Y14, Y0, Y0
+	VPADDD   Y9, Y5, Y5
+	VPADDD   Y10, Y6, Y6
+	VPADDD   Y11, Y7, Y7
+	IMULQ    R12, DX
+	ADDQ     AX, R15
+	ADCQ     DX, R8
+	VPXOR    Y0, Y4, Y4
+	VPXOR    Y5, Y1, Y1
+	VPXOR    Y6, Y2, Y2
+	VPXOR    Y7, Y3, Y3
+	VPSHUFB  ·rol16<>+0(SB), Y4, Y4
+	VPSHUFB  ·rol16<>+0(SB), Y1, Y1
+	VPSHUFB  ·rol16<>+0(SB), Y2, Y2
+	VPSHUFB  ·rol16<>+0(SB), Y3, Y3
+	MOVQ     R13, R10
+	MOVQ     R14, R11
+	MOVQ     R15, R12
+	ANDQ     $0x03, R12
+	MOVQ     R15, R13
+	ANDQ     $-4, R13
+	MOVQ     R8, R14
+	SHRQ     $0x02, R8, R15
+	SHRQ     $0x02, R8
+	ADDQ     R13, R10
+	ADCQ     R14, R11
+	ADCQ     $0x00, R12
+	ADDQ     R15, R10
+	ADCQ     R8, R11
+	ADCQ     $0x00, R12
+	VPADDD   Y4, Y12, Y12
+	VPADDD   Y1, Y13, Y13
+	VPADDD   Y2, Y8, Y8
+	VPADDD   Y3, Y15, Y15
+	VPXOR    Y12, Y14, Y14
+	VPXOR    Y13, Y9, Y9
+	VPXOR    Y8, Y10, Y10
+	VPXOR    Y15, Y11, Y11
+	ADDQ     32(DI), R10
+	ADCQ     40(DI), R11
+	ADCQ     $0x01, R12
+	LEAQ     48(DI), DI
+	VMOVDQA  Y15, 224(BP)
+	VPSLLD   $0x0c, Y14, Y15
+	VPSRLD   $0x14, Y14, Y14
+	VPXOR    Y15, Y14, Y14
+	VPSLLD   $0x0c, Y9, Y15
+	VPSRLD   $0x14, Y9, Y9
+	VPXOR    Y15, Y9, Y9
+	VPSLLD   $0x0c, Y10, Y15
+	VPSRLD   $0x14, Y10, Y10
+	VPXOR    Y15, Y10, Y10
+	VPSLLD   $0x0c, Y11, Y15
+	VPSRLD   $0x14, Y11, Y11
+	VPXOR    Y15, Y11, Y11
+	VMOVDQA  224(BP), Y15
+	MOVQ     (BP), DX
+	MOVQ     DX, R15
+	MULXQ    R10, R13, R14
+	IMULQ    R12, R15
+	MULXQ    R11, AX, DX
+	ADDQ     AX, R14
+	ADCQ     DX, R15
+	VPADDD   Y14, Y0, Y0
+	VPADDD   Y9, Y5, Y5
+	VPADDD   Y10, Y6, Y6
+	VPADDD   Y11, Y7, Y7
+	VPXOR    Y0, Y4, Y4
+	VPXOR    Y5, Y1, Y1
+	VPXOR    Y6, Y2, Y2
+	VPXOR    Y7, Y3, Y3
+	MOVQ     8(BP), DX
+	MULXQ    R10, R10, AX
+	ADDQ     R10, R14
+	MULXQ    R11, R11, R8
+	ADCQ     R11, R15
+	ADCQ     $0x00, R8
+	VPSHUFB  ·rol8<>+0(SB), Y4, Y4
+	VPSHUFB  ·rol8<>+0(SB), Y1, Y1
+	VPSHUFB  ·rol8<>+0(SB), Y2, Y2
+	VPSHUFB  ·rol8<>+0(SB), Y3, Y3
+	VPADDD   Y4, Y12, Y12
+	VPADDD   Y1, Y13, Y13
+	VPADDD   Y2, Y8, Y8
+	VPADDD   Y3, Y15, Y15
+	IMULQ    R12, DX
+	ADDQ     AX, R15
+	ADCQ     DX, R8
+	VPXOR    Y12, Y14, Y14
+	VPXOR    Y13, Y9, Y9
+	VPXOR    Y8, Y10, Y10
+	VPXOR    Y15, Y11, Y11
+	VMOVDQA  Y15, 224(BP)
+	VPSLLD   $0x07, Y14, Y15
+	VPSRLD   $0x19, Y14, Y14
+	VPXOR    Y15, Y14, Y14
+	VPSLLD   $0x07, Y9, Y15
+	VPSRLD   $0x19, Y9, Y9
+	VPXOR    Y15, Y9, Y9
+	VPSLLD   $0x07, Y10, Y15
+	VPSRLD   $0x19, Y10, Y10
+	VPXOR    Y15, Y10, Y10
+	VPSLLD   $0x07, Y11, Y15
+	VPSRLD   $0x19, Y11, Y11
+	VPXOR    Y15, Y11, Y11
+	VMOVDQA  224(BP), Y15
+	MOVQ     R13, R10
+	MOVQ     R14, R11
+	MOVQ     R15, R12
+	ANDQ     $0x03, R12
+	MOVQ     R15, R13
+	ANDQ     $-4, R13
+	MOVQ     R8, R14
+	SHRQ     $0x02, R8, R15
+	SHRQ     $0x02, R8
+	ADDQ     R13, R10
+	ADCQ     R14, R11
+	ADCQ     $0x00, R12
+	ADDQ     R15, R10
+	ADCQ     R8, R11
+	ADCQ     $0x00, R12
+	VPALIGNR $0x0c, Y14, Y14, Y14
+	VPALIGNR $0x0c, Y9, Y9, Y9
+	VPALIGNR $0x0c, Y10, Y10, Y10
+	VPALIGNR $0x0c, Y11, Y11, Y11
+	VPALIGNR $0x08, Y12, Y12, Y12
+	VPALIGNR $0x08, Y13, Y13, Y13
+	VPALIGNR $0x08, Y8, Y8, Y8
+	VPALIGNR $0x08, Y15, Y15, Y15
+	VPALIGNR $0x04, Y4, Y4, Y4
+	VPALIGNR $0x04, Y1, Y1, Y1
+	VPALIGNR $0x04, Y2, Y2, Y2
+	VPALIGNR $0x04, Y3, Y3, Y3
+	DECQ     CX
 	JNE      sealAVX2InternalLoop
-
-	VPADDD  ·chacha20Constants<>(SB), AA0, AA0; VPADDD ·chacha20Constants<>(SB), AA1, AA1; VPADDD ·chacha20Constants<>(SB), AA2, AA2; VPADDD ·chacha20Constants<>(SB), AA3, AA3
-	VPADDD  state1StoreAVX2, BB0, BB0; VPADDD state1StoreAVX2, BB1, BB1; VPADDD state1StoreAVX2, BB2, BB2; VPADDD state1StoreAVX2, BB3, BB3
-	VPADDD  state2StoreAVX2, CC0, CC0; VPADDD state2StoreAVX2, CC1, CC1; VPADDD state2StoreAVX2, CC2, CC2; VPADDD state2StoreAVX2, CC3, CC3
-	VPADDD  ctr0StoreAVX2, DD0, DD0; VPADDD ctr1StoreAVX2, DD1, DD1; VPADDD ctr2StoreAVX2, DD2, DD2; VPADDD ctr3StoreAVX2, DD3, DD3
-	VMOVDQA CC3, tmpStoreAVX2
+	VPADDD   ·chacha20Constants<>+0(SB), Y0, Y0
+	VPADDD   ·chacha20Constants<>+0(SB), Y5, Y5
+	VPADDD   ·chacha20Constants<>+0(SB), Y6, Y6
+	VPADDD   ·chacha20Constants<>+0(SB), Y7, Y7
+	VPADDD   32(BP), Y14, Y14
+	VPADDD   32(BP), Y9, Y9
+	VPADDD   32(BP), Y10, Y10
+	VPADDD   32(BP), Y11, Y11
+	VPADDD   64(BP), Y12, Y12
+	VPADDD   64(BP), Y13, Y13
+	VPADDD   64(BP), Y8, Y8
+	VPADDD   64(BP), Y15, Y15
+	VPADDD   96(BP), Y4, Y4
+	VPADDD   128(BP), Y1, Y1
+	VPADDD   160(BP), Y2, Y2
+	VPADDD   192(BP), Y3, Y3
+	VMOVDQA  Y15, 224(BP)
 
 	// We only hashed 480 of the 512 bytes available - hash the remaining 32 here
-	polyAdd(0*8(oup))
-	polyMulAVX2
-	LEAQ       (4*8)(oup), oup
-	VPERM2I128 $0x02, AA0, BB0, CC3; VPERM2I128 $0x13, AA0, BB0, BB0; VPERM2I128 $0x02, CC0, DD0, AA0; VPERM2I128 $0x13, CC0, DD0, CC0
-	VPXOR      (0*32)(inp), CC3, CC3; VPXOR (1*32)(inp), AA0, AA0; VPXOR (2*32)(inp), BB0, BB0; VPXOR (3*32)(inp), CC0, CC0
-	VMOVDQU    CC3, (0*32)(oup); VMOVDQU AA0, (1*32)(oup); VMOVDQU BB0, (2*32)(oup); VMOVDQU CC0, (3*32)(oup)
-	VPERM2I128 $0x02, AA1, BB1, AA0; VPERM2I128 $0x02, CC1, DD1, BB0; VPERM2I128 $0x13, AA1, BB1, CC0; VPERM2I128 $0x13, CC1, DD1, DD0
-	VPXOR      (4*32)(inp), AA0, AA0; VPXOR (5*32)(inp), BB0, BB0; VPXOR (6*32)(inp), CC0, CC0; VPXOR (7*32)(inp), DD0, DD0
-	VMOVDQU    AA0, (4*32)(oup); VMOVDQU BB0, (5*32)(oup); VMOVDQU CC0, (6*32)(oup); VMOVDQU DD0, (7*32)(oup)
+	ADDQ       (DI), R10
+	ADCQ       8(DI), R11
+	ADCQ       $0x01, R12
+	MOVQ       (BP), DX
+	MOVQ       DX, R15
+	MULXQ      R10, R13, R14
+	IMULQ      R12, R15
+	MULXQ      R11, AX, DX
+	ADDQ       AX, R14
+	ADCQ       DX, R15
+	MOVQ       8(BP), DX
+	MULXQ      R10, R10, AX
+	ADDQ       R10, R14
+	MULXQ      R11, R11, R8
+	ADCQ       R11, R15
+	ADCQ       $0x00, R8
+	IMULQ      R12, DX
+	ADDQ       AX, R15
+	ADCQ       DX, R8
+	MOVQ       R13, R10
+	MOVQ       R14, R11
+	MOVQ       R15, R12
+	ANDQ       $0x03, R12
+	MOVQ       R15, R13
+	ANDQ       $-4, R13
+	MOVQ       R8, R14
+	SHRQ       $0x02, R8, R15
+	SHRQ       $0x02, R8
+	ADDQ       R13, R10
+	ADCQ       R14, R11
+	ADCQ       $0x00, R12
+	ADDQ       R15, R10
+	ADCQ       R8, R11
+	ADCQ       $0x00, R12
+	LEAQ       32(DI), DI
+	VPERM2I128 $0x02, Y0, Y14, Y15
+	VPERM2I128 $0x13, Y0, Y14, Y14
+	VPERM2I128 $0x02, Y12, Y4, Y0
+	VPERM2I128 $0x13, Y12, Y4, Y12
+	VPXOR      (SI), Y15, Y15
+	VPXOR      32(SI), Y0, Y0
+	VPXOR      64(SI), Y14, Y14
+	VPXOR      96(SI), Y12, Y12
+	VMOVDQU    Y15, (DI)
+	VMOVDQU    Y0, 32(DI)
+	VMOVDQU    Y14, 64(DI)
+	VMOVDQU    Y12, 96(DI)
+	VPERM2I128 $0x02, Y5, Y9, Y0
+	VPERM2I128 $0x02, Y13, Y1, Y14
+	VPERM2I128 $0x13, Y5, Y9, Y12
+	VPERM2I128 $0x13, Y13, Y1, Y4
+	VPXOR      128(SI), Y0, Y0
+	VPXOR      160(SI), Y14, Y14
+	VPXOR      192(SI), Y12, Y12
+	VPXOR      224(SI), Y4, Y4
+	VMOVDQU    Y0, 128(DI)
+	VMOVDQU    Y14, 160(DI)
+	VMOVDQU    Y12, 192(DI)
+	VMOVDQU    Y4, 224(DI)
 
 	// and here
-	polyAdd(-2*8(oup))
-	polyMulAVX2
-	VPERM2I128 $0x02, AA2, BB2, AA0; VPERM2I128 $0x02, CC2, DD2, BB0; VPERM2I128 $0x13, AA2, BB2, CC0; VPERM2I128 $0x13, CC2, DD2, DD0
-	VPXOR      (8*32)(inp), AA0, AA0; VPXOR (9*32)(inp), BB0, BB0; VPXOR (10*32)(inp), CC0, CC0; VPXOR (11*32)(inp), DD0, DD0
-	VMOVDQU    AA0, (8*32)(oup); VMOVDQU BB0, (9*32)(oup); VMOVDQU CC0, (10*32)(oup); VMOVDQU DD0, (11*32)(oup)
-	VPERM2I128 $0x02, AA3, BB3, AA0; VPERM2I128 $0x02, tmpStoreAVX2, DD3, BB0; VPERM2I128 $0x13, AA3, BB3, CC0; VPERM2I128 $0x13, tmpStoreAVX2, DD3, DD0
-	VPXOR      (12*32)(inp), AA0, AA0; VPXOR (13*32)(inp), BB0, BB0; VPXOR (14*32)(inp), CC0, CC0; VPXOR (15*32)(inp), DD0, DD0
-	VMOVDQU    AA0, (12*32)(oup); VMOVDQU BB0, (13*32)(oup); VMOVDQU CC0, (14*32)(oup); VMOVDQU DD0, (15*32)(oup)
-	LEAQ       (32*16)(inp), inp
-	SUBQ       $(32*16), inl
-	CMPQ       inl, $512
+	ADDQ       -16(DI), R10
+	ADCQ       -8(DI), R11
+	ADCQ       $0x01, R12
+	MOVQ       (BP), DX
+	MOVQ       DX, R15
+	MULXQ      R10, R13, R14
+	IMULQ      R12, R15
+	MULXQ      R11, AX, DX
+	ADDQ       AX, R14
+	ADCQ       DX, R15
+	MOVQ       8(BP), DX
+	MULXQ      R10, R10, AX
+	ADDQ       R10, R14
+	MULXQ      R11, R11, R8
+	ADCQ       R11, R15
+	ADCQ       $0x00, R8
+	IMULQ      R12, DX
+	ADDQ       AX, R15
+	ADCQ       DX, R8
+	MOVQ       R13, R10
+	MOVQ       R14, R11
+	MOVQ       R15, R12
+	ANDQ       $0x03, R12
+	MOVQ       R15, R13
+	ANDQ       $-4, R13
+	MOVQ       R8, R14
+	SHRQ       $0x02, R8, R15
+	SHRQ       $0x02, R8
+	ADDQ       R13, R10
+	ADCQ       R14, R11
+	ADCQ       $0x00, R12
+	ADDQ       R15, R10
+	ADCQ       R8, R11
+	ADCQ       $0x00, R12
+	VPERM2I128 $0x02, Y6, Y10, Y0
+	VPERM2I128 $0x02, Y8, Y2, Y14
+	VPERM2I128 $0x13, Y6, Y10, Y12
+	VPERM2I128 $0x13, Y8, Y2, Y4
+	VPXOR      256(SI), Y0, Y0
+	VPXOR      288(SI), Y14, Y14
+	VPXOR      320(SI), Y12, Y12
+	VPXOR      352(SI), Y4, Y4
+	VMOVDQU    Y0, 256(DI)
+	VMOVDQU    Y14, 288(DI)
+	VMOVDQU    Y12, 320(DI)
+	VMOVDQU    Y4, 352(DI)
+	VPERM2I128 $0x02, Y7, Y11, Y0
+	VPERM2I128 $0x02, 224(BP), Y3, Y14
+	VPERM2I128 $0x13, Y7, Y11, Y12
+	VPERM2I128 $0x13, 224(BP), Y3, Y4
+	VPXOR      384(SI), Y0, Y0
+	VPXOR      416(SI), Y14, Y14
+	VPXOR      448(SI), Y12, Y12
+	VPXOR      480(SI), Y4, Y4
+	VMOVDQU    Y0, 384(DI)
+	VMOVDQU    Y14, 416(DI)
+	VMOVDQU    Y12, 448(DI)
+	VMOVDQU    Y4, 480(DI)
+	LEAQ       512(SI), SI
+	SUBQ       $0x00000200, BX
+	CMPQ       BX, $0x00000200
 	JG         sealAVX2MainLoop
 
 	// Tail can only hash 480 bytes
-	polyAdd(0*8(oup))
-	polyMulAVX2
-	polyAdd(2*8(oup))
-	polyMulAVX2
-	LEAQ 32(oup), oup
-
-	MOVQ $10, itr1
-	MOVQ $0, itr2
-	CMPQ inl, $128
-	JBE  sealAVX2Tail128
-	CMPQ inl, $256
-	JBE  sealAVX2Tail256
-	CMPQ inl, $384
-	JBE  sealAVX2Tail384
-	JMP  sealAVX2Tail512
-
-// ----------------------------------------------------------------------------
-// Special optimization for buffers smaller than 193 bytes
+	ADDQ  (DI), R10
+	ADCQ  8(DI), R11
+	ADCQ  $0x01, R12
+	MOVQ  (BP), DX
+	MOVQ  DX, R15
+	MULXQ R10, R13, R14
+	IMULQ R12, R15
+	MULXQ R11, AX, DX
+	ADDQ  AX, R14
+	ADCQ  DX, R15
+	MOVQ  8(BP), DX
+	MULXQ R10, R10, AX
+	ADDQ  R10, R14
+	MULXQ R11, R11, R8
+	ADCQ  R11, R15
+	ADCQ  $0x00, R8
+	IMULQ R12, DX
+	ADDQ  AX, R15
+	ADCQ  DX, R8
+	MOVQ  R13, R10
+	MOVQ  R14, R11
+	MOVQ  R15, R12
+	ANDQ  $0x03, R12
+	MOVQ  R15, R13
+	ANDQ  $-4, R13
+	MOVQ  R8, R14
+	SHRQ  $0x02, R8, R15
+	SHRQ  $0x02, R8
+	ADDQ  R13, R10
+	ADCQ  R14, R11
+	ADCQ  $0x00, R12
+	ADDQ  R15, R10
+	ADCQ  R8, R11
+	ADCQ  $0x00, R12
+	ADDQ  16(DI), R10
+	ADCQ  24(DI), R11
+	ADCQ  $0x01, R12
+	MOVQ  (BP), DX
+	MOVQ  DX, R15
+	MULXQ R10, R13, R14
+	IMULQ R12, R15
+	MULXQ R11, AX, DX
+	ADDQ  AX, R14
+	ADCQ  DX, R15
+	MOVQ  8(BP), DX
+	MULXQ R10, R10, AX
+	ADDQ  R10, R14
+	MULXQ R11, R11, R8
+	ADCQ  R11, R15
+	ADCQ  $0x00, R8
+	IMULQ R12, DX
+	ADDQ  AX, R15
+	ADCQ  DX, R8
+	MOVQ  R13, R10
+	MOVQ  R14, R11
+	MOVQ  R15, R12
+	ANDQ  $0x03, R12
+	MOVQ  R15, R13
+	ANDQ  $-4, R13
+	MOVQ  R8, R14
+	SHRQ  $0x02, R8, R15
+	SHRQ  $0x02, R8
+	ADDQ  R13, R10
+	ADCQ  R14, R11
+	ADCQ  $0x00, R12
+	ADDQ  R15, R10
+	ADCQ  R8, R11
+	ADCQ  $0x00, R12
+	LEAQ  32(DI), DI
+	MOVQ  $0x0000000a, CX
+	MOVQ  $0x00000000, R9
+	CMPQ  BX, $0x80
+	JBE   sealAVX2Tail128
+	CMPQ  BX, $0x00000100
+	JBE   sealAVX2Tail256
+	CMPQ  BX, $0x00000180
+	JBE   sealAVX2Tail384
+	JMP   sealAVX2Tail512
+
 seal192AVX2:
-	// For up to 192 bytes of ciphertext and 64 bytes for the poly key, we process four blocks
-	VMOVDQA AA0, AA1
-	VMOVDQA BB0, BB1
-	VMOVDQA CC0, CC1
-	VPADDD  ·avx2IncMask<>(SB), DD0, DD1
-	VMOVDQA AA0, AA2
-	VMOVDQA BB0, BB2
-	VMOVDQA CC0, CC2
-	VMOVDQA DD0, DD2
-	VMOVDQA DD1, TT3
-	MOVQ    $10, itr2
+	VMOVDQA Y0, Y5
+	VMOVDQA Y14, Y9
+	VMOVDQA Y12, Y13
+	VPADDD  ·avx2IncMask<>+0(SB), Y4, Y1
+	VMOVDQA Y0, Y6
+	VMOVDQA Y14, Y10
+	VMOVDQA Y12, Y8
+	VMOVDQA Y4, Y2
+	VMOVDQA Y1, Y15
+	MOVQ    $0x0000000a, R9
 
 sealAVX2192InnerCipherLoop:
-	chachaQR_AVX2(AA0, BB0, CC0, DD0, TT0); chachaQR_AVX2(AA1, BB1, CC1, DD1, TT0)
-	VPALIGNR   $4, BB0, BB0, BB0; VPALIGNR $4, BB1, BB1, BB1
-	VPALIGNR   $8, CC0, CC0, CC0; VPALIGNR $8, CC1, CC1, CC1
-	VPALIGNR   $12, DD0, DD0, DD0; VPALIGNR $12, DD1, DD1, DD1
-	chachaQR_AVX2(AA0, BB0, CC0, DD0, TT0); chachaQR_AVX2(AA1, BB1, CC1, DD1, TT0)
-	VPALIGNR   $12, BB0, BB0, BB0; VPALIGNR $12, BB1, BB1, BB1
-	VPALIGNR   $8, CC0, CC0, CC0; VPALIGNR $8, CC1, CC1, CC1
-	VPALIGNR   $4, DD0, DD0, DD0; VPALIGNR $4, DD1, DD1, DD1
-	DECQ       itr2
+	VPADDD     Y14, Y0, Y0
+	VPXOR      Y0, Y4, Y4
+	VPSHUFB    ·rol16<>+0(SB), Y4, Y4
+	VPADDD     Y4, Y12, Y12
+	VPXOR      Y12, Y14, Y14
+	VPSLLD     $0x0c, Y14, Y3
+	VPSRLD     $0x14, Y14, Y14
+	VPXOR      Y3, Y14, Y14
+	VPADDD     Y14, Y0, Y0
+	VPXOR      Y0, Y4, Y4
+	VPSHUFB    ·rol8<>+0(SB), Y4, Y4
+	VPADDD     Y4, Y12, Y12
+	VPXOR      Y12, Y14, Y14
+	VPSLLD     $0x07, Y14, Y3
+	VPSRLD     $0x19, Y14, Y14
+	VPXOR      Y3, Y14, Y14
+	VPADDD     Y9, Y5, Y5
+	VPXOR      Y5, Y1, Y1
+	VPSHUFB    ·rol16<>+0(SB), Y1, Y1
+	VPADDD     Y1, Y13, Y13
+	VPXOR      Y13, Y9, Y9
+	VPSLLD     $0x0c, Y9, Y3
+	VPSRLD     $0x14, Y9, Y9
+	VPXOR      Y3, Y9, Y9
+	VPADDD     Y9, Y5, Y5
+	VPXOR      Y5, Y1, Y1
+	VPSHUFB    ·rol8<>+0(SB), Y1, Y1
+	VPADDD     Y1, Y13, Y13
+	VPXOR      Y13, Y9, Y9
+	VPSLLD     $0x07, Y9, Y3
+	VPSRLD     $0x19, Y9, Y9
+	VPXOR      Y3, Y9, Y9
+	VPALIGNR   $0x04, Y14, Y14, Y14
+	VPALIGNR   $0x04, Y9, Y9, Y9
+	VPALIGNR   $0x08, Y12, Y12, Y12
+	VPALIGNR   $0x08, Y13, Y13, Y13
+	VPALIGNR   $0x0c, Y4, Y4, Y4
+	VPALIGNR   $0x0c, Y1, Y1, Y1
+	VPADDD     Y14, Y0, Y0
+	VPXOR      Y0, Y4, Y4
+	VPSHUFB    ·rol16<>+0(SB), Y4, Y4
+	VPADDD     Y4, Y12, Y12
+	VPXOR      Y12, Y14, Y14
+	VPSLLD     $0x0c, Y14, Y3
+	VPSRLD     $0x14, Y14, Y14
+	VPXOR      Y3, Y14, Y14
+	VPADDD     Y14, Y0, Y0
+	VPXOR      Y0, Y4, Y4
+	VPSHUFB    ·rol8<>+0(SB), Y4, Y4
+	VPADDD     Y4, Y12, Y12
+	VPXOR      Y12, Y14, Y14
+	VPSLLD     $0x07, Y14, Y3
+	VPSRLD     $0x19, Y14, Y14
+	VPXOR      Y3, Y14, Y14
+	VPADDD     Y9, Y5, Y5
+	VPXOR      Y5, Y1, Y1
+	VPSHUFB    ·rol16<>+0(SB), Y1, Y1
+	VPADDD     Y1, Y13, Y13
+	VPXOR      Y13, Y9, Y9
+	VPSLLD     $0x0c, Y9, Y3
+	VPSRLD     $0x14, Y9, Y9
+	VPXOR      Y3, Y9, Y9
+	VPADDD     Y9, Y5, Y5
+	VPXOR      Y5, Y1, Y1
+	VPSHUFB    ·rol8<>+0(SB), Y1, Y1
+	VPADDD     Y1, Y13, Y13
+	VPXOR      Y13, Y9, Y9
+	VPSLLD     $0x07, Y9, Y3
+	VPSRLD     $0x19, Y9, Y9
+	VPXOR      Y3, Y9, Y9
+	VPALIGNR   $0x0c, Y14, Y14, Y14
+	VPALIGNR   $0x0c, Y9, Y9, Y9
+	VPALIGNR   $0x08, Y12, Y12, Y12
+	VPALIGNR   $0x08, Y13, Y13, Y13
+	VPALIGNR   $0x04, Y4, Y4, Y4
+	VPALIGNR   $0x04, Y1, Y1, Y1
+	DECQ       R9
 	JNE        sealAVX2192InnerCipherLoop
-	VPADDD     AA2, AA0, AA0; VPADDD AA2, AA1, AA1
-	VPADDD     BB2, BB0, BB0; VPADDD BB2, BB1, BB1
-	VPADDD     CC2, CC0, CC0; VPADDD CC2, CC1, CC1
-	VPADDD     DD2, DD0, DD0; VPADDD TT3, DD1, DD1
-	VPERM2I128 $0x02, AA0, BB0, TT0
+	VPADDD     Y6, Y0, Y0
+	VPADDD     Y6, Y5, Y5
+	VPADDD     Y10, Y14, Y14
+	VPADDD     Y10, Y9, Y9
+	VPADDD     Y8, Y12, Y12
+	VPADDD     Y8, Y13, Y13
+	VPADDD     Y2, Y4, Y4
+	VPADDD     Y15, Y1, Y1
+	VPERM2I128 $0x02, Y0, Y14, Y3
 
 	// Clamp and store poly key
-	VPAND   ·polyClampMask<>(SB), TT0, TT0
-	VMOVDQA TT0, rsStoreAVX2
+	VPAND   ·polyClampMask<>+0(SB), Y3, Y3
+	VMOVDQA Y3, (BP)
 
 	// Stream for up to 192 bytes
-	VPERM2I128 $0x13, AA0, BB0, AA0
-	VPERM2I128 $0x13, CC0, DD0, BB0
-	VPERM2I128 $0x02, AA1, BB1, CC0
-	VPERM2I128 $0x02, CC1, DD1, DD0
-	VPERM2I128 $0x13, AA1, BB1, AA1
-	VPERM2I128 $0x13, CC1, DD1, BB1
+	VPERM2I128 $0x13, Y0, Y14, Y0
+	VPERM2I128 $0x13, Y12, Y4, Y14
+	VPERM2I128 $0x02, Y5, Y9, Y12
+	VPERM2I128 $0x02, Y13, Y1, Y4
+	VPERM2I128 $0x13, Y5, Y9, Y5
+	VPERM2I128 $0x13, Y13, Y1, Y9
 
 sealAVX2ShortSeal:
 	// Hash aad
-	MOVQ ad_len+80(FP), itr2
+	MOVQ ad_len+80(FP), R9
 	CALL polyHashADInternal<>(SB)
-	XORQ itr1, itr1
+	XORQ CX, CX
 
 sealAVX2SealHash:
 	// itr1 holds the number of bytes encrypted but not yet hashed
-	CMPQ itr1, $16
-	JB   sealAVX2ShortSealLoop
-	polyAdd(0(oup))
-	polyMul
-	SUBQ $16, itr1
-	ADDQ $16, oup
-	JMP  sealAVX2SealHash
+	CMPQ  CX, $0x10
+	JB    sealAVX2ShortSealLoop
+	ADDQ  (DI), R10
+	ADCQ  8(DI), R11
+	ADCQ  $0x01, R12
+	MOVQ  (BP), AX
+	MOVQ  AX, R15
+	MULQ  R10
+	MOVQ  AX, R13
+	MOVQ  DX, R14
+	MOVQ  (BP), AX
+	MULQ  R11
+	IMULQ R12, R15
+	ADDQ  AX, R14
+	ADCQ  DX, R15
+	MOVQ  8(BP), AX
+	MOVQ  AX, R8
+	MULQ  R10
+	ADDQ  AX, R14
+	ADCQ  $0x00, DX
+	MOVQ  DX, R10
+	MOVQ  8(BP), AX
+	MULQ  R11
+	ADDQ  AX, R15
+	ADCQ  $0x00, DX
+	IMULQ R12, R8
+	ADDQ  R10, R15
+	ADCQ  DX, R8
+	MOVQ  R13, R10
+	MOVQ  R14, R11
+	MOVQ  R15, R12
+	ANDQ  $0x03, R12
+	MOVQ  R15, R13
+	ANDQ  $-4, R13
+	MOVQ  R8, R14
+	SHRQ  $0x02, R8, R15
+	SHRQ  $0x02, R8
+	ADDQ  R13, R10
+	ADCQ  R14, R11
+	ADCQ  $0x00, R12
+	ADDQ  R15, R10
+	ADCQ  R8, R11
+	ADCQ  $0x00, R12
+	SUBQ  $0x10, CX
+	ADDQ  $0x10, DI
+	JMP   sealAVX2SealHash
 
 sealAVX2ShortSealLoop:
-	CMPQ inl, $32
+	CMPQ BX, $0x20
 	JB   sealAVX2ShortTail32
-	SUBQ $32, inl
+	SUBQ $0x20, BX
 
 	// Load for encryption
-	VPXOR   (inp), AA0, AA0
-	VMOVDQU AA0, (oup)
-	LEAQ    (1*32)(inp), inp
+	VPXOR   (SI), Y0, Y0
+	VMOVDQU Y0, (DI)
+	LEAQ    32(SI), SI
 
 	// Now can hash
-	polyAdd(0*8(oup))
-	polyMulAVX2
-	polyAdd(2*8(oup))
-	polyMulAVX2
-	LEAQ (1*32)(oup), oup
+	ADDQ  (DI), R10
+	ADCQ  8(DI), R11
+	ADCQ  $0x01, R12
+	MOVQ  (BP), DX
+	MOVQ  DX, R15
+	MULXQ R10, R13, R14
+	IMULQ R12, R15
+	MULXQ R11, AX, DX
+	ADDQ  AX, R14
+	ADCQ  DX, R15
+	MOVQ  8(BP), DX
+	MULXQ R10, R10, AX
+	ADDQ  R10, R14
+	MULXQ R11, R11, R8
+	ADCQ  R11, R15
+	ADCQ  $0x00, R8
+	IMULQ R12, DX
+	ADDQ  AX, R15
+	ADCQ  DX, R8
+	MOVQ  R13, R10
+	MOVQ  R14, R11
+	MOVQ  R15, R12
+	ANDQ  $0x03, R12
+	MOVQ  R15, R13
+	ANDQ  $-4, R13
+	MOVQ  R8, R14
+	SHRQ  $0x02, R8, R15
+	SHRQ  $0x02, R8
+	ADDQ  R13, R10
+	ADCQ  R14, R11
+	ADCQ  $0x00, R12
+	ADDQ  R15, R10
+	ADCQ  R8, R11
+	ADCQ  $0x00, R12
+	ADDQ  16(DI), R10
+	ADCQ  24(DI), R11
+	ADCQ  $0x01, R12
+	MOVQ  (BP), DX
+	MOVQ  DX, R15
+	MULXQ R10, R13, R14
+	IMULQ R12, R15
+	MULXQ R11, AX, DX
+	ADDQ  AX, R14
+	ADCQ  DX, R15
+	MOVQ  8(BP), DX
+	MULXQ R10, R10, AX
+	ADDQ  R10, R14
+	MULXQ R11, R11, R8
+	ADCQ  R11, R15
+	ADCQ  $0x00, R8
+	IMULQ R12, DX
+	ADDQ  AX, R15
+	ADCQ  DX, R8
+	MOVQ  R13, R10
+	MOVQ  R14, R11
+	MOVQ  R15, R12
+	ANDQ  $0x03, R12
+	MOVQ  R15, R13
+	ANDQ  $-4, R13
+	MOVQ  R8, R14
+	SHRQ  $0x02, R8, R15
+	SHRQ  $0x02, R8
+	ADDQ  R13, R10
+	ADCQ  R14, R11
+	ADCQ  $0x00, R12
+	ADDQ  R15, R10
+	ADCQ  R8, R11
+	ADCQ  $0x00, R12
+	LEAQ  32(DI), DI
 
 	// Shift stream left
-	VMOVDQA BB0, AA0
-	VMOVDQA CC0, BB0
-	VMOVDQA DD0, CC0
-	VMOVDQA AA1, DD0
-	VMOVDQA BB1, AA1
-	VMOVDQA CC1, BB1
-	VMOVDQA DD1, CC1
-	VMOVDQA AA2, DD1
-	VMOVDQA BB2, AA2
+	VMOVDQA Y14, Y0
+	VMOVDQA Y12, Y14
+	VMOVDQA Y4, Y12
+	VMOVDQA Y5, Y4
+	VMOVDQA Y9, Y5
+	VMOVDQA Y13, Y9
+	VMOVDQA Y1, Y13
+	VMOVDQA Y6, Y1
+	VMOVDQA Y10, Y6
 	JMP     sealAVX2ShortSealLoop
 
 sealAVX2ShortTail32:
-	CMPQ    inl, $16
-	VMOVDQA A0, A1
+	CMPQ    BX, $0x10
+	VMOVDQA X0, X1
 	JB      sealAVX2ShortDone
-
-	SUBQ $16, inl
+	SUBQ    $0x10, BX
 
 	// Load for encryption
-	VPXOR   (inp), A0, T0
-	VMOVDQU T0, (oup)
-	LEAQ    (1*16)(inp), inp
+	VPXOR   (SI), X0, X12
+	VMOVDQU X12, (DI)
+	LEAQ    16(SI), SI
 
 	// Hash
-	polyAdd(0*8(oup))
-	polyMulAVX2
-	LEAQ       (1*16)(oup), oup
-	VPERM2I128 $0x11, AA0, AA0, AA0
-	VMOVDQA    A0, A1
+	ADDQ       (DI), R10
+	ADCQ       8(DI), R11
+	ADCQ       $0x01, R12
+	MOVQ       (BP), DX
+	MOVQ       DX, R15
+	MULXQ      R10, R13, R14
+	IMULQ      R12, R15
+	MULXQ      R11, AX, DX
+	ADDQ       AX, R14
+	ADCQ       DX, R15
+	MOVQ       8(BP), DX
+	MULXQ      R10, R10, AX
+	ADDQ       R10, R14
+	MULXQ      R11, R11, R8
+	ADCQ       R11, R15
+	ADCQ       $0x00, R8
+	IMULQ      R12, DX
+	ADDQ       AX, R15
+	ADCQ       DX, R8
+	MOVQ       R13, R10
+	MOVQ       R14, R11
+	MOVQ       R15, R12
+	ANDQ       $0x03, R12
+	MOVQ       R15, R13
+	ANDQ       $-4, R13
+	MOVQ       R8, R14
+	SHRQ       $0x02, R8, R15
+	SHRQ       $0x02, R8
+	ADDQ       R13, R10
+	ADCQ       R14, R11
+	ADCQ       $0x00, R12
+	ADDQ       R15, R10
+	ADCQ       R8, R11
+	ADCQ       $0x00, R12
+	LEAQ       16(DI), DI
+	VPERM2I128 $0x11, Y0, Y0, Y0
+	VMOVDQA    X0, X1
 
 sealAVX2ShortDone:
 	VZEROUPPER
 	JMP sealSSETail
 
-// ----------------------------------------------------------------------------
-// Special optimization for buffers smaller than 321 bytes
 seal320AVX2:
-	// For up to 320 bytes of ciphertext and 64 bytes for the poly key, we process six blocks
-	VMOVDQA AA0, AA1; VMOVDQA BB0, BB1; VMOVDQA CC0, CC1; VPADDD ·avx2IncMask<>(SB), DD0, DD1
-	VMOVDQA AA0, AA2; VMOVDQA BB0, BB2; VMOVDQA CC0, CC2; VPADDD ·avx2IncMask<>(SB), DD1, DD2
-	VMOVDQA BB0, TT1; VMOVDQA CC0, TT2; VMOVDQA DD0, TT3
-	MOVQ    $10, itr2
+	VMOVDQA Y0, Y5
+	VMOVDQA Y14, Y9
+	VMOVDQA Y12, Y13
+	VPADDD  ·avx2IncMask<>+0(SB), Y4, Y1
+	VMOVDQA Y0, Y6
+	VMOVDQA Y14, Y10
+	VMOVDQA Y12, Y8
+	VPADDD  ·avx2IncMask<>+0(SB), Y1, Y2
+	VMOVDQA Y14, Y7
+	VMOVDQA Y12, Y11
+	VMOVDQA Y4, Y15
+	MOVQ    $0x0000000a, R9
 
 sealAVX2320InnerCipherLoop:
-	chachaQR_AVX2(AA0, BB0, CC0, DD0, TT0); chachaQR_AVX2(AA1, BB1, CC1, DD1, TT0); chachaQR_AVX2(AA2, BB2, CC2, DD2, TT0)
-	VPALIGNR $4, BB0, BB0, BB0; VPALIGNR $4, BB1, BB1, BB1; VPALIGNR $4, BB2, BB2, BB2
-	VPALIGNR $8, CC0, CC0, CC0; VPALIGNR $8, CC1, CC1, CC1; VPALIGNR $8, CC2, CC2, CC2
-	VPALIGNR $12, DD0, DD0, DD0; VPALIGNR $12, DD1, DD1, DD1; VPALIGNR $12, DD2, DD2, DD2
-	chachaQR_AVX2(AA0, BB0, CC0, DD0, TT0); chachaQR_AVX2(AA1, BB1, CC1, DD1, TT0); chachaQR_AVX2(AA2, BB2, CC2, DD2, TT0)
-	VPALIGNR $12, BB0, BB0, BB0; VPALIGNR $12, BB1, BB1, BB1; VPALIGNR $12, BB2, BB2, BB2
-	VPALIGNR $8, CC0, CC0, CC0; VPALIGNR $8, CC1, CC1, CC1; VPALIGNR $8, CC2, CC2, CC2
-	VPALIGNR $4, DD0, DD0, DD0; VPALIGNR $4, DD1, DD1, DD1; VPALIGNR $4, DD2, DD2, DD2
-	DECQ     itr2
+	VPADDD   Y14, Y0, Y0
+	VPXOR    Y0, Y4, Y4
+	VPSHUFB  ·rol16<>+0(SB), Y4, Y4
+	VPADDD   Y4, Y12, Y12
+	VPXOR    Y12, Y14, Y14
+	VPSLLD   $0x0c, Y14, Y3
+	VPSRLD   $0x14, Y14, Y14
+	VPXOR    Y3, Y14, Y14
+	VPADDD   Y14, Y0, Y0
+	VPXOR    Y0, Y4, Y4
+	VPSHUFB  ·rol8<>+0(SB), Y4, Y4
+	VPADDD   Y4, Y12, Y12
+	VPXOR    Y12, Y14, Y14
+	VPSLLD   $0x07, Y14, Y3
+	VPSRLD   $0x19, Y14, Y14
+	VPXOR    Y3, Y14, Y14
+	VPADDD   Y9, Y5, Y5
+	VPXOR    Y5, Y1, Y1
+	VPSHUFB  ·rol16<>+0(SB), Y1, Y1
+	VPADDD   Y1, Y13, Y13
+	VPXOR    Y13, Y9, Y9
+	VPSLLD   $0x0c, Y9, Y3
+	VPSRLD   $0x14, Y9, Y9
+	VPXOR    Y3, Y9, Y9
+	VPADDD   Y9, Y5, Y5
+	VPXOR    Y5, Y1, Y1
+	VPSHUFB  ·rol8<>+0(SB), Y1, Y1
+	VPADDD   Y1, Y13, Y13
+	VPXOR    Y13, Y9, Y9
+	VPSLLD   $0x07, Y9, Y3
+	VPSRLD   $0x19, Y9, Y9
+	VPXOR    Y3, Y9, Y9
+	VPADDD   Y10, Y6, Y6
+	VPXOR    Y6, Y2, Y2
+	VPSHUFB  ·rol16<>+0(SB), Y2, Y2
+	VPADDD   Y2, Y8, Y8
+	VPXOR    Y8, Y10, Y10
+	VPSLLD   $0x0c, Y10, Y3
+	VPSRLD   $0x14, Y10, Y10
+	VPXOR    Y3, Y10, Y10
+	VPADDD   Y10, Y6, Y6
+	VPXOR    Y6, Y2, Y2
+	VPSHUFB  ·rol8<>+0(SB), Y2, Y2
+	VPADDD   Y2, Y8, Y8
+	VPXOR    Y8, Y10, Y10
+	VPSLLD   $0x07, Y10, Y3
+	VPSRLD   $0x19, Y10, Y10
+	VPXOR    Y3, Y10, Y10
+	VPALIGNR $0x04, Y14, Y14, Y14
+	VPALIGNR $0x04, Y9, Y9, Y9
+	VPALIGNR $0x04, Y10, Y10, Y10
+	VPALIGNR $0x08, Y12, Y12, Y12
+	VPALIGNR $0x08, Y13, Y13, Y13
+	VPALIGNR $0x08, Y8, Y8, Y8
+	VPALIGNR $0x0c, Y4, Y4, Y4
+	VPALIGNR $0x0c, Y1, Y1, Y1
+	VPALIGNR $0x0c, Y2, Y2, Y2
+	VPADDD   Y14, Y0, Y0
+	VPXOR    Y0, Y4, Y4
+	VPSHUFB  ·rol16<>+0(SB), Y4, Y4
+	VPADDD   Y4, Y12, Y12
+	VPXOR    Y12, Y14, Y14
+	VPSLLD   $0x0c, Y14, Y3
+	VPSRLD   $0x14, Y14, Y14
+	VPXOR    Y3, Y14, Y14
+	VPADDD   Y14, Y0, Y0
+	VPXOR    Y0, Y4, Y4
+	VPSHUFB  ·rol8<>+0(SB), Y4, Y4
+	VPADDD   Y4, Y12, Y12
+	VPXOR    Y12, Y14, Y14
+	VPSLLD   $0x07, Y14, Y3
+	VPSRLD   $0x19, Y14, Y14
+	VPXOR    Y3, Y14, Y14
+	VPADDD   Y9, Y5, Y5
+	VPXOR    Y5, Y1, Y1
+	VPSHUFB  ·rol16<>+0(SB), Y1, Y1
+	VPADDD   Y1, Y13, Y13
+	VPXOR    Y13, Y9, Y9
+	VPSLLD   $0x0c, Y9, Y3
+	VPSRLD   $0x14, Y9, Y9
+	VPXOR    Y3, Y9, Y9
+	VPADDD   Y9, Y5, Y5
+	VPXOR    Y5, Y1, Y1
+	VPSHUFB  ·rol8<>+0(SB), Y1, Y1
+	VPADDD   Y1, Y13, Y13
+	VPXOR    Y13, Y9, Y9
+	VPSLLD   $0x07, Y9, Y3
+	VPSRLD   $0x19, Y9, Y9
+	VPXOR    Y3, Y9, Y9
+	VPADDD   Y10, Y6, Y6
+	VPXOR    Y6, Y2, Y2
+	VPSHUFB  ·rol16<>+0(SB), Y2, Y2
+	VPADDD   Y2, Y8, Y8
+	VPXOR    Y8, Y10, Y10
+	VPSLLD   $0x0c, Y10, Y3
+	VPSRLD   $0x14, Y10, Y10
+	VPXOR    Y3, Y10, Y10
+	VPADDD   Y10, Y6, Y6
+	VPXOR    Y6, Y2, Y2
+	VPSHUFB  ·rol8<>+0(SB), Y2, Y2
+	VPADDD   Y2, Y8, Y8
+	VPXOR    Y8, Y10, Y10
+	VPSLLD   $0x07, Y10, Y3
+	VPSRLD   $0x19, Y10, Y10
+	VPXOR    Y3, Y10, Y10
+	VPALIGNR $0x0c, Y14, Y14, Y14
+	VPALIGNR $0x0c, Y9, Y9, Y9
+	VPALIGNR $0x0c, Y10, Y10, Y10
+	VPALIGNR $0x08, Y12, Y12, Y12
+	VPALIGNR $0x08, Y13, Y13, Y13
+	VPALIGNR $0x08, Y8, Y8, Y8
+	VPALIGNR $0x04, Y4, Y4, Y4
+	VPALIGNR $0x04, Y1, Y1, Y1
+	VPALIGNR $0x04, Y2, Y2, Y2
+	DECQ     R9
 	JNE      sealAVX2320InnerCipherLoop
-
-	VMOVDQA ·chacha20Constants<>(SB), TT0
-	VPADDD  TT0, AA0, AA0; VPADDD TT0, AA1, AA1; VPADDD TT0, AA2, AA2
-	VPADDD  TT1, BB0, BB0; VPADDD TT1, BB1, BB1; VPADDD TT1, BB2, BB2
-	VPADDD  TT2, CC0, CC0; VPADDD TT2, CC1, CC1; VPADDD TT2, CC2, CC2
-	VMOVDQA ·avx2IncMask<>(SB), TT0
-	VPADDD  TT3, DD0, DD0; VPADDD TT0, TT3, TT3
-	VPADDD  TT3, DD1, DD1; VPADDD TT0, TT3, TT3
-	VPADDD  TT3, DD2, DD2
+	VMOVDQA  ·chacha20Constants<>+0(SB), Y3
+	VPADDD   Y3, Y0, Y0
+	VPADDD   Y3, Y5, Y5
+	VPADDD   Y3, Y6, Y6
+	VPADDD   Y7, Y14, Y14
+	VPADDD   Y7, Y9, Y9
+	VPADDD   Y7, Y10, Y10
+	VPADDD   Y11, Y12, Y12
+	VPADDD   Y11, Y13, Y13
+	VPADDD   Y11, Y8, Y8
+	VMOVDQA  ·avx2IncMask<>+0(SB), Y3
+	VPADDD   Y15, Y4, Y4
+	VPADDD   Y3, Y15, Y15
+	VPADDD   Y15, Y1, Y1
+	VPADDD   Y3, Y15, Y15
+	VPADDD   Y15, Y2, Y2
 
 	// Clamp and store poly key
-	VPERM2I128 $0x02, AA0, BB0, TT0
-	VPAND      ·polyClampMask<>(SB), TT0, TT0
-	VMOVDQA    TT0, rsStoreAVX2
+	VPERM2I128 $0x02, Y0, Y14, Y3
+	VPAND      ·polyClampMask<>+0(SB), Y3, Y3
+	VMOVDQA    Y3, (BP)
 
 	// Stream for up to 320 bytes
-	VPERM2I128 $0x13, AA0, BB0, AA0
-	VPERM2I128 $0x13, CC0, DD0, BB0
-	VPERM2I128 $0x02, AA1, BB1, CC0
-	VPERM2I128 $0x02, CC1, DD1, DD0
-	VPERM2I128 $0x13, AA1, BB1, AA1
-	VPERM2I128 $0x13, CC1, DD1, BB1
-	VPERM2I128 $0x02, AA2, BB2, CC1
-	VPERM2I128 $0x02, CC2, DD2, DD1
-	VPERM2I128 $0x13, AA2, BB2, AA2
-	VPERM2I128 $0x13, CC2, DD2, BB2
+	VPERM2I128 $0x13, Y0, Y14, Y0
+	VPERM2I128 $0x13, Y12, Y4, Y14
+	VPERM2I128 $0x02, Y5, Y9, Y12
+	VPERM2I128 $0x02, Y13, Y1, Y4
+	VPERM2I128 $0x13, Y5, Y9, Y5
+	VPERM2I128 $0x13, Y13, Y1, Y9
+	VPERM2I128 $0x02, Y6, Y10, Y13
+	VPERM2I128 $0x02, Y8, Y2, Y1
+	VPERM2I128 $0x13, Y6, Y10, Y6
+	VPERM2I128 $0x13, Y8, Y2, Y10
 	JMP        sealAVX2ShortSeal
 
-// ----------------------------------------------------------------------------
-// Special optimization for the last 128 bytes of ciphertext
 sealAVX2Tail128:
-	// Need to decrypt up to 128 bytes - prepare two blocks
-	// If we got here after the main loop - there are 512 encrypted bytes waiting to be hashed
-	// If we got here before the main loop - there are 448 encrpyred bytes waiting to be hashed
-	VMOVDQA ·chacha20Constants<>(SB), AA0
-	VMOVDQA state1StoreAVX2, BB0
-	VMOVDQA state2StoreAVX2, CC0
-	VMOVDQA ctr3StoreAVX2, DD0
-	VPADDD  ·avx2IncMask<>(SB), DD0, DD0
-	VMOVDQA DD0, DD1
+	VMOVDQA ·chacha20Constants<>+0(SB), Y0
+	VMOVDQA 32(BP), Y14
+	VMOVDQA 64(BP), Y12
+	VMOVDQA 192(BP), Y4
+	VPADDD  ·avx2IncMask<>+0(SB), Y4, Y4
+	VMOVDQA Y4, Y1
 
 sealAVX2Tail128LoopA:
-	polyAdd(0(oup))
-	polyMul
-	LEAQ 16(oup), oup
+	ADDQ  (DI), R10
+	ADCQ  8(DI), R11
+	ADCQ  $0x01, R12
+	MOVQ  (BP), AX
+	MOVQ  AX, R15
+	MULQ  R10
+	MOVQ  AX, R13
+	MOVQ  DX, R14
+	MOVQ  (BP), AX
+	MULQ  R11
+	IMULQ R12, R15
+	ADDQ  AX, R14
+	ADCQ  DX, R15
+	MOVQ  8(BP), AX
+	MOVQ  AX, R8
+	MULQ  R10
+	ADDQ  AX, R14
+	ADCQ  $0x00, DX
+	MOVQ  DX, R10
+	MOVQ  8(BP), AX
+	MULQ  R11
+	ADDQ  AX, R15
+	ADCQ  $0x00, DX
+	IMULQ R12, R8
+	ADDQ  R10, R15
+	ADCQ  DX, R8
+	MOVQ  R13, R10
+	MOVQ  R14, R11
+	MOVQ  R15, R12
+	ANDQ  $0x03, R12
+	MOVQ  R15, R13
+	ANDQ  $-4, R13
+	MOVQ  R8, R14
+	SHRQ  $0x02, R8, R15
+	SHRQ  $0x02, R8
+	ADDQ  R13, R10
+	ADCQ  R14, R11
+	ADCQ  $0x00, R12
+	ADDQ  R15, R10
+	ADCQ  R8, R11
+	ADCQ  $0x00, R12
+	LEAQ  16(DI), DI
 
 sealAVX2Tail128LoopB:
-	chachaQR_AVX2(AA0, BB0, CC0, DD0, TT0)
-	polyAdd(0(oup))
-	polyMul
-	VPALIGNR $4, BB0, BB0, BB0
-	VPALIGNR $8, CC0, CC0, CC0
-	VPALIGNR $12, DD0, DD0, DD0
-	chachaQR_AVX2(AA0, BB0, CC0, DD0, TT0)
-	polyAdd(16(oup))
-	polyMul
-	LEAQ     32(oup), oup
-	VPALIGNR $12, BB0, BB0, BB0
-	VPALIGNR $8, CC0, CC0, CC0
-	VPALIGNR $4, DD0, DD0, DD0
-	DECQ     itr1
-	JG       sealAVX2Tail128LoopA
-	DECQ     itr2
-	JGE      sealAVX2Tail128LoopB
-
-	VPADDD ·chacha20Constants<>(SB), AA0, AA1
-	VPADDD state1StoreAVX2, BB0, BB1
-	VPADDD state2StoreAVX2, CC0, CC1
-	VPADDD DD1, DD0, DD1
-
-	VPERM2I128 $0x02, AA1, BB1, AA0
-	VPERM2I128 $0x02, CC1, DD1, BB0
-	VPERM2I128 $0x13, AA1, BB1, CC0
-	VPERM2I128 $0x13, CC1, DD1, DD0
+	VPADDD     Y14, Y0, Y0
+	VPXOR      Y0, Y4, Y4
+	VPSHUFB    ·rol16<>+0(SB), Y4, Y4
+	VPADDD     Y4, Y12, Y12
+	VPXOR      Y12, Y14, Y14
+	VPSLLD     $0x0c, Y14, Y3
+	VPSRLD     $0x14, Y14, Y14
+	VPXOR      Y3, Y14, Y14
+	VPADDD     Y14, Y0, Y0
+	VPXOR      Y0, Y4, Y4
+	VPSHUFB    ·rol8<>+0(SB), Y4, Y4
+	VPADDD     Y4, Y12, Y12
+	VPXOR      Y12, Y14, Y14
+	VPSLLD     $0x07, Y14, Y3
+	VPSRLD     $0x19, Y14, Y14
+	VPXOR      Y3, Y14, Y14
+	ADDQ       (DI), R10
+	ADCQ       8(DI), R11
+	ADCQ       $0x01, R12
+	MOVQ       (BP), AX
+	MOVQ       AX, R15
+	MULQ       R10
+	MOVQ       AX, R13
+	MOVQ       DX, R14
+	MOVQ       (BP), AX
+	MULQ       R11
+	IMULQ      R12, R15
+	ADDQ       AX, R14
+	ADCQ       DX, R15
+	MOVQ       8(BP), AX
+	MOVQ       AX, R8
+	MULQ       R10
+	ADDQ       AX, R14
+	ADCQ       $0x00, DX
+	MOVQ       DX, R10
+	MOVQ       8(BP), AX
+	MULQ       R11
+	ADDQ       AX, R15
+	ADCQ       $0x00, DX
+	IMULQ      R12, R8
+	ADDQ       R10, R15
+	ADCQ       DX, R8
+	MOVQ       R13, R10
+	MOVQ       R14, R11
+	MOVQ       R15, R12
+	ANDQ       $0x03, R12
+	MOVQ       R15, R13
+	ANDQ       $-4, R13
+	MOVQ       R8, R14
+	SHRQ       $0x02, R8, R15
+	SHRQ       $0x02, R8
+	ADDQ       R13, R10
+	ADCQ       R14, R11
+	ADCQ       $0x00, R12
+	ADDQ       R15, R10
+	ADCQ       R8, R11
+	ADCQ       $0x00, R12
+	VPALIGNR   $0x04, Y14, Y14, Y14
+	VPALIGNR   $0x08, Y12, Y12, Y12
+	VPALIGNR   $0x0c, Y4, Y4, Y4
+	VPADDD     Y14, Y0, Y0
+	VPXOR      Y0, Y4, Y4
+	VPSHUFB    ·rol16<>+0(SB), Y4, Y4
+	VPADDD     Y4, Y12, Y12
+	VPXOR      Y12, Y14, Y14
+	VPSLLD     $0x0c, Y14, Y3
+	VPSRLD     $0x14, Y14, Y14
+	VPXOR      Y3, Y14, Y14
+	VPADDD     Y14, Y0, Y0
+	VPXOR      Y0, Y4, Y4
+	VPSHUFB    ·rol8<>+0(SB), Y4, Y4
+	VPADDD     Y4, Y12, Y12
+	VPXOR      Y12, Y14, Y14
+	VPSLLD     $0x07, Y14, Y3
+	VPSRLD     $0x19, Y14, Y14
+	VPXOR      Y3, Y14, Y14
+	ADDQ       16(DI), R10
+	ADCQ       24(DI), R11
+	ADCQ       $0x01, R12
+	MOVQ       (BP), AX
+	MOVQ       AX, R15
+	MULQ       R10
+	MOVQ       AX, R13
+	MOVQ       DX, R14
+	MOVQ       (BP), AX
+	MULQ       R11
+	IMULQ      R12, R15
+	ADDQ       AX, R14
+	ADCQ       DX, R15
+	MOVQ       8(BP), AX
+	MOVQ       AX, R8
+	MULQ       R10
+	ADDQ       AX, R14
+	ADCQ       $0x00, DX
+	MOVQ       DX, R10
+	MOVQ       8(BP), AX
+	MULQ       R11
+	ADDQ       AX, R15
+	ADCQ       $0x00, DX
+	IMULQ      R12, R8
+	ADDQ       R10, R15
+	ADCQ       DX, R8
+	MOVQ       R13, R10
+	MOVQ       R14, R11
+	MOVQ       R15, R12
+	ANDQ       $0x03, R12
+	MOVQ       R15, R13
+	ANDQ       $-4, R13
+	MOVQ       R8, R14
+	SHRQ       $0x02, R8, R15
+	SHRQ       $0x02, R8
+	ADDQ       R13, R10
+	ADCQ       R14, R11
+	ADCQ       $0x00, R12
+	ADDQ       R15, R10
+	ADCQ       R8, R11
+	ADCQ       $0x00, R12
+	LEAQ       32(DI), DI
+	VPALIGNR   $0x0c, Y14, Y14, Y14
+	VPALIGNR   $0x08, Y12, Y12, Y12
+	VPALIGNR   $0x04, Y4, Y4, Y4
+	DECQ       CX
+	JG         sealAVX2Tail128LoopA
+	DECQ       R9
+	JGE        sealAVX2Tail128LoopB
+	VPADDD     ·chacha20Constants<>+0(SB), Y0, Y5
+	VPADDD     32(BP), Y14, Y9
+	VPADDD     64(BP), Y12, Y13
+	VPADDD     Y1, Y4, Y1
+	VPERM2I128 $0x02, Y5, Y9, Y0
+	VPERM2I128 $0x02, Y13, Y1, Y14
+	VPERM2I128 $0x13, Y5, Y9, Y12
+	VPERM2I128 $0x13, Y13, Y1, Y4
 	JMP        sealAVX2ShortSealLoop
 
-// ----------------------------------------------------------------------------
-// Special optimization for the last 256 bytes of ciphertext
 sealAVX2Tail256:
-	// Need to decrypt up to 256 bytes - prepare two blocks
-	// If we got here after the main loop - there are 512 encrypted bytes waiting to be hashed
-	// If we got here before the main loop - there are 448 encrpyred bytes waiting to be hashed
-	VMOVDQA ·chacha20Constants<>(SB), AA0; VMOVDQA ·chacha20Constants<>(SB), AA1
-	VMOVDQA state1StoreAVX2, BB0; VMOVDQA state1StoreAVX2, BB1
-	VMOVDQA state2StoreAVX2, CC0; VMOVDQA state2StoreAVX2, CC1
-	VMOVDQA ctr3StoreAVX2, DD0
-	VPADDD  ·avx2IncMask<>(SB), DD0, DD0
-	VPADDD  ·avx2IncMask<>(SB), DD0, DD1
-	VMOVDQA DD0, TT1
-	VMOVDQA DD1, TT2
+	VMOVDQA ·chacha20Constants<>+0(SB), Y0
+	VMOVDQA ·chacha20Constants<>+0(SB), Y5
+	VMOVDQA 32(BP), Y14
+	VMOVDQA 32(BP), Y9
+	VMOVDQA 64(BP), Y12
+	VMOVDQA 64(BP), Y13
+	VMOVDQA 192(BP), Y4
+	VPADDD  ·avx2IncMask<>+0(SB), Y4, Y4
+	VPADDD  ·avx2IncMask<>+0(SB), Y4, Y1
+	VMOVDQA Y4, Y7
+	VMOVDQA Y1, Y11
 
 sealAVX2Tail256LoopA:
-	polyAdd(0(oup))
-	polyMul
-	LEAQ 16(oup), oup
+	ADDQ  (DI), R10
+	ADCQ  8(DI), R11
+	ADCQ  $0x01, R12
+	MOVQ  (BP), AX
+	MOVQ  AX, R15
+	MULQ  R10
+	MOVQ  AX, R13
+	MOVQ  DX, R14
+	MOVQ  (BP), AX
+	MULQ  R11
+	IMULQ R12, R15
+	ADDQ  AX, R14
+	ADCQ  DX, R15
+	MOVQ  8(BP), AX
+	MOVQ  AX, R8
+	MULQ  R10
+	ADDQ  AX, R14
+	ADCQ  $0x00, DX
+	MOVQ  DX, R10
+	MOVQ  8(BP), AX
+	MULQ  R11
+	ADDQ  AX, R15
+	ADCQ  $0x00, DX
+	IMULQ R12, R8
+	ADDQ  R10, R15
+	ADCQ  DX, R8
+	MOVQ  R13, R10
+	MOVQ  R14, R11
+	MOVQ  R15, R12
+	ANDQ  $0x03, R12
+	MOVQ  R15, R13
+	ANDQ  $-4, R13
+	MOVQ  R8, R14
+	SHRQ  $0x02, R8, R15
+	SHRQ  $0x02, R8
+	ADDQ  R13, R10
+	ADCQ  R14, R11
+	ADCQ  $0x00, R12
+	ADDQ  R15, R10
+	ADCQ  R8, R11
+	ADCQ  $0x00, R12
+	LEAQ  16(DI), DI
 
 sealAVX2Tail256LoopB:
-	chachaQR_AVX2(AA0, BB0, CC0, DD0, TT0); chachaQR_AVX2(AA1, BB1, CC1, DD1, TT0)
-	polyAdd(0(oup))
-	polyMul
-	VPALIGNR $4, BB0, BB0, BB0; VPALIGNR $4, BB1, BB1, BB1
-	VPALIGNR $8, CC0, CC0, CC0; VPALIGNR $8, CC1, CC1, CC1
-	VPALIGNR $12, DD0, DD0, DD0; VPALIGNR $12, DD1, DD1, DD1
-	chachaQR_AVX2(AA0, BB0, CC0, DD0, TT0); chachaQR_AVX2(AA1, BB1, CC1, DD1, TT0)
-	polyAdd(16(oup))
-	polyMul
-	LEAQ     32(oup), oup
-	VPALIGNR $12, BB0, BB0, BB0; VPALIGNR $12, BB1, BB1, BB1
-	VPALIGNR $8, CC0, CC0, CC0; VPALIGNR $8, CC1, CC1, CC1
-	VPALIGNR $4, DD0, DD0, DD0; VPALIGNR $4, DD1, DD1, DD1
-	DECQ     itr1
-	JG       sealAVX2Tail256LoopA
-	DECQ     itr2
-	JGE      sealAVX2Tail256LoopB
-
-	VPADDD     ·chacha20Constants<>(SB), AA0, AA0; VPADDD ·chacha20Constants<>(SB), AA1, AA1
-	VPADDD     state1StoreAVX2, BB0, BB0; VPADDD state1StoreAVX2, BB1, BB1
-	VPADDD     state2StoreAVX2, CC0, CC0; VPADDD state2StoreAVX2, CC1, CC1
-	VPADDD     TT1, DD0, DD0; VPADDD TT2, DD1, DD1
-	VPERM2I128 $0x02, AA0, BB0, TT0
-	VPERM2I128 $0x02, CC0, DD0, TT1
-	VPERM2I128 $0x13, AA0, BB0, TT2
-	VPERM2I128 $0x13, CC0, DD0, TT3
-	VPXOR      (0*32)(inp), TT0, TT0; VPXOR (1*32)(inp), TT1, TT1; VPXOR (2*32)(inp), TT2, TT2; VPXOR (3*32)(inp), TT3, TT3
-	VMOVDQU    TT0, (0*32)(oup); VMOVDQU TT1, (1*32)(oup); VMOVDQU TT2, (2*32)(oup); VMOVDQU TT3, (3*32)(oup)
-	MOVQ       $128, itr1
-	LEAQ       128(inp), inp
-	SUBQ       $128, inl
-	VPERM2I128 $0x02, AA1, BB1, AA0
-	VPERM2I128 $0x02, CC1, DD1, BB0
-	VPERM2I128 $0x13, AA1, BB1, CC0
-	VPERM2I128 $0x13, CC1, DD1, DD0
-
-	JMP sealAVX2SealHash
-
-// ----------------------------------------------------------------------------
-// Special optimization for the last 384 bytes of ciphertext
+	VPADDD     Y14, Y0, Y0
+	VPXOR      Y0, Y4, Y4
+	VPSHUFB    ·rol16<>+0(SB), Y4, Y4
+	VPADDD     Y4, Y12, Y12
+	VPXOR      Y12, Y14, Y14
+	VPSLLD     $0x0c, Y14, Y3
+	VPSRLD     $0x14, Y14, Y14
+	VPXOR      Y3, Y14, Y14
+	VPADDD     Y14, Y0, Y0
+	VPXOR      Y0, Y4, Y4
+	VPSHUFB    ·rol8<>+0(SB), Y4, Y4
+	VPADDD     Y4, Y12, Y12
+	VPXOR      Y12, Y14, Y14
+	VPSLLD     $0x07, Y14, Y3
+	VPSRLD     $0x19, Y14, Y14
+	VPXOR      Y3, Y14, Y14
+	VPADDD     Y9, Y5, Y5
+	VPXOR      Y5, Y1, Y1
+	VPSHUFB    ·rol16<>+0(SB), Y1, Y1
+	VPADDD     Y1, Y13, Y13
+	VPXOR      Y13, Y9, Y9
+	VPSLLD     $0x0c, Y9, Y3
+	VPSRLD     $0x14, Y9, Y9
+	VPXOR      Y3, Y9, Y9
+	VPADDD     Y9, Y5, Y5
+	VPXOR      Y5, Y1, Y1
+	VPSHUFB    ·rol8<>+0(SB), Y1, Y1
+	VPADDD     Y1, Y13, Y13
+	VPXOR      Y13, Y9, Y9
+	VPSLLD     $0x07, Y9, Y3
+	VPSRLD     $0x19, Y9, Y9
+	VPXOR      Y3, Y9, Y9
+	ADDQ       (DI), R10
+	ADCQ       8(DI), R11
+	ADCQ       $0x01, R12
+	MOVQ       (BP), AX
+	MOVQ       AX, R15
+	MULQ       R10
+	MOVQ       AX, R13
+	MOVQ       DX, R14
+	MOVQ       (BP), AX
+	MULQ       R11
+	IMULQ      R12, R15
+	ADDQ       AX, R14
+	ADCQ       DX, R15
+	MOVQ       8(BP), AX
+	MOVQ       AX, R8
+	MULQ       R10
+	ADDQ       AX, R14
+	ADCQ       $0x00, DX
+	MOVQ       DX, R10
+	MOVQ       8(BP), AX
+	MULQ       R11
+	ADDQ       AX, R15
+	ADCQ       $0x00, DX
+	IMULQ      R12, R8
+	ADDQ       R10, R15
+	ADCQ       DX, R8
+	MOVQ       R13, R10
+	MOVQ       R14, R11
+	MOVQ       R15, R12
+	ANDQ       $0x03, R12
+	MOVQ       R15, R13
+	ANDQ       $-4, R13
+	MOVQ       R8, R14
+	SHRQ       $0x02, R8, R15
+	SHRQ       $0x02, R8
+	ADDQ       R13, R10
+	ADCQ       R14, R11
+	ADCQ       $0x00, R12
+	ADDQ       R15, R10
+	ADCQ       R8, R11
+	ADCQ       $0x00, R12
+	VPALIGNR   $0x04, Y14, Y14, Y14
+	VPALIGNR   $0x04, Y9, Y9, Y9
+	VPALIGNR   $0x08, Y12, Y12, Y12
+	VPALIGNR   $0x08, Y13, Y13, Y13
+	VPALIGNR   $0x0c, Y4, Y4, Y4
+	VPALIGNR   $0x0c, Y1, Y1, Y1
+	VPADDD     Y14, Y0, Y0
+	VPXOR      Y0, Y4, Y4
+	VPSHUFB    ·rol16<>+0(SB), Y4, Y4
+	VPADDD     Y4, Y12, Y12
+	VPXOR      Y12, Y14, Y14
+	VPSLLD     $0x0c, Y14, Y3
+	VPSRLD     $0x14, Y14, Y14
+	VPXOR      Y3, Y14, Y14
+	VPADDD     Y14, Y0, Y0
+	VPXOR      Y0, Y4, Y4
+	VPSHUFB    ·rol8<>+0(SB), Y4, Y4
+	VPADDD     Y4, Y12, Y12
+	VPXOR      Y12, Y14, Y14
+	VPSLLD     $0x07, Y14, Y3
+	VPSRLD     $0x19, Y14, Y14
+	VPXOR      Y3, Y14, Y14
+	VPADDD     Y9, Y5, Y5
+	VPXOR      Y5, Y1, Y1
+	VPSHUFB    ·rol16<>+0(SB), Y1, Y1
+	VPADDD     Y1, Y13, Y13
+	VPXOR      Y13, Y9, Y9
+	VPSLLD     $0x0c, Y9, Y3
+	VPSRLD     $0x14, Y9, Y9
+	VPXOR      Y3, Y9, Y9
+	VPADDD     Y9, Y5, Y5
+	VPXOR      Y5, Y1, Y1
+	VPSHUFB    ·rol8<>+0(SB), Y1, Y1
+	VPADDD     Y1, Y13, Y13
+	VPXOR      Y13, Y9, Y9
+	VPSLLD     $0x07, Y9, Y3
+	VPSRLD     $0x19, Y9, Y9
+	VPXOR      Y3, Y9, Y9
+	ADDQ       16(DI), R10
+	ADCQ       24(DI), R11
+	ADCQ       $0x01, R12
+	MOVQ       (BP), AX
+	MOVQ       AX, R15
+	MULQ       R10
+	MOVQ       AX, R13
+	MOVQ       DX, R14
+	MOVQ       (BP), AX
+	MULQ       R11
+	IMULQ      R12, R15
+	ADDQ       AX, R14
+	ADCQ       DX, R15
+	MOVQ       8(BP), AX
+	MOVQ       AX, R8
+	MULQ       R10
+	ADDQ       AX, R14
+	ADCQ       $0x00, DX
+	MOVQ       DX, R10
+	MOVQ       8(BP), AX
+	MULQ       R11
+	ADDQ       AX, R15
+	ADCQ       $0x00, DX
+	IMULQ      R12, R8
+	ADDQ       R10, R15
+	ADCQ       DX, R8
+	MOVQ       R13, R10
+	MOVQ       R14, R11
+	MOVQ       R15, R12
+	ANDQ       $0x03, R12
+	MOVQ       R15, R13
+	ANDQ       $-4, R13
+	MOVQ       R8, R14
+	SHRQ       $0x02, R8, R15
+	SHRQ       $0x02, R8
+	ADDQ       R13, R10
+	ADCQ       R14, R11
+	ADCQ       $0x00, R12
+	ADDQ       R15, R10
+	ADCQ       R8, R11
+	ADCQ       $0x00, R12
+	LEAQ       32(DI), DI
+	VPALIGNR   $0x0c, Y14, Y14, Y14
+	VPALIGNR   $0x0c, Y9, Y9, Y9
+	VPALIGNR   $0x08, Y12, Y12, Y12
+	VPALIGNR   $0x08, Y13, Y13, Y13
+	VPALIGNR   $0x04, Y4, Y4, Y4
+	VPALIGNR   $0x04, Y1, Y1, Y1
+	DECQ       CX
+	JG         sealAVX2Tail256LoopA
+	DECQ       R9
+	JGE        sealAVX2Tail256LoopB
+	VPADDD     ·chacha20Constants<>+0(SB), Y0, Y0
+	VPADDD     ·chacha20Constants<>+0(SB), Y5, Y5
+	VPADDD     32(BP), Y14, Y14
+	VPADDD     32(BP), Y9, Y9
+	VPADDD     64(BP), Y12, Y12
+	VPADDD     64(BP), Y13, Y13
+	VPADDD     Y7, Y4, Y4
+	VPADDD     Y11, Y1, Y1
+	VPERM2I128 $0x02, Y0, Y14, Y3
+	VPERM2I128 $0x02, Y12, Y4, Y7
+	VPERM2I128 $0x13, Y0, Y14, Y11
+	VPERM2I128 $0x13, Y12, Y4, Y15
+	VPXOR      (SI), Y3, Y3
+	VPXOR      32(SI), Y7, Y7
+	VPXOR      64(SI), Y11, Y11
+	VPXOR      96(SI), Y15, Y15
+	VMOVDQU    Y3, (DI)
+	VMOVDQU    Y7, 32(DI)
+	VMOVDQU    Y11, 64(DI)
+	VMOVDQU    Y15, 96(DI)
+	MOVQ       $0x00000080, CX
+	LEAQ       128(SI), SI
+	SUBQ       $0x80, BX
+	VPERM2I128 $0x02, Y5, Y9, Y0
+	VPERM2I128 $0x02, Y13, Y1, Y14
+	VPERM2I128 $0x13, Y5, Y9, Y12
+	VPERM2I128 $0x13, Y13, Y1, Y4
+	JMP        sealAVX2SealHash
+
 sealAVX2Tail384:
-	// Need to decrypt up to 384 bytes - prepare two blocks
-	// If we got here after the main loop - there are 512 encrypted bytes waiting to be hashed
-	// If we got here before the main loop - there are 448 encrpyred bytes waiting to be hashed
-	VMOVDQA ·chacha20Constants<>(SB), AA0; VMOVDQA AA0, AA1; VMOVDQA AA0, AA2
-	VMOVDQA state1StoreAVX2, BB0; VMOVDQA BB0, BB1; VMOVDQA BB0, BB2
-	VMOVDQA state2StoreAVX2, CC0; VMOVDQA CC0, CC1; VMOVDQA CC0, CC2
-	VMOVDQA ctr3StoreAVX2, DD0
-	VPADDD  ·avx2IncMask<>(SB), DD0, DD0; VPADDD ·avx2IncMask<>(SB), DD0, DD1; VPADDD ·avx2IncMask<>(SB), DD1, DD2
-	VMOVDQA DD0, TT1; VMOVDQA DD1, TT2; VMOVDQA DD2, TT3
+	VMOVDQA ·chacha20Constants<>+0(SB), Y0
+	VMOVDQA Y0, Y5
+	VMOVDQA Y0, Y6
+	VMOVDQA 32(BP), Y14
+	VMOVDQA Y14, Y9
+	VMOVDQA Y14, Y10
+	VMOVDQA 64(BP), Y12
+	VMOVDQA Y12, Y13
+	VMOVDQA Y12, Y8
+	VMOVDQA 192(BP), Y4
+	VPADDD  ·avx2IncMask<>+0(SB), Y4, Y4
+	VPADDD  ·avx2IncMask<>+0(SB), Y4, Y1
+	VPADDD  ·avx2IncMask<>+0(SB), Y1, Y2
+	VMOVDQA Y4, Y7
+	VMOVDQA Y1, Y11
+	VMOVDQA Y2, Y15
 
 sealAVX2Tail384LoopA:
-	polyAdd(0(oup))
-	polyMul
-	LEAQ 16(oup), oup
+	ADDQ  (DI), R10
+	ADCQ  8(DI), R11
+	ADCQ  $0x01, R12
+	MOVQ  (BP), AX
+	MOVQ  AX, R15
+	MULQ  R10
+	MOVQ  AX, R13
+	MOVQ  DX, R14
+	MOVQ  (BP), AX
+	MULQ  R11
+	IMULQ R12, R15
+	ADDQ  AX, R14
+	ADCQ  DX, R15
+	MOVQ  8(BP), AX
+	MOVQ  AX, R8
+	MULQ  R10
+	ADDQ  AX, R14
+	ADCQ  $0x00, DX
+	MOVQ  DX, R10
+	MOVQ  8(BP), AX
+	MULQ  R11
+	ADDQ  AX, R15
+	ADCQ  $0x00, DX
+	IMULQ R12, R8
+	ADDQ  R10, R15
+	ADCQ  DX, R8
+	MOVQ  R13, R10
+	MOVQ  R14, R11
+	MOVQ  R15, R12
+	ANDQ  $0x03, R12
+	MOVQ  R15, R13
+	ANDQ  $-4, R13
+	MOVQ  R8, R14
+	SHRQ  $0x02, R8, R15
+	SHRQ  $0x02, R8
+	ADDQ  R13, R10
+	ADCQ  R14, R11
+	ADCQ  $0x00, R12
+	ADDQ  R15, R10
+	ADCQ  R8, R11
+	ADCQ  $0x00, R12
+	LEAQ  16(DI), DI
 
 sealAVX2Tail384LoopB:
-	chachaQR_AVX2(AA0, BB0, CC0, DD0, TT0); chachaQR_AVX2(AA1, BB1, CC1, DD1, TT0); chachaQR_AVX2(AA2, BB2, CC2, DD2, TT0)
-	polyAdd(0(oup))
-	polyMul
-	VPALIGNR $4, BB0, BB0, BB0; VPALIGNR $4, BB1, BB1, BB1; VPALIGNR $4, BB2, BB2, BB2
-	VPALIGNR $8, CC0, CC0, CC0; VPALIGNR $8, CC1, CC1, CC1; VPALIGNR $8, CC2, CC2, CC2
-	VPALIGNR $12, DD0, DD0, DD0; VPALIGNR $12, DD1, DD1, DD1; VPALIGNR $12, DD2, DD2, DD2
-	chachaQR_AVX2(AA0, BB0, CC0, DD0, TT0); chachaQR_AVX2(AA1, BB1, CC1, DD1, TT0); chachaQR_AVX2(AA2, BB2, CC2, DD2, TT0)
-	polyAdd(16(oup))
-	polyMul
-	LEAQ     32(oup), oup
-	VPALIGNR $12, BB0, BB0, BB0; VPALIGNR $12, BB1, BB1, BB1; VPALIGNR $12, BB2, BB2, BB2
-	VPALIGNR $8, CC0, CC0, CC0; VPALIGNR $8, CC1, CC1, CC1; VPALIGNR $8, CC2, CC2, CC2
-	VPALIGNR $4, DD0, DD0, DD0; VPALIGNR $4, DD1, DD1, DD1; VPALIGNR $4, DD2, DD2, DD2
-	DECQ     itr1
-	JG       sealAVX2Tail384LoopA
-	DECQ     itr2
-	JGE      sealAVX2Tail384LoopB
-
-	VPADDD     ·chacha20Constants<>(SB), AA0, AA0; VPADDD ·chacha20Constants<>(SB), AA1, AA1; VPADDD ·chacha20Constants<>(SB), AA2, AA2
-	VPADDD     state1StoreAVX2, BB0, BB0; VPADDD state1StoreAVX2, BB1, BB1; VPADDD state1StoreAVX2, BB2, BB2
-	VPADDD     state2StoreAVX2, CC0, CC0; VPADDD state2StoreAVX2, CC1, CC1; VPADDD state2StoreAVX2, CC2, CC2
-	VPADDD     TT1, DD0, DD0; VPADDD TT2, DD1, DD1; VPADDD TT3, DD2, DD2
-	VPERM2I128 $0x02, AA0, BB0, TT0
-	VPERM2I128 $0x02, CC0, DD0, TT1
-	VPERM2I128 $0x13, AA0, BB0, TT2
-	VPERM2I128 $0x13, CC0, DD0, TT3
-	VPXOR      (0*32)(inp), TT0, TT0; VPXOR (1*32)(inp), TT1, TT1; VPXOR (2*32)(inp), TT2, TT2; VPXOR (3*32)(inp), TT3, TT3
-	VMOVDQU    TT0, (0*32)(oup); VMOVDQU TT1, (1*32)(oup); VMOVDQU TT2, (2*32)(oup); VMOVDQU TT3, (3*32)(oup)
-	VPERM2I128 $0x02, AA1, BB1, TT0
-	VPERM2I128 $0x02, CC1, DD1, TT1
-	VPERM2I128 $0x13, AA1, BB1, TT2
-	VPERM2I128 $0x13, CC1, DD1, TT3
-	VPXOR      (4*32)(inp), TT0, TT0; VPXOR (5*32)(inp), TT1, TT1; VPXOR (6*32)(inp), TT2, TT2; VPXOR (7*32)(inp), TT3, TT3
-	VMOVDQU    TT0, (4*32)(oup); VMOVDQU TT1, (5*32)(oup); VMOVDQU TT2, (6*32)(oup); VMOVDQU TT3, (7*32)(oup)
-	MOVQ       $256, itr1
-	LEAQ       256(inp), inp
-	SUBQ       $256, inl
-	VPERM2I128 $0x02, AA2, BB2, AA0
-	VPERM2I128 $0x02, CC2, DD2, BB0
-	VPERM2I128 $0x13, AA2, BB2, CC0
-	VPERM2I128 $0x13, CC2, DD2, DD0
-
-	JMP sealAVX2SealHash
-
-// ----------------------------------------------------------------------------
-// Special optimization for the last 512 bytes of ciphertext
+	VPADDD     Y14, Y0, Y0
+	VPXOR      Y0, Y4, Y4
+	VPSHUFB    ·rol16<>+0(SB), Y4, Y4
+	VPADDD     Y4, Y12, Y12
+	VPXOR      Y12, Y14, Y14
+	VPSLLD     $0x0c, Y14, Y3
+	VPSRLD     $0x14, Y14, Y14
+	VPXOR      Y3, Y14, Y14
+	VPADDD     Y14, Y0, Y0
+	VPXOR      Y0, Y4, Y4
+	VPSHUFB    ·rol8<>+0(SB), Y4, Y4
+	VPADDD     Y4, Y12, Y12
+	VPXOR      Y12, Y14, Y14
+	VPSLLD     $0x07, Y14, Y3
+	VPSRLD     $0x19, Y14, Y14
+	VPXOR      Y3, Y14, Y14
+	VPADDD     Y9, Y5, Y5
+	VPXOR      Y5, Y1, Y1
+	VPSHUFB    ·rol16<>+0(SB), Y1, Y1
+	VPADDD     Y1, Y13, Y13
+	VPXOR      Y13, Y9, Y9
+	VPSLLD     $0x0c, Y9, Y3
+	VPSRLD     $0x14, Y9, Y9
+	VPXOR      Y3, Y9, Y9
+	VPADDD     Y9, Y5, Y5
+	VPXOR      Y5, Y1, Y1
+	VPSHUFB    ·rol8<>+0(SB), Y1, Y1
+	VPADDD     Y1, Y13, Y13
+	VPXOR      Y13, Y9, Y9
+	VPSLLD     $0x07, Y9, Y3
+	VPSRLD     $0x19, Y9, Y9
+	VPXOR      Y3, Y9, Y9
+	VPADDD     Y10, Y6, Y6
+	VPXOR      Y6, Y2, Y2
+	VPSHUFB    ·rol16<>+0(SB), Y2, Y2
+	VPADDD     Y2, Y8, Y8
+	VPXOR      Y8, Y10, Y10
+	VPSLLD     $0x0c, Y10, Y3
+	VPSRLD     $0x14, Y10, Y10
+	VPXOR      Y3, Y10, Y10
+	VPADDD     Y10, Y6, Y6
+	VPXOR      Y6, Y2, Y2
+	VPSHUFB    ·rol8<>+0(SB), Y2, Y2
+	VPADDD     Y2, Y8, Y8
+	VPXOR      Y8, Y10, Y10
+	VPSLLD     $0x07, Y10, Y3
+	VPSRLD     $0x19, Y10, Y10
+	VPXOR      Y3, Y10, Y10
+	ADDQ       (DI), R10
+	ADCQ       8(DI), R11
+	ADCQ       $0x01, R12
+	MOVQ       (BP), AX
+	MOVQ       AX, R15
+	MULQ       R10
+	MOVQ       AX, R13
+	MOVQ       DX, R14
+	MOVQ       (BP), AX
+	MULQ       R11
+	IMULQ      R12, R15
+	ADDQ       AX, R14
+	ADCQ       DX, R15
+	MOVQ       8(BP), AX
+	MOVQ       AX, R8
+	MULQ       R10
+	ADDQ       AX, R14
+	ADCQ       $0x00, DX
+	MOVQ       DX, R10
+	MOVQ       8(BP), AX
+	MULQ       R11
+	ADDQ       AX, R15
+	ADCQ       $0x00, DX
+	IMULQ      R12, R8
+	ADDQ       R10, R15
+	ADCQ       DX, R8
+	MOVQ       R13, R10
+	MOVQ       R14, R11
+	MOVQ       R15, R12
+	ANDQ       $0x03, R12
+	MOVQ       R15, R13
+	ANDQ       $-4, R13
+	MOVQ       R8, R14
+	SHRQ       $0x02, R8, R15
+	SHRQ       $0x02, R8
+	ADDQ       R13, R10
+	ADCQ       R14, R11
+	ADCQ       $0x00, R12
+	ADDQ       R15, R10
+	ADCQ       R8, R11
+	ADCQ       $0x00, R12
+	VPALIGNR   $0x04, Y14, Y14, Y14
+	VPALIGNR   $0x04, Y9, Y9, Y9
+	VPALIGNR   $0x04, Y10, Y10, Y10
+	VPALIGNR   $0x08, Y12, Y12, Y12
+	VPALIGNR   $0x08, Y13, Y13, Y13
+	VPALIGNR   $0x08, Y8, Y8, Y8
+	VPALIGNR   $0x0c, Y4, Y4, Y4
+	VPALIGNR   $0x0c, Y1, Y1, Y1
+	VPALIGNR   $0x0c, Y2, Y2, Y2
+	VPADDD     Y14, Y0, Y0
+	VPXOR      Y0, Y4, Y4
+	VPSHUFB    ·rol16<>+0(SB), Y4, Y4
+	VPADDD     Y4, Y12, Y12
+	VPXOR      Y12, Y14, Y14
+	VPSLLD     $0x0c, Y14, Y3
+	VPSRLD     $0x14, Y14, Y14
+	VPXOR      Y3, Y14, Y14
+	VPADDD     Y14, Y0, Y0
+	VPXOR      Y0, Y4, Y4
+	VPSHUFB    ·rol8<>+0(SB), Y4, Y4
+	VPADDD     Y4, Y12, Y12
+	VPXOR      Y12, Y14, Y14
+	VPSLLD     $0x07, Y14, Y3
+	VPSRLD     $0x19, Y14, Y14
+	VPXOR      Y3, Y14, Y14
+	VPADDD     Y9, Y5, Y5
+	VPXOR      Y5, Y1, Y1
+	VPSHUFB    ·rol16<>+0(SB), Y1, Y1
+	VPADDD     Y1, Y13, Y13
+	VPXOR      Y13, Y9, Y9
+	VPSLLD     $0x0c, Y9, Y3
+	VPSRLD     $0x14, Y9, Y9
+	VPXOR      Y3, Y9, Y9
+	VPADDD     Y9, Y5, Y5
+	VPXOR      Y5, Y1, Y1
+	VPSHUFB    ·rol8<>+0(SB), Y1, Y1
+	VPADDD     Y1, Y13, Y13
+	VPXOR      Y13, Y9, Y9
+	VPSLLD     $0x07, Y9, Y3
+	VPSRLD     $0x19, Y9, Y9
+	VPXOR      Y3, Y9, Y9
+	VPADDD     Y10, Y6, Y6
+	VPXOR      Y6, Y2, Y2
+	VPSHUFB    ·rol16<>+0(SB), Y2, Y2
+	VPADDD     Y2, Y8, Y8
+	VPXOR      Y8, Y10, Y10
+	VPSLLD     $0x0c, Y10, Y3
+	VPSRLD     $0x14, Y10, Y10
+	VPXOR      Y3, Y10, Y10
+	VPADDD     Y10, Y6, Y6
+	VPXOR      Y6, Y2, Y2
+	VPSHUFB    ·rol8<>+0(SB), Y2, Y2
+	VPADDD     Y2, Y8, Y8
+	VPXOR      Y8, Y10, Y10
+	VPSLLD     $0x07, Y10, Y3
+	VPSRLD     $0x19, Y10, Y10
+	VPXOR      Y3, Y10, Y10
+	ADDQ       16(DI), R10
+	ADCQ       24(DI), R11
+	ADCQ       $0x01, R12
+	MOVQ       (BP), AX
+	MOVQ       AX, R15
+	MULQ       R10
+	MOVQ       AX, R13
+	MOVQ       DX, R14
+	MOVQ       (BP), AX
+	MULQ       R11
+	IMULQ      R12, R15
+	ADDQ       AX, R14
+	ADCQ       DX, R15
+	MOVQ       8(BP), AX
+	MOVQ       AX, R8
+	MULQ       R10
+	ADDQ       AX, R14
+	ADCQ       $0x00, DX
+	MOVQ       DX, R10
+	MOVQ       8(BP), AX
+	MULQ       R11
+	ADDQ       AX, R15
+	ADCQ       $0x00, DX
+	IMULQ      R12, R8
+	ADDQ       R10, R15
+	ADCQ       DX, R8
+	MOVQ       R13, R10
+	MOVQ       R14, R11
+	MOVQ       R15, R12
+	ANDQ       $0x03, R12
+	MOVQ       R15, R13
+	ANDQ       $-4, R13
+	MOVQ       R8, R14
+	SHRQ       $0x02, R8, R15
+	SHRQ       $0x02, R8
+	ADDQ       R13, R10
+	ADCQ       R14, R11
+	ADCQ       $0x00, R12
+	ADDQ       R15, R10
+	ADCQ       R8, R11
+	ADCQ       $0x00, R12
+	LEAQ       32(DI), DI
+	VPALIGNR   $0x0c, Y14, Y14, Y14
+	VPALIGNR   $0x0c, Y9, Y9, Y9
+	VPALIGNR   $0x0c, Y10, Y10, Y10
+	VPALIGNR   $0x08, Y12, Y12, Y12
+	VPALIGNR   $0x08, Y13, Y13, Y13
+	VPALIGNR   $0x08, Y8, Y8, Y8
+	VPALIGNR   $0x04, Y4, Y4, Y4
+	VPALIGNR   $0x04, Y1, Y1, Y1
+	VPALIGNR   $0x04, Y2, Y2, Y2
+	DECQ       CX
+	JG         sealAVX2Tail384LoopA
+	DECQ       R9
+	JGE        sealAVX2Tail384LoopB
+	VPADDD     ·chacha20Constants<>+0(SB), Y0, Y0
+	VPADDD     ·chacha20Constants<>+0(SB), Y5, Y5
+	VPADDD     ·chacha20Constants<>+0(SB), Y6, Y6
+	VPADDD     32(BP), Y14, Y14
+	VPADDD     32(BP), Y9, Y9
+	VPADDD     32(BP), Y10, Y10
+	VPADDD     64(BP), Y12, Y12
+	VPADDD     64(BP), Y13, Y13
+	VPADDD     64(BP), Y8, Y8
+	VPADDD     Y7, Y4, Y4
+	VPADDD     Y11, Y1, Y1
+	VPADDD     Y15, Y2, Y2
+	VPERM2I128 $0x02, Y0, Y14, Y3
+	VPERM2I128 $0x02, Y12, Y4, Y7
+	VPERM2I128 $0x13, Y0, Y14, Y11
+	VPERM2I128 $0x13, Y12, Y4, Y15
+	VPXOR      (SI), Y3, Y3
+	VPXOR      32(SI), Y7, Y7
+	VPXOR      64(SI), Y11, Y11
+	VPXOR      96(SI), Y15, Y15
+	VMOVDQU    Y3, (DI)
+	VMOVDQU    Y7, 32(DI)
+	VMOVDQU    Y11, 64(DI)
+	VMOVDQU    Y15, 96(DI)
+	VPERM2I128 $0x02, Y5, Y9, Y3
+	VPERM2I128 $0x02, Y13, Y1, Y7
+	VPERM2I128 $0x13, Y5, Y9, Y11
+	VPERM2I128 $0x13, Y13, Y1, Y15
+	VPXOR      128(SI), Y3, Y3
+	VPXOR      160(SI), Y7, Y7
+	VPXOR      192(SI), Y11, Y11
+	VPXOR      224(SI), Y15, Y15
+	VMOVDQU    Y3, 128(DI)
+	VMOVDQU    Y7, 160(DI)
+	VMOVDQU    Y11, 192(DI)
+	VMOVDQU    Y15, 224(DI)
+	MOVQ       $0x00000100, CX
+	LEAQ       256(SI), SI
+	SUBQ       $0x00000100, BX
+	VPERM2I128 $0x02, Y6, Y10, Y0
+	VPERM2I128 $0x02, Y8, Y2, Y14
+	VPERM2I128 $0x13, Y6, Y10, Y12
+	VPERM2I128 $0x13, Y8, Y2, Y4
+	JMP        sealAVX2SealHash
+
 sealAVX2Tail512:
-	// Need to decrypt up to 512 bytes - prepare two blocks
-	// If we got here after the main loop - there are 512 encrypted bytes waiting to be hashed
-	// If we got here before the main loop - there are 448 encrpyred bytes waiting to be hashed
-	VMOVDQA ·chacha20Constants<>(SB), AA0; VMOVDQA AA0, AA1; VMOVDQA AA0, AA2; VMOVDQA AA0, AA3
-	VMOVDQA state1StoreAVX2, BB0; VMOVDQA BB0, BB1; VMOVDQA BB0, BB2; VMOVDQA BB0, BB3
-	VMOVDQA state2StoreAVX2, CC0; VMOVDQA CC0, CC1; VMOVDQA CC0, CC2; VMOVDQA CC0, CC3
-	VMOVDQA ctr3StoreAVX2, DD0
-	VPADDD  ·avx2IncMask<>(SB), DD0, DD0; VPADDD ·avx2IncMask<>(SB), DD0, DD1; VPADDD ·avx2IncMask<>(SB), DD1, DD2; VPADDD ·avx2IncMask<>(SB), DD2, DD3
-	VMOVDQA DD0, ctr0StoreAVX2; VMOVDQA DD1, ctr1StoreAVX2; VMOVDQA DD2, ctr2StoreAVX2; VMOVDQA DD3, ctr3StoreAVX2
+	VMOVDQA ·chacha20Constants<>+0(SB), Y0
+	VMOVDQA Y0, Y5
+	VMOVDQA Y0, Y6
+	VMOVDQA Y0, Y7
+	VMOVDQA 32(BP), Y14
+	VMOVDQA Y14, Y9
+	VMOVDQA Y14, Y10
+	VMOVDQA Y14, Y11
+	VMOVDQA 64(BP), Y12
+	VMOVDQA Y12, Y13
+	VMOVDQA Y12, Y8
+	VMOVDQA Y12, Y15
+	VMOVDQA 192(BP), Y4
+	VPADDD  ·avx2IncMask<>+0(SB), Y4, Y4
+	VPADDD  ·avx2IncMask<>+0(SB), Y4, Y1
+	VPADDD  ·avx2IncMask<>+0(SB), Y1, Y2
+	VPADDD  ·avx2IncMask<>+0(SB), Y2, Y3
+	VMOVDQA Y4, 96(BP)
+	VMOVDQA Y1, 128(BP)
+	VMOVDQA Y2, 160(BP)
+	VMOVDQA Y3, 192(BP)
 
 sealAVX2Tail512LoopA:
-	polyAdd(0(oup))
-	polyMul
-	LEAQ 16(oup), oup
+	ADDQ  (DI), R10
+	ADCQ  8(DI), R11
+	ADCQ  $0x01, R12
+	MOVQ  (BP), AX
+	MOVQ  AX, R15
+	MULQ  R10
+	MOVQ  AX, R13
+	MOVQ  DX, R14
+	MOVQ  (BP), AX
+	MULQ  R11
+	IMULQ R12, R15
+	ADDQ  AX, R14
+	ADCQ  DX, R15
+	MOVQ  8(BP), AX
+	MOVQ  AX, R8
+	MULQ  R10
+	ADDQ  AX, R14
+	ADCQ  $0x00, DX
+	MOVQ  DX, R10
+	MOVQ  8(BP), AX
+	MULQ  R11
+	ADDQ  AX, R15
+	ADCQ  $0x00, DX
+	IMULQ R12, R8
+	ADDQ  R10, R15
+	ADCQ  DX, R8
+	MOVQ  R13, R10
+	MOVQ  R14, R11
+	MOVQ  R15, R12
+	ANDQ  $0x03, R12
+	MOVQ  R15, R13
+	ANDQ  $-4, R13
+	MOVQ  R8, R14
+	SHRQ  $0x02, R8, R15
+	SHRQ  $0x02, R8
+	ADDQ  R13, R10
+	ADCQ  R14, R11
+	ADCQ  $0x00, R12
+	ADDQ  R15, R10
+	ADCQ  R8, R11
+	ADCQ  $0x00, R12
+	LEAQ  16(DI), DI
 
 sealAVX2Tail512LoopB:
-	VPADDD   BB0, AA0, AA0; VPADDD BB1, AA1, AA1; VPADDD BB2, AA2, AA2; VPADDD BB3, AA3, AA3
-	VPXOR    AA0, DD0, DD0; VPXOR AA1, DD1, DD1; VPXOR AA2, DD2, DD2; VPXOR AA3, DD3, DD3
-	VPSHUFB  ·rol16<>(SB), DD0, DD0; VPSHUFB ·rol16<>(SB), DD1, DD1; VPSHUFB ·rol16<>(SB), DD2, DD2; VPSHUFB ·rol16<>(SB), DD3, DD3
-	VPADDD   DD0, CC0, CC0; VPADDD DD1, CC1, CC1; VPADDD DD2, CC2, CC2; VPADDD DD3, CC3, CC3
-	VPXOR    CC0, BB0, BB0; VPXOR CC1, BB1, BB1; VPXOR CC2, BB2, BB2; VPXOR CC3, BB3, BB3
-	VMOVDQA  CC3, tmpStoreAVX2
-	VPSLLD   $12, BB0, CC3; VPSRLD $20, BB0, BB0; VPXOR CC3, BB0, BB0
-	VPSLLD   $12, BB1, CC3; VPSRLD $20, BB1, BB1; VPXOR CC3, BB1, BB1
-	VPSLLD   $12, BB2, CC3; VPSRLD $20, BB2, BB2; VPXOR CC3, BB2, BB2
-	VPSLLD   $12, BB3, CC3; VPSRLD $20, BB3, BB3; VPXOR CC3, BB3, BB3
-	VMOVDQA  tmpStoreAVX2, CC3
-	polyAdd(0*8(oup))
-	polyMulAVX2
-	VPADDD   BB0, AA0, AA0; VPADDD BB1, AA1, AA1; VPADDD BB2, AA2, AA2; VPADDD BB3, AA3, AA3
-	VPXOR    AA0, DD0, DD0; VPXOR AA1, DD1, DD1; VPXOR AA2, DD2, DD2; VPXOR AA3, DD3, DD3
-	VPSHUFB  ·rol8<>(SB), DD0, DD0; VPSHUFB ·rol8<>(SB), DD1, DD1; VPSHUFB ·rol8<>(SB), DD2, DD2; VPSHUFB ·rol8<>(SB), DD3, DD3
-	VPADDD   DD0, CC0, CC0; VPADDD DD1, CC1, CC1; VPADDD DD2, CC2, CC2; VPADDD DD3, CC3, CC3
-	VPXOR    CC0, BB0, BB0; VPXOR CC1, BB1, BB1; VPXOR CC2, BB2, BB2; VPXOR CC3, BB3, BB3
-	VMOVDQA  CC3, tmpStoreAVX2
-	VPSLLD   $7, BB0, CC3; VPSRLD $25, BB0, BB0; VPXOR CC3, BB0, BB0
-	VPSLLD   $7, BB1, CC3; VPSRLD $25, BB1, BB1; VPXOR CC3, BB1, BB1
-	VPSLLD   $7, BB2, CC3; VPSRLD $25, BB2, BB2; VPXOR CC3, BB2, BB2
-	VPSLLD   $7, BB3, CC3; VPSRLD $25, BB3, BB3; VPXOR CC3, BB3, BB3
-	VMOVDQA  tmpStoreAVX2, CC3
-	VPALIGNR $4, BB0, BB0, BB0; VPALIGNR $4, BB1, BB1, BB1; VPALIGNR $4, BB2, BB2, BB2; VPALIGNR $4, BB3, BB3, BB3
-	VPALIGNR $8, CC0, CC0, CC0; VPALIGNR $8, CC1, CC1, CC1; VPALIGNR $8, CC2, CC2, CC2; VPALIGNR $8, CC3, CC3, CC3
-	VPALIGNR $12, DD0, DD0, DD0; VPALIGNR $12, DD1, DD1, DD1; VPALIGNR $12, DD2, DD2, DD2; VPALIGNR $12, DD3, DD3, DD3
-	VPADDD   BB0, AA0, AA0; VPADDD BB1, AA1, AA1; VPADDD BB2, AA2, AA2; VPADDD BB3, AA3, AA3
-	VPXOR    AA0, DD0, DD0; VPXOR AA1, DD1, DD1; VPXOR AA2, DD2, DD2; VPXOR AA3, DD3, DD3
-	VPSHUFB  ·rol16<>(SB), DD0, DD0; VPSHUFB ·rol16<>(SB), DD1, DD1; VPSHUFB ·rol16<>(SB), DD2, DD2; VPSHUFB ·rol16<>(SB), DD3, DD3
-	VPADDD   DD0, CC0, CC0; VPADDD DD1, CC1, CC1; VPADDD DD2, CC2, CC2; VPADDD DD3, CC3, CC3
-	VPXOR    CC0, BB0, BB0; VPXOR CC1, BB1, BB1; VPXOR CC2, BB2, BB2; VPXOR CC3, BB3, BB3
-	polyAdd(2*8(oup))
-	polyMulAVX2
-	LEAQ     (4*8)(oup), oup
-	VMOVDQA  CC3, tmpStoreAVX2
-	VPSLLD   $12, BB0, CC3; VPSRLD $20, BB0, BB0; VPXOR CC3, BB0, BB0
-	VPSLLD   $12, BB1, CC3; VPSRLD $20, BB1, BB1; VPXOR CC3, BB1, BB1
-	VPSLLD   $12, BB2, CC3; VPSRLD $20, BB2, BB2; VPXOR CC3, BB2, BB2
-	VPSLLD   $12, BB3, CC3; VPSRLD $20, BB3, BB3; VPXOR CC3, BB3, BB3
-	VMOVDQA  tmpStoreAVX2, CC3
-	VPADDD   BB0, AA0, AA0; VPADDD BB1, AA1, AA1; VPADDD BB2, AA2, AA2; VPADDD BB3, AA3, AA3
-	VPXOR    AA0, DD0, DD0; VPXOR AA1, DD1, DD1; VPXOR AA2, DD2, DD2; VPXOR AA3, DD3, DD3
-	VPSHUFB  ·rol8<>(SB), DD0, DD0; VPSHUFB ·rol8<>(SB), DD1, DD1; VPSHUFB ·rol8<>(SB), DD2, DD2; VPSHUFB ·rol8<>(SB), DD3, DD3
-	VPADDD   DD0, CC0, CC0; VPADDD DD1, CC1, CC1; VPADDD DD2, CC2, CC2; VPADDD DD3, CC3, CC3
-	VPXOR    CC0, BB0, BB0; VPXOR CC1, BB1, BB1; VPXOR CC2, BB2, BB2; VPXOR CC3, BB3, BB3
-	VMOVDQA  CC3, tmpStoreAVX2
-	VPSLLD   $7, BB0, CC3; VPSRLD $25, BB0, BB0; VPXOR CC3, BB0, BB0
-	VPSLLD   $7, BB1, CC3; VPSRLD $25, BB1, BB1; VPXOR CC3, BB1, BB1
-	VPSLLD   $7, BB2, CC3; VPSRLD $25, BB2, BB2; VPXOR CC3, BB2, BB2
-	VPSLLD   $7, BB3, CC3; VPSRLD $25, BB3, BB3; VPXOR CC3, BB3, BB3
-	VMOVDQA  tmpStoreAVX2, CC3
-	VPALIGNR $12, BB0, BB0, BB0; VPALIGNR $12, BB1, BB1, BB1; VPALIGNR $12, BB2, BB2, BB2; VPALIGNR $12, BB3, BB3, BB3
-	VPALIGNR $8, CC0, CC0, CC0; VPALIGNR $8, CC1, CC1, CC1; VPALIGNR $8, CC2, CC2, CC2; VPALIGNR $8, CC3, CC3, CC3
-	VPALIGNR $4, DD0, DD0, DD0; VPALIGNR $4, DD1, DD1, DD1; VPALIGNR $4, DD2, DD2, DD2; VPALIGNR $4, DD3, DD3, DD3
-
-	DECQ itr1
-	JG   sealAVX2Tail512LoopA
-	DECQ itr2
-	JGE  sealAVX2Tail512LoopB
-
-	VPADDD     ·chacha20Constants<>(SB), AA0, AA0; VPADDD ·chacha20Constants<>(SB), AA1, AA1; VPADDD ·chacha20Constants<>(SB), AA2, AA2; VPADDD ·chacha20Constants<>(SB), AA3, AA3
-	VPADDD     state1StoreAVX2, BB0, BB0; VPADDD state1StoreAVX2, BB1, BB1; VPADDD state1StoreAVX2, BB2, BB2; VPADDD state1StoreAVX2, BB3, BB3
-	VPADDD     state2StoreAVX2, CC0, CC0; VPADDD state2StoreAVX2, CC1, CC1; VPADDD state2StoreAVX2, CC2, CC2; VPADDD state2StoreAVX2, CC3, CC3
-	VPADDD     ctr0StoreAVX2, DD0, DD0; VPADDD ctr1StoreAVX2, DD1, DD1; VPADDD ctr2StoreAVX2, DD2, DD2; VPADDD ctr3StoreAVX2, DD3, DD3
-	VMOVDQA    CC3, tmpStoreAVX2
-	VPERM2I128 $0x02, AA0, BB0, CC3
-	VPXOR      (0*32)(inp), CC3, CC3
-	VMOVDQU    CC3, (0*32)(oup)
-	VPERM2I128 $0x02, CC0, DD0, CC3
-	VPXOR      (1*32)(inp), CC3, CC3
-	VMOVDQU    CC3, (1*32)(oup)
-	VPERM2I128 $0x13, AA0, BB0, CC3
-	VPXOR      (2*32)(inp), CC3, CC3
-	VMOVDQU    CC3, (2*32)(oup)
-	VPERM2I128 $0x13, CC0, DD0, CC3
-	VPXOR      (3*32)(inp), CC3, CC3
-	VMOVDQU    CC3, (3*32)(oup)
-
-	VPERM2I128 $0x02, AA1, BB1, AA0
-	VPERM2I128 $0x02, CC1, DD1, BB0
-	VPERM2I128 $0x13, AA1, BB1, CC0
-	VPERM2I128 $0x13, CC1, DD1, DD0
-	VPXOR      (4*32)(inp), AA0, AA0; VPXOR (5*32)(inp), BB0, BB0; VPXOR (6*32)(inp), CC0, CC0; VPXOR (7*32)(inp), DD0, DD0
-	VMOVDQU    AA0, (4*32)(oup); VMOVDQU BB0, (5*32)(oup); VMOVDQU CC0, (6*32)(oup); VMOVDQU DD0, (7*32)(oup)
-
-	VPERM2I128 $0x02, AA2, BB2, AA0
-	VPERM2I128 $0x02, CC2, DD2, BB0
-	VPERM2I128 $0x13, AA2, BB2, CC0
-	VPERM2I128 $0x13, CC2, DD2, DD0
-	VPXOR      (8*32)(inp), AA0, AA0; VPXOR (9*32)(inp), BB0, BB0; VPXOR (10*32)(inp), CC0, CC0; VPXOR (11*32)(inp), DD0, DD0
-	VMOVDQU    AA0, (8*32)(oup); VMOVDQU BB0, (9*32)(oup); VMOVDQU CC0, (10*32)(oup); VMOVDQU DD0, (11*32)(oup)
-
-	MOVQ       $384, itr1
-	LEAQ       384(inp), inp
-	SUBQ       $384, inl
-	VPERM2I128 $0x02, AA3, BB3, AA0
-	VPERM2I128 $0x02, tmpStoreAVX2, DD3, BB0
-	VPERM2I128 $0x13, AA3, BB3, CC0
-	VPERM2I128 $0x13, tmpStoreAVX2, DD3, DD0
-
-	JMP sealAVX2SealHash
+	VPADDD     Y14, Y0, Y0
+	VPADDD     Y9, Y5, Y5
+	VPADDD     Y10, Y6, Y6
+	VPADDD     Y11, Y7, Y7
+	VPXOR      Y0, Y4, Y4
+	VPXOR      Y5, Y1, Y1
+	VPXOR      Y6, Y2, Y2
+	VPXOR      Y7, Y3, Y3
+	VPSHUFB    ·rol16<>+0(SB), Y4, Y4
+	VPSHUFB    ·rol16<>+0(SB), Y1, Y1
+	VPSHUFB    ·rol16<>+0(SB), Y2, Y2
+	VPSHUFB    ·rol16<>+0(SB), Y3, Y3
+	VPADDD     Y4, Y12, Y12
+	VPADDD     Y1, Y13, Y13
+	VPADDD     Y2, Y8, Y8
+	VPADDD     Y3, Y15, Y15
+	VPXOR      Y12, Y14, Y14
+	VPXOR      Y13, Y9, Y9
+	VPXOR      Y8, Y10, Y10
+	VPXOR      Y15, Y11, Y11
+	VMOVDQA    Y15, 224(BP)
+	VPSLLD     $0x0c, Y14, Y15
+	VPSRLD     $0x14, Y14, Y14
+	VPXOR      Y15, Y14, Y14
+	VPSLLD     $0x0c, Y9, Y15
+	VPSRLD     $0x14, Y9, Y9
+	VPXOR      Y15, Y9, Y9
+	VPSLLD     $0x0c, Y10, Y15
+	VPSRLD     $0x14, Y10, Y10
+	VPXOR      Y15, Y10, Y10
+	VPSLLD     $0x0c, Y11, Y15
+	VPSRLD     $0x14, Y11, Y11
+	VPXOR      Y15, Y11, Y11
+	VMOVDQA    224(BP), Y15
+	ADDQ       (DI), R10
+	ADCQ       8(DI), R11
+	ADCQ       $0x01, R12
+	MOVQ       (BP), DX
+	MOVQ       DX, R15
+	MULXQ      R10, R13, R14
+	IMULQ      R12, R15
+	MULXQ      R11, AX, DX
+	ADDQ       AX, R14
+	ADCQ       DX, R15
+	MOVQ       8(BP), DX
+	MULXQ      R10, R10, AX
+	ADDQ       R10, R14
+	MULXQ      R11, R11, R8
+	ADCQ       R11, R15
+	ADCQ       $0x00, R8
+	IMULQ      R12, DX
+	ADDQ       AX, R15
+	ADCQ       DX, R8
+	MOVQ       R13, R10
+	MOVQ       R14, R11
+	MOVQ       R15, R12
+	ANDQ       $0x03, R12
+	MOVQ       R15, R13
+	ANDQ       $-4, R13
+	MOVQ       R8, R14
+	SHRQ       $0x02, R8, R15
+	SHRQ       $0x02, R8
+	ADDQ       R13, R10
+	ADCQ       R14, R11
+	ADCQ       $0x00, R12
+	ADDQ       R15, R10
+	ADCQ       R8, R11
+	ADCQ       $0x00, R12
+	VPADDD     Y14, Y0, Y0
+	VPADDD     Y9, Y5, Y5
+	VPADDD     Y10, Y6, Y6
+	VPADDD     Y11, Y7, Y7
+	VPXOR      Y0, Y4, Y4
+	VPXOR      Y5, Y1, Y1
+	VPXOR      Y6, Y2, Y2
+	VPXOR      Y7, Y3, Y3
+	VPSHUFB    ·rol8<>+0(SB), Y4, Y4
+	VPSHUFB    ·rol8<>+0(SB), Y1, Y1
+	VPSHUFB    ·rol8<>+0(SB), Y2, Y2
+	VPSHUFB    ·rol8<>+0(SB), Y3, Y3
+	VPADDD     Y4, Y12, Y12
+	VPADDD     Y1, Y13, Y13
+	VPADDD     Y2, Y8, Y8
+	VPADDD     Y3, Y15, Y15
+	VPXOR      Y12, Y14, Y14
+	VPXOR      Y13, Y9, Y9
+	VPXOR      Y8, Y10, Y10
+	VPXOR      Y15, Y11, Y11
+	VMOVDQA    Y15, 224(BP)
+	VPSLLD     $0x07, Y14, Y15
+	VPSRLD     $0x19, Y14, Y14
+	VPXOR      Y15, Y14, Y14
+	VPSLLD     $0x07, Y9, Y15
+	VPSRLD     $0x19, Y9, Y9
+	VPXOR      Y15, Y9, Y9
+	VPSLLD     $0x07, Y10, Y15
+	VPSRLD     $0x19, Y10, Y10
+	VPXOR      Y15, Y10, Y10
+	VPSLLD     $0x07, Y11, Y15
+	VPSRLD     $0x19, Y11, Y11
+	VPXOR      Y15, Y11, Y11
+	VMOVDQA    224(BP), Y15
+	VPALIGNR   $0x04, Y14, Y14, Y14
+	VPALIGNR   $0x04, Y9, Y9, Y9
+	VPALIGNR   $0x04, Y10, Y10, Y10
+	VPALIGNR   $0x04, Y11, Y11, Y11
+	VPALIGNR   $0x08, Y12, Y12, Y12
+	VPALIGNR   $0x08, Y13, Y13, Y13
+	VPALIGNR   $0x08, Y8, Y8, Y8
+	VPALIGNR   $0x08, Y15, Y15, Y15
+	VPALIGNR   $0x0c, Y4, Y4, Y4
+	VPALIGNR   $0x0c, Y1, Y1, Y1
+	VPALIGNR   $0x0c, Y2, Y2, Y2
+	VPALIGNR   $0x0c, Y3, Y3, Y3
+	VPADDD     Y14, Y0, Y0
+	VPADDD     Y9, Y5, Y5
+	VPADDD     Y10, Y6, Y6
+	VPADDD     Y11, Y7, Y7
+	VPXOR      Y0, Y4, Y4
+	VPXOR      Y5, Y1, Y1
+	VPXOR      Y6, Y2, Y2
+	VPXOR      Y7, Y3, Y3
+	VPSHUFB    ·rol16<>+0(SB), Y4, Y4
+	VPSHUFB    ·rol16<>+0(SB), Y1, Y1
+	VPSHUFB    ·rol16<>+0(SB), Y2, Y2
+	VPSHUFB    ·rol16<>+0(SB), Y3, Y3
+	VPADDD     Y4, Y12, Y12
+	VPADDD     Y1, Y13, Y13
+	VPADDD     Y2, Y8, Y8
+	VPADDD     Y3, Y15, Y15
+	VPXOR      Y12, Y14, Y14
+	VPXOR      Y13, Y9, Y9
+	VPXOR      Y8, Y10, Y10
+	VPXOR      Y15, Y11, Y11
+	ADDQ       16(DI), R10
+	ADCQ       24(DI), R11
+	ADCQ       $0x01, R12
+	MOVQ       (BP), DX
+	MOVQ       DX, R15
+	MULXQ      R10, R13, R14
+	IMULQ      R12, R15
+	MULXQ      R11, AX, DX
+	ADDQ       AX, R14
+	ADCQ       DX, R15
+	MOVQ       8(BP), DX
+	MULXQ      R10, R10, AX
+	ADDQ       R10, R14
+	MULXQ      R11, R11, R8
+	ADCQ       R11, R15
+	ADCQ       $0x00, R8
+	IMULQ      R12, DX
+	ADDQ       AX, R15
+	ADCQ       DX, R8
+	MOVQ       R13, R10
+	MOVQ       R14, R11
+	MOVQ       R15, R12
+	ANDQ       $0x03, R12
+	MOVQ       R15, R13
+	ANDQ       $-4, R13
+	MOVQ       R8, R14
+	SHRQ       $0x02, R8, R15
+	SHRQ       $0x02, R8
+	ADDQ       R13, R10
+	ADCQ       R14, R11
+	ADCQ       $0x00, R12
+	ADDQ       R15, R10
+	ADCQ       R8, R11
+	ADCQ       $0x00, R12
+	LEAQ       32(DI), DI
+	VMOVDQA    Y15, 224(BP)
+	VPSLLD     $0x0c, Y14, Y15
+	VPSRLD     $0x14, Y14, Y14
+	VPXOR      Y15, Y14, Y14
+	VPSLLD     $0x0c, Y9, Y15
+	VPSRLD     $0x14, Y9, Y9
+	VPXOR      Y15, Y9, Y9
+	VPSLLD     $0x0c, Y10, Y15
+	VPSRLD     $0x14, Y10, Y10
+	VPXOR      Y15, Y10, Y10
+	VPSLLD     $0x0c, Y11, Y15
+	VPSRLD     $0x14, Y11, Y11
+	VPXOR      Y15, Y11, Y11
+	VMOVDQA    224(BP), Y15
+	VPADDD     Y14, Y0, Y0
+	VPADDD     Y9, Y5, Y5
+	VPADDD     Y10, Y6, Y6
+	VPADDD     Y11, Y7, Y7
+	VPXOR      Y0, Y4, Y4
+	VPXOR      Y5, Y1, Y1
+	VPXOR      Y6, Y2, Y2
+	VPXOR      Y7, Y3, Y3
+	VPSHUFB    ·rol8<>+0(SB), Y4, Y4
+	VPSHUFB    ·rol8<>+0(SB), Y1, Y1
+	VPSHUFB    ·rol8<>+0(SB), Y2, Y2
+	VPSHUFB    ·rol8<>+0(SB), Y3, Y3
+	VPADDD     Y4, Y12, Y12
+	VPADDD     Y1, Y13, Y13
+	VPADDD     Y2, Y8, Y8
+	VPADDD     Y3, Y15, Y15
+	VPXOR      Y12, Y14, Y14
+	VPXOR      Y13, Y9, Y9
+	VPXOR      Y8, Y10, Y10
+	VPXOR      Y15, Y11, Y11
+	VMOVDQA    Y15, 224(BP)
+	VPSLLD     $0x07, Y14, Y15
+	VPSRLD     $0x19, Y14, Y14
+	VPXOR      Y15, Y14, Y14
+	VPSLLD     $0x07, Y9, Y15
+	VPSRLD     $0x19, Y9, Y9
+	VPXOR      Y15, Y9, Y9
+	VPSLLD     $0x07, Y10, Y15
+	VPSRLD     $0x19, Y10, Y10
+	VPXOR      Y15, Y10, Y10
+	VPSLLD     $0x07, Y11, Y15
+	VPSRLD     $0x19, Y11, Y11
+	VPXOR      Y15, Y11, Y11
+	VMOVDQA    224(BP), Y15
+	VPALIGNR   $0x0c, Y14, Y14, Y14
+	VPALIGNR   $0x0c, Y9, Y9, Y9
+	VPALIGNR   $0x0c, Y10, Y10, Y10
+	VPALIGNR   $0x0c, Y11, Y11, Y11
+	VPALIGNR   $0x08, Y12, Y12, Y12
+	VPALIGNR   $0x08, Y13, Y13, Y13
+	VPALIGNR   $0x08, Y8, Y8, Y8
+	VPALIGNR   $0x08, Y15, Y15, Y15
+	VPALIGNR   $0x04, Y4, Y4, Y4
+	VPALIGNR   $0x04, Y1, Y1, Y1
+	VPALIGNR   $0x04, Y2, Y2, Y2
+	VPALIGNR   $0x04, Y3, Y3, Y3
+	DECQ       CX
+	JG         sealAVX2Tail512LoopA
+	DECQ       R9
+	JGE        sealAVX2Tail512LoopB
+	VPADDD     ·chacha20Constants<>+0(SB), Y0, Y0
+	VPADDD     ·chacha20Constants<>+0(SB), Y5, Y5
+	VPADDD     ·chacha20Constants<>+0(SB), Y6, Y6
+	VPADDD     ·chacha20Constants<>+0(SB), Y7, Y7
+	VPADDD     32(BP), Y14, Y14
+	VPADDD     32(BP), Y9, Y9
+	VPADDD     32(BP), Y10, Y10
+	VPADDD     32(BP), Y11, Y11
+	VPADDD     64(BP), Y12, Y12
+	VPADDD     64(BP), Y13, Y13
+	VPADDD     64(BP), Y8, Y8
+	VPADDD     64(BP), Y15, Y15
+	VPADDD     96(BP), Y4, Y4
+	VPADDD     128(BP), Y1, Y1
+	VPADDD     160(BP), Y2, Y2
+	VPADDD     192(BP), Y3, Y3
+	VMOVDQA    Y15, 224(BP)
+	VPERM2I128 $0x02, Y0, Y14, Y15
+	VPXOR      (SI), Y15, Y15
+	VMOVDQU    Y15, (DI)
+	VPERM2I128 $0x02, Y12, Y4, Y15
+	VPXOR      32(SI), Y15, Y15
+	VMOVDQU    Y15, 32(DI)
+	VPERM2I128 $0x13, Y0, Y14, Y15
+	VPXOR      64(SI), Y15, Y15
+	VMOVDQU    Y15, 64(DI)
+	VPERM2I128 $0x13, Y12, Y4, Y15
+	VPXOR      96(SI), Y15, Y15
+	VMOVDQU    Y15, 96(DI)
+	VPERM2I128 $0x02, Y5, Y9, Y0
+	VPERM2I128 $0x02, Y13, Y1, Y14
+	VPERM2I128 $0x13, Y5, Y9, Y12
+	VPERM2I128 $0x13, Y13, Y1, Y4
+	VPXOR      128(SI), Y0, Y0
+	VPXOR      160(SI), Y14, Y14
+	VPXOR      192(SI), Y12, Y12
+	VPXOR      224(SI), Y4, Y4
+	VMOVDQU    Y0, 128(DI)
+	VMOVDQU    Y14, 160(DI)
+	VMOVDQU    Y12, 192(DI)
+	VMOVDQU    Y4, 224(DI)
+	VPERM2I128 $0x02, Y6, Y10, Y0
+	VPERM2I128 $0x02, Y8, Y2, Y14
+	VPERM2I128 $0x13, Y6, Y10, Y12
+	VPERM2I128 $0x13, Y8, Y2, Y4
+	VPXOR      256(SI), Y0, Y0
+	VPXOR      288(SI), Y14, Y14
+	VPXOR      320(SI), Y12, Y12
+	VPXOR      352(SI), Y4, Y4
+	VMOVDQU    Y0, 256(DI)
+	VMOVDQU    Y14, 288(DI)
+	VMOVDQU    Y12, 320(DI)
+	VMOVDQU    Y4, 352(DI)
+	MOVQ       $0x00000180, CX
+	LEAQ       384(SI), SI
+	SUBQ       $0x00000180, BX
+	VPERM2I128 $0x02, Y7, Y11, Y0
+	VPERM2I128 $0x02, 224(BP), Y3, Y14
+	VPERM2I128 $0x13, Y7, Y11, Y12
+	VPERM2I128 $0x13, 224(BP), Y3, Y4
+	JMP        sealAVX2SealHash
diff --git a/vendor/golang.org/x/crypto/internal/poly1305/sum_amd64.s b/vendor/golang.org/x/crypto/internal/poly1305/sum_amd64.s
index e0d3c64756692..133757384b787 100644
--- a/vendor/golang.org/x/crypto/internal/poly1305/sum_amd64.s
+++ b/vendor/golang.org/x/crypto/internal/poly1305/sum_amd64.s
@@ -1,108 +1,93 @@
-// Copyright 2012 The Go Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
+// Code generated by command: go run sum_amd64_asm.go -out ../sum_amd64.s -pkg poly1305. DO NOT EDIT.
 
 //go:build gc && !purego
 
-#include "textflag.h"
-
-#define POLY1305_ADD(msg, h0, h1, h2) \
-	ADDQ 0(msg), h0;  \
-	ADCQ 8(msg), h1;  \
-	ADCQ $1, h2;      \
-	LEAQ 16(msg), msg
-
-#define POLY1305_MUL(h0, h1, h2, r0, r1, t0, t1, t2, t3) \
-	MOVQ  r0, AX;                  \
-	MULQ  h0;                      \
-	MOVQ  AX, t0;                  \
-	MOVQ  DX, t1;                  \
-	MOVQ  r0, AX;                  \
-	MULQ  h1;                      \
-	ADDQ  AX, t1;                  \
-	ADCQ  $0, DX;                  \
-	MOVQ  r0, t2;                  \
-	IMULQ h2, t2;                  \
-	ADDQ  DX, t2;                  \
-	                               \
-	MOVQ  r1, AX;                  \
-	MULQ  h0;                      \
-	ADDQ  AX, t1;                  \
-	ADCQ  $0, DX;                  \
-	MOVQ  DX, h0;                  \
-	MOVQ  r1, t3;                  \
-	IMULQ h2, t3;                  \
-	MOVQ  r1, AX;                  \
-	MULQ  h1;                      \
-	ADDQ  AX, t2;                  \
-	ADCQ  DX, t3;                  \
-	ADDQ  h0, t2;                  \
-	ADCQ  $0, t3;                  \
-	                               \
-	MOVQ  t0, h0;                  \
-	MOVQ  t1, h1;                  \
-	MOVQ  t2, h2;                  \
-	ANDQ  $3, h2;                  \
-	MOVQ  t2, t0;                  \
-	ANDQ  $0xFFFFFFFFFFFFFFFC, t0; \
-	ADDQ  t0, h0;                  \
-	ADCQ  t3, h1;                  \
-	ADCQ  $0, h2;                  \
-	SHRQ  $2, t3, t2;              \
-	SHRQ  $2, t3;                  \
-	ADDQ  t2, h0;                  \
-	ADCQ  t3, h1;                  \
-	ADCQ  $0, h2
-
-// func update(state *[7]uint64, msg []byte)
+// func update(state *macState, msg []byte)
 TEXT ·update(SB), $0-32
 	MOVQ state+0(FP), DI
 	MOVQ msg_base+8(FP), SI
 	MOVQ msg_len+16(FP), R15
-
-	MOVQ 0(DI), R8   // h0
-	MOVQ 8(DI), R9   // h1
-	MOVQ 16(DI), R10 // h2
-	MOVQ 24(DI), R11 // r0
-	MOVQ 32(DI), R12 // r1
-
-	CMPQ R15, $16
+	MOVQ (DI), R8
+	MOVQ 8(DI), R9
+	MOVQ 16(DI), R10
+	MOVQ 24(DI), R11
+	MOVQ 32(DI), R12
+	CMPQ R15, $0x10
 	JB   bytes_between_0_and_15
 
 loop:
-	POLY1305_ADD(SI, R8, R9, R10)
+	ADDQ (SI), R8
+	ADCQ 8(SI), R9
+	ADCQ $0x01, R10
+	LEAQ 16(SI), SI
 
 multiply:
-	POLY1305_MUL(R8, R9, R10, R11, R12, BX, CX, R13, R14)
-	SUBQ $16, R15
-	CMPQ R15, $16
-	JAE  loop
+	MOVQ  R11, AX
+	MULQ  R8
+	MOVQ  AX, BX
+	MOVQ  DX, CX
+	MOVQ  R11, AX
+	MULQ  R9
+	ADDQ  AX, CX
+	ADCQ  $0x00, DX
+	MOVQ  R11, R13
+	IMULQ R10, R13
+	ADDQ  DX, R13
+	MOVQ  R12, AX
+	MULQ  R8
+	ADDQ  AX, CX
+	ADCQ  $0x00, DX
+	MOVQ  DX, R8
+	MOVQ  R12, R14
+	IMULQ R10, R14
+	MOVQ  R12, AX
+	MULQ  R9
+	ADDQ  AX, R13
+	ADCQ  DX, R14
+	ADDQ  R8, R13
+	ADCQ  $0x00, R14
+	MOVQ  BX, R8
+	MOVQ  CX, R9
+	MOVQ  R13, R10
+	ANDQ  $0x03, R10
+	MOVQ  R13, BX
+	ANDQ  $-4, BX
+	ADDQ  BX, R8
+	ADCQ  R14, R9
+	ADCQ  $0x00, R10
+	SHRQ  $0x02, R14, R13
+	SHRQ  $0x02, R14
+	ADDQ  R13, R8
+	ADCQ  R14, R9
+	ADCQ  $0x00, R10
+	SUBQ  $0x10, R15
+	CMPQ  R15, $0x10
+	JAE   loop
 
 bytes_between_0_and_15:
 	TESTQ R15, R15
 	JZ    done
-	MOVQ  $1, BX
+	MOVQ  $0x00000001, BX
 	XORQ  CX, CX
 	XORQ  R13, R13
 	ADDQ  R15, SI
 
 flush_buffer:
-	SHLQ $8, BX, CX
-	SHLQ $8, BX
+	SHLQ $0x08, BX, CX
+	SHLQ $0x08, BX
 	MOVB -1(SI), R13
 	XORQ R13, BX
 	DECQ SI
 	DECQ R15
 	JNZ  flush_buffer
-
 	ADDQ BX, R8
 	ADCQ CX, R9
-	ADCQ $0, R10
-	MOVQ $16, R15
+	ADCQ $0x00, R10
+	MOVQ $0x00000010, R15
 	JMP  multiply
 
 done:
-	MOVQ R8, 0(DI)
+	MOVQ R8, (DI)
 	MOVQ R9, 8(DI)
 	MOVQ R10, 16(DI)
 	RET
diff --git a/vendor/golang.org/x/sys/cpu/cpu.go b/vendor/golang.org/x/sys/cpu/cpu.go
index ec07aab05788c..02609d5b21d56 100644
--- a/vendor/golang.org/x/sys/cpu/cpu.go
+++ b/vendor/golang.org/x/sys/cpu/cpu.go
@@ -201,6 +201,25 @@ var S390X struct {
 	_         CacheLinePad
 }
 
+// RISCV64 contains the supported CPU features and performance characteristics for riscv64
+// platforms. The booleans in RISCV64, with the exception of HasFastMisaligned, indicate
+// the presence of RISC-V extensions.
+//
+// It is safe to assume that all the RV64G extensions are supported and so they are omitted from
+// this structure. As riscv64 Go programs require at least RV64G, the code that populates
+// this structure cannot run successfully if some of the RV64G extensions are missing.
+// The struct is padded to avoid false sharing.
+var RISCV64 struct {
+	_                 CacheLinePad
+	HasFastMisaligned bool // Fast misaligned accesses
+	HasC              bool // Compressed instruction-set extension
+	HasV              bool // Vector extension compatible with RVV 1.0
+	HasZba            bool // Address generation instructions extension
+	HasZbb            bool // Basic bit-manipulation extension
+	HasZbs            bool // Single-bit instructions extension
+	_                 CacheLinePad
+}
+
 func init() {
 	archInit()
 	initOptions()
diff --git a/vendor/golang.org/x/sys/cpu/cpu_linux_noinit.go b/vendor/golang.org/x/sys/cpu/cpu_linux_noinit.go
index cd63e73355734..7d902b6847bbf 100644
--- a/vendor/golang.org/x/sys/cpu/cpu_linux_noinit.go
+++ b/vendor/golang.org/x/sys/cpu/cpu_linux_noinit.go
@@ -2,7 +2,7 @@
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.
 
-//go:build linux && !arm && !arm64 && !mips64 && !mips64le && !ppc64 && !ppc64le && !s390x
+//go:build linux && !arm && !arm64 && !mips64 && !mips64le && !ppc64 && !ppc64le && !s390x && !riscv64
 
 package cpu
 
diff --git a/vendor/golang.org/x/sys/cpu/cpu_linux_riscv64.go b/vendor/golang.org/x/sys/cpu/cpu_linux_riscv64.go
new file mode 100644
index 0000000000000..cb4a0c5728073
--- /dev/null
+++ b/vendor/golang.org/x/sys/cpu/cpu_linux_riscv64.go
@@ -0,0 +1,137 @@
+// Copyright 2024 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package cpu
+
+import (
+	"syscall"
+	"unsafe"
+)
+
+// RISC-V extension discovery code for Linux. The approach here is to first try the riscv_hwprobe
+// syscall falling back to HWCAP to check for the C extension if riscv_hwprobe is not available.
+//
+// A note on detection of the Vector extension using HWCAP.
+//
+// Support for the Vector extension version 1.0 was added to the Linux kernel in release 6.5.
+// Support for the riscv_hwprobe syscall was added in 6.4. It follows that if the riscv_hwprobe
+// syscall is not available then neither is the Vector extension (which needs kernel support).
+// The riscv_hwprobe syscall should then be all we need to detect the Vector extension.
+// However, some RISC-V board manufacturers ship boards with an older kernel on top of which
+// they have back-ported various versions of the Vector extension patches but not the riscv_hwprobe
+// patches. These kernels advertise support for the Vector extension using HWCAP. Falling
+// back to HWCAP to detect the Vector extension, if riscv_hwprobe is not available, or simply not
+// bothering with riscv_hwprobe at all and just using HWCAP may then seem like an attractive option.
+//
+// Unfortunately, simply checking the 'V' bit in AT_HWCAP will not work as this bit is used by
+// RISC-V board and cloud instance providers to mean different things. The Lichee Pi 4A board
+// and the Scaleway RV1 cloud instances use the 'V' bit to advertise their support for the unratified
+// 0.7.1 version of the Vector Specification. The Banana Pi BPI-F3 and the CanMV-K230 board use
+// it to advertise support for 1.0 of the Vector extension. Versions 0.7.1 and 1.0 of the Vector
+// extension are binary incompatible. HWCAP can then not be used in isolation to populate the
+// HasV field as this field indicates that the underlying CPU is compatible with RVV 1.0.
+//
+// There is a way at runtime to distinguish between versions 0.7.1 and 1.0 of the Vector
+// specification by issuing a RVV 1.0 vsetvli instruction and checking the vill bit of the vtype
+// register. This check would allow us to safely detect version 1.0 of the Vector extension
+// with HWCAP, if riscv_hwprobe were not available. However, the check cannot
+// be added until the assembler supports the Vector instructions.
+//
+// Note the riscv_hwprobe syscall does not suffer from these ambiguities by design as all of the
+// extensions it advertises support for are explicitly versioned. It's also worth noting that
+// the riscv_hwprobe syscall is the only way to detect multi-letter RISC-V extensions, e.g., Zba.
+// These cannot be detected using HWCAP and so riscv_hwprobe must be used to detect the majority
+// of RISC-V extensions.
+//
+// Please see https://docs.kernel.org/arch/riscv/hwprobe.html for more information.
+
+// golang.org/x/sys/cpu is not allowed to depend on golang.org/x/sys/unix so we must
+// reproduce the constants, types and functions needed to make the riscv_hwprobe syscall
+// here.
+
+const (
+	// Copied from golang.org/x/sys/unix/ztypes_linux_riscv64.go.
+	riscv_HWPROBE_KEY_IMA_EXT_0   = 0x4
+	riscv_HWPROBE_IMA_C           = 0x2
+	riscv_HWPROBE_IMA_V           = 0x4
+	riscv_HWPROBE_EXT_ZBA         = 0x8
+	riscv_HWPROBE_EXT_ZBB         = 0x10
+	riscv_HWPROBE_EXT_ZBS         = 0x20
+	riscv_HWPROBE_KEY_CPUPERF_0   = 0x5
+	riscv_HWPROBE_MISALIGNED_FAST = 0x3
+	riscv_HWPROBE_MISALIGNED_MASK = 0x7
+)
+
+const (
+	// sys_RISCV_HWPROBE is copied from golang.org/x/sys/unix/zsysnum_linux_riscv64.go.
+	sys_RISCV_HWPROBE = 258
+)
+
+// riscvHWProbePairs is copied from golang.org/x/sys/unix/ztypes_linux_riscv64.go.
+type riscvHWProbePairs struct {
+	key   int64
+	value uint64
+}
+
+const (
+	// CPU features
+	hwcap_RISCV_ISA_C = 1 << ('C' - 'A')
+)
+
+func doinit() {
+	// A slice of key/value pair structures is passed to the RISCVHWProbe syscall. The key
+	// field should be initialised with one of the key constants defined above, e.g.,
+	// RISCV_HWPROBE_KEY_IMA_EXT_0. The syscall will set the value field to the appropriate value.
+	// If the kernel does not recognise a key it will set the key field to -1 and the value field to 0.
+
+	pairs := []riscvHWProbePairs{
+		{riscv_HWPROBE_KEY_IMA_EXT_0, 0},
+		{riscv_HWPROBE_KEY_CPUPERF_0, 0},
+	}
+
+	// This call only indicates that extensions are supported if they are implemented on all cores.
+	if riscvHWProbe(pairs, 0) {
+		if pairs[0].key != -1 {
+			v := uint(pairs[0].value)
+			RISCV64.HasC = isSet(v, riscv_HWPROBE_IMA_C)
+			RISCV64.HasV = isSet(v, riscv_HWPROBE_IMA_V)
+			RISCV64.HasZba = isSet(v, riscv_HWPROBE_EXT_ZBA)
+			RISCV64.HasZbb = isSet(v, riscv_HWPROBE_EXT_ZBB)
+			RISCV64.HasZbs = isSet(v, riscv_HWPROBE_EXT_ZBS)
+		}
+		if pairs[1].key != -1 {
+			v := pairs[1].value & riscv_HWPROBE_MISALIGNED_MASK
+			RISCV64.HasFastMisaligned = v == riscv_HWPROBE_MISALIGNED_FAST
+		}
+	}
+
+	// Let's double check with HWCAP if the C extension does not appear to be supported.
+	// This may happen if we're running on a kernel older than 6.4.
+
+	if !RISCV64.HasC {
+		RISCV64.HasC = isSet(hwCap, hwcap_RISCV_ISA_C)
+	}
+}
+
+func isSet(hwc uint, value uint) bool {
+	return hwc&value != 0
+}
+
+// riscvHWProbe is a simplified version of the generated wrapper function found in
+// golang.org/x/sys/unix/zsyscall_linux_riscv64.go. We simplify it by removing the
+// cpuCount and cpus parameters which we do not need. We always want to pass 0 for
+// these parameters here so the kernel only reports the extensions that are present
+// on all cores.
+func riscvHWProbe(pairs []riscvHWProbePairs, flags uint) bool {
+	var _zero uintptr
+	var p0 unsafe.Pointer
+	if len(pairs) > 0 {
+		p0 = unsafe.Pointer(&pairs[0])
+	} else {
+		p0 = unsafe.Pointer(&_zero)
+	}
+
+	_, _, e1 := syscall.Syscall6(sys_RISCV_HWPROBE, uintptr(p0), uintptr(len(pairs)), uintptr(0), uintptr(0), uintptr(flags), 0)
+	return e1 == 0
+}
diff --git a/vendor/golang.org/x/sys/cpu/cpu_riscv64.go b/vendor/golang.org/x/sys/cpu/cpu_riscv64.go
index 7f0c79c004b44..aca3199c91169 100644
--- a/vendor/golang.org/x/sys/cpu/cpu_riscv64.go
+++ b/vendor/golang.org/x/sys/cpu/cpu_riscv64.go
@@ -8,4 +8,13 @@ package cpu
 
 const cacheLineSize = 64
 
-func initOptions() {}
+func initOptions() {
+	options = []option{
+		{Name: "fastmisaligned", Feature: &RISCV64.HasFastMisaligned},
+		{Name: "c", Feature: &RISCV64.HasC},
+		{Name: "v", Feature: &RISCV64.HasV},
+		{Name: "zba", Feature: &RISCV64.HasZba},
+		{Name: "zbb", Feature: &RISCV64.HasZbb},
+		{Name: "zbs", Feature: &RISCV64.HasZbs},
+	}
+}
diff --git a/vendor/golang.org/x/sys/unix/mkerrors.sh b/vendor/golang.org/x/sys/unix/mkerrors.sh
index d07dd09eb503c..e14b766a32c5f 100644
--- a/vendor/golang.org/x/sys/unix/mkerrors.sh
+++ b/vendor/golang.org/x/sys/unix/mkerrors.sh
@@ -552,6 +552,7 @@ ccflags="$@"
 		$2 !~ /^RTC_VL_(ACCURACY|BACKUP|DATA)/ &&
 		$2 ~ /^(NETLINK|NLM|NLMSG|NLA|IFA|IFAN|RT|RTC|RTCF|RTN|RTPROT|RTNH|ARPHRD|ETH_P|NETNSA)_/ ||
 		$2 ~ /^SOCK_|SK_DIAG_|SKNLGRP_$/ ||
+		$2 ~ /^(CONNECT|SAE)_/ ||
 		$2 ~ /^FIORDCHK$/ ||
 		$2 ~ /^SIOC/ ||
 		$2 ~ /^TIOC/ ||
diff --git a/vendor/golang.org/x/sys/unix/syscall_darwin.go b/vendor/golang.org/x/sys/unix/syscall_darwin.go
index 2d15200adb497..099867deedec6 100644
--- a/vendor/golang.org/x/sys/unix/syscall_darwin.go
+++ b/vendor/golang.org/x/sys/unix/syscall_darwin.go
@@ -566,6 +566,43 @@ func PthreadFchdir(fd int) (err error) {
 	return pthread_fchdir_np(fd)
 }
 
+// Connectx calls connectx(2) to initiate a connection on a socket.
+//
+// srcIf, srcAddr, and dstAddr are filled into a [SaEndpoints] struct and passed as the endpoints argument.
+//
+//   - srcIf is the optional source interface index. 0 means unspecified.
+//   - srcAddr is the optional source address. nil means unspecified.
+//   - dstAddr is the destination address.
+//
+// On success, Connectx returns the number of bytes enqueued for transmission.
+func Connectx(fd int, srcIf uint32, srcAddr, dstAddr Sockaddr, associd SaeAssocID, flags uint32, iov []Iovec, connid *SaeConnID) (n uintptr, err error) {
+	endpoints := SaEndpoints{
+		Srcif: srcIf,
+	}
+
+	if srcAddr != nil {
+		addrp, addrlen, err := srcAddr.sockaddr()
+		if err != nil {
+			return 0, err
+		}
+		endpoints.Srcaddr = (*RawSockaddr)(addrp)
+		endpoints.Srcaddrlen = uint32(addrlen)
+	}
+
+	if dstAddr != nil {
+		addrp, addrlen, err := dstAddr.sockaddr()
+		if err != nil {
+			return 0, err
+		}
+		endpoints.Dstaddr = (*RawSockaddr)(addrp)
+		endpoints.Dstaddrlen = uint32(addrlen)
+	}
+
+	err = connectx(fd, &endpoints, associd, flags, iov, &n, connid)
+	return
+}
+
+//sys	connectx(fd int, endpoints *SaEndpoints, associd SaeAssocID, flags uint32, iov []Iovec, n *uintptr, connid *SaeConnID) (err error)
 //sys	sendfile(infd int, outfd int, offset int64, len *int64, hdtr unsafe.Pointer, flags int) (err error)
 
 //sys	shmat(id int, addr uintptr, flag int) (ret uintptr, err error)
diff --git a/vendor/golang.org/x/sys/unix/syscall_hurd.go b/vendor/golang.org/x/sys/unix/syscall_hurd.go
index ba46651f8e38a..a6a2d2fc2b90c 100644
--- a/vendor/golang.org/x/sys/unix/syscall_hurd.go
+++ b/vendor/golang.org/x/sys/unix/syscall_hurd.go
@@ -11,6 +11,7 @@ package unix
 int ioctl(int, unsigned long int, uintptr_t);
 */
 import "C"
+import "unsafe"
 
 func ioctl(fd int, req uint, arg uintptr) (err error) {
 	r0, er := C.ioctl(C.int(fd), C.ulong(req), C.uintptr_t(arg))
diff --git a/vendor/golang.org/x/sys/unix/zerrors_darwin_amd64.go b/vendor/golang.org/x/sys/unix/zerrors_darwin_amd64.go
index 4308ac1772b9f..d73c4652e6c54 100644
--- a/vendor/golang.org/x/sys/unix/zerrors_darwin_amd64.go
+++ b/vendor/golang.org/x/sys/unix/zerrors_darwin_amd64.go
@@ -237,6 +237,9 @@ const (
 	CLOCK_UPTIME_RAW_APPROX                 = 0x9
 	CLONE_NOFOLLOW                          = 0x1
 	CLONE_NOOWNERCOPY                       = 0x2
+	CONNECT_DATA_AUTHENTICATED              = 0x4
+	CONNECT_DATA_IDEMPOTENT                 = 0x2
+	CONNECT_RESUME_ON_READ_WRITE            = 0x1
 	CR0                                     = 0x0
 	CR1                                     = 0x1000
 	CR2                                     = 0x2000
@@ -1265,6 +1268,10 @@ const (
 	RTV_SSTHRESH                            = 0x20
 	RUSAGE_CHILDREN                         = -0x1
 	RUSAGE_SELF                             = 0x0
+	SAE_ASSOCID_ALL                         = 0xffffffff
+	SAE_ASSOCID_ANY                         = 0x0
+	SAE_CONNID_ALL                          = 0xffffffff
+	SAE_CONNID_ANY                          = 0x0
 	SCM_CREDS                               = 0x3
 	SCM_RIGHTS                              = 0x1
 	SCM_TIMESTAMP                           = 0x2
diff --git a/vendor/golang.org/x/sys/unix/zerrors_darwin_arm64.go b/vendor/golang.org/x/sys/unix/zerrors_darwin_arm64.go
index c8068a7a16900..4a55a40058894 100644
--- a/vendor/golang.org/x/sys/unix/zerrors_darwin_arm64.go
+++ b/vendor/golang.org/x/sys/unix/zerrors_darwin_arm64.go
@@ -237,6 +237,9 @@ const (
 	CLOCK_UPTIME_RAW_APPROX                 = 0x9
 	CLONE_NOFOLLOW                          = 0x1
 	CLONE_NOOWNERCOPY                       = 0x2
+	CONNECT_DATA_AUTHENTICATED              = 0x4
+	CONNECT_DATA_IDEMPOTENT                 = 0x2
+	CONNECT_RESUME_ON_READ_WRITE            = 0x1
 	CR0                                     = 0x0
 	CR1                                     = 0x1000
 	CR2                                     = 0x2000
@@ -1265,6 +1268,10 @@ const (
 	RTV_SSTHRESH                            = 0x20
 	RUSAGE_CHILDREN                         = -0x1
 	RUSAGE_SELF                             = 0x0
+	SAE_ASSOCID_ALL                         = 0xffffffff
+	SAE_ASSOCID_ANY                         = 0x0
+	SAE_CONNID_ALL                          = 0xffffffff
+	SAE_CONNID_ANY                          = 0x0
 	SCM_CREDS                               = 0x3
 	SCM_RIGHTS                              = 0x1
 	SCM_TIMESTAMP                           = 0x2
diff --git a/vendor/golang.org/x/sys/unix/zerrors_zos_s390x.go b/vendor/golang.org/x/sys/unix/zerrors_zos_s390x.go
index da08b2ab3d939..1ec2b1407b126 100644
--- a/vendor/golang.org/x/sys/unix/zerrors_zos_s390x.go
+++ b/vendor/golang.org/x/sys/unix/zerrors_zos_s390x.go
@@ -581,6 +581,8 @@ const (
 	AT_EMPTY_PATH                   = 0x1000
 	AT_REMOVEDIR                    = 0x200
 	RENAME_NOREPLACE                = 1 << 0
+	ST_RDONLY                       = 1
+	ST_NOSUID                       = 2
 )
 
 const (
diff --git a/vendor/golang.org/x/sys/unix/zsyscall_darwin_amd64.go b/vendor/golang.org/x/sys/unix/zsyscall_darwin_amd64.go
index b622533ef2c2b..24b346e1a3515 100644
--- a/vendor/golang.org/x/sys/unix/zsyscall_darwin_amd64.go
+++ b/vendor/golang.org/x/sys/unix/zsyscall_darwin_amd64.go
@@ -841,6 +841,26 @@ var libc_pthread_fchdir_np_trampoline_addr uintptr
 
 // THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT
 
+func connectx(fd int, endpoints *SaEndpoints, associd SaeAssocID, flags uint32, iov []Iovec, n *uintptr, connid *SaeConnID) (err error) {
+	var _p0 unsafe.Pointer
+	if len(iov) > 0 {
+		_p0 = unsafe.Pointer(&iov[0])
+	} else {
+		_p0 = unsafe.Pointer(&_zero)
+	}
+	_, _, e1 := syscall_syscall9(libc_connectx_trampoline_addr, uintptr(fd), uintptr(unsafe.Pointer(endpoints)), uintptr(associd), uintptr(flags), uintptr(_p0), uintptr(len(iov)), uintptr(unsafe.Pointer(n)), uintptr(unsafe.Pointer(connid)), 0)
+	if e1 != 0 {
+		err = errnoErr(e1)
+	}
+	return
+}
+
+var libc_connectx_trampoline_addr uintptr
+
+//go:cgo_import_dynamic libc_connectx connectx "/usr/lib/libSystem.B.dylib"
+
+// THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT
+
 func sendfile(infd int, outfd int, offset int64, len *int64, hdtr unsafe.Pointer, flags int) (err error) {
 	_, _, e1 := syscall_syscall6(libc_sendfile_trampoline_addr, uintptr(infd), uintptr(outfd), uintptr(offset), uintptr(unsafe.Pointer(len)), uintptr(hdtr), uintptr(flags))
 	if e1 != 0 {
diff --git a/vendor/golang.org/x/sys/unix/zsyscall_darwin_amd64.s b/vendor/golang.org/x/sys/unix/zsyscall_darwin_amd64.s
index cfe6646baf230..ebd213100b352 100644
--- a/vendor/golang.org/x/sys/unix/zsyscall_darwin_amd64.s
+++ b/vendor/golang.org/x/sys/unix/zsyscall_darwin_amd64.s
@@ -248,6 +248,11 @@ TEXT libc_pthread_fchdir_np_trampoline<>(SB),NOSPLIT,$0-0
 GLOBL	·libc_pthread_fchdir_np_trampoline_addr(SB), RODATA, $8
 DATA	·libc_pthread_fchdir_np_trampoline_addr(SB)/8, $libc_pthread_fchdir_np_trampoline<>(SB)
 
+TEXT libc_connectx_trampoline<>(SB),NOSPLIT,$0-0
+	JMP	libc_connectx(SB)
+GLOBL	·libc_connectx_trampoline_addr(SB), RODATA, $8
+DATA	·libc_connectx_trampoline_addr(SB)/8, $libc_connectx_trampoline<>(SB)
+
 TEXT libc_sendfile_trampoline<>(SB),NOSPLIT,$0-0
 	JMP	libc_sendfile(SB)
 GLOBL	·libc_sendfile_trampoline_addr(SB), RODATA, $8
diff --git a/vendor/golang.org/x/sys/unix/zsyscall_darwin_arm64.go b/vendor/golang.org/x/sys/unix/zsyscall_darwin_arm64.go
index 13f624f69f19d..824b9c2d5e0e0 100644
--- a/vendor/golang.org/x/sys/unix/zsyscall_darwin_arm64.go
+++ b/vendor/golang.org/x/sys/unix/zsyscall_darwin_arm64.go
@@ -841,6 +841,26 @@ var libc_pthread_fchdir_np_trampoline_addr uintptr
 
 // THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT
 
+func connectx(fd int, endpoints *SaEndpoints, associd SaeAssocID, flags uint32, iov []Iovec, n *uintptr, connid *SaeConnID) (err error) {
+	var _p0 unsafe.Pointer
+	if len(iov) > 0 {
+		_p0 = unsafe.Pointer(&iov[0])
+	} else {
+		_p0 = unsafe.Pointer(&_zero)
+	}
+	_, _, e1 := syscall_syscall9(libc_connectx_trampoline_addr, uintptr(fd), uintptr(unsafe.Pointer(endpoints)), uintptr(associd), uintptr(flags), uintptr(_p0), uintptr(len(iov)), uintptr(unsafe.Pointer(n)), uintptr(unsafe.Pointer(connid)), 0)
+	if e1 != 0 {
+		err = errnoErr(e1)
+	}
+	return
+}
+
+var libc_connectx_trampoline_addr uintptr
+
+//go:cgo_import_dynamic libc_connectx connectx "/usr/lib/libSystem.B.dylib"
+
+// THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT
+
 func sendfile(infd int, outfd int, offset int64, len *int64, hdtr unsafe.Pointer, flags int) (err error) {
 	_, _, e1 := syscall_syscall6(libc_sendfile_trampoline_addr, uintptr(infd), uintptr(outfd), uintptr(offset), uintptr(unsafe.Pointer(len)), uintptr(hdtr), uintptr(flags))
 	if e1 != 0 {
diff --git a/vendor/golang.org/x/sys/unix/zsyscall_darwin_arm64.s b/vendor/golang.org/x/sys/unix/zsyscall_darwin_arm64.s
index fe222b75df04f..4f178a229345e 100644
--- a/vendor/golang.org/x/sys/unix/zsyscall_darwin_arm64.s
+++ b/vendor/golang.org/x/sys/unix/zsyscall_darwin_arm64.s
@@ -248,6 +248,11 @@ TEXT libc_pthread_fchdir_np_trampoline<>(SB),NOSPLIT,$0-0
 GLOBL	·libc_pthread_fchdir_np_trampoline_addr(SB), RODATA, $8
 DATA	·libc_pthread_fchdir_np_trampoline_addr(SB)/8, $libc_pthread_fchdir_np_trampoline<>(SB)
 
+TEXT libc_connectx_trampoline<>(SB),NOSPLIT,$0-0
+	JMP	libc_connectx(SB)
+GLOBL	·libc_connectx_trampoline_addr(SB), RODATA, $8
+DATA	·libc_connectx_trampoline_addr(SB)/8, $libc_connectx_trampoline<>(SB)
+
 TEXT libc_sendfile_trampoline<>(SB),NOSPLIT,$0-0
 	JMP	libc_sendfile(SB)
 GLOBL	·libc_sendfile_trampoline_addr(SB), RODATA, $8
diff --git a/vendor/golang.org/x/sys/unix/ztypes_darwin_amd64.go b/vendor/golang.org/x/sys/unix/ztypes_darwin_amd64.go
index 091d107f3a5c9..d003c3d43780c 100644
--- a/vendor/golang.org/x/sys/unix/ztypes_darwin_amd64.go
+++ b/vendor/golang.org/x/sys/unix/ztypes_darwin_amd64.go
@@ -306,6 +306,19 @@ type XVSockPgen struct {
 
 type _Socklen uint32
 
+type SaeAssocID uint32
+
+type SaeConnID uint32
+
+type SaEndpoints struct {
+	Srcif      uint32
+	Srcaddr    *RawSockaddr
+	Srcaddrlen uint32
+	Dstaddr    *RawSockaddr
+	Dstaddrlen uint32
+	_          [4]byte
+}
+
 type Xucred struct {
 	Version uint32
 	Uid     uint32
diff --git a/vendor/golang.org/x/sys/unix/ztypes_darwin_arm64.go b/vendor/golang.org/x/sys/unix/ztypes_darwin_arm64.go
index 28ff4ef74d0d3..0d45a941aaecc 100644
--- a/vendor/golang.org/x/sys/unix/ztypes_darwin_arm64.go
+++ b/vendor/golang.org/x/sys/unix/ztypes_darwin_arm64.go
@@ -306,6 +306,19 @@ type XVSockPgen struct {
 
 type _Socklen uint32
 
+type SaeAssocID uint32
+
+type SaeConnID uint32
+
+type SaEndpoints struct {
+	Srcif      uint32
+	Srcaddr    *RawSockaddr
+	Srcaddrlen uint32
+	Dstaddr    *RawSockaddr
+	Dstaddrlen uint32
+	_          [4]byte
+}
+
 type Xucred struct {
 	Version uint32
 	Uid     uint32
diff --git a/vendor/golang.org/x/sys/unix/ztypes_freebsd_386.go b/vendor/golang.org/x/sys/unix/ztypes_freebsd_386.go
index 6cbd094a3aa19..51e13eb055fc5 100644
--- a/vendor/golang.org/x/sys/unix/ztypes_freebsd_386.go
+++ b/vendor/golang.org/x/sys/unix/ztypes_freebsd_386.go
@@ -625,6 +625,7 @@ const (
 	POLLRDNORM   = 0x40
 	POLLWRBAND   = 0x100
 	POLLWRNORM   = 0x4
+	POLLRDHUP    = 0x4000
 )
 
 type CapRights struct {
diff --git a/vendor/golang.org/x/sys/unix/ztypes_freebsd_amd64.go b/vendor/golang.org/x/sys/unix/ztypes_freebsd_amd64.go
index 7c03b6ee77fa0..d002d8ef3cc2d 100644
--- a/vendor/golang.org/x/sys/unix/ztypes_freebsd_amd64.go
+++ b/vendor/golang.org/x/sys/unix/ztypes_freebsd_amd64.go
@@ -630,6 +630,7 @@ const (
 	POLLRDNORM   = 0x40
 	POLLWRBAND   = 0x100
 	POLLWRNORM   = 0x4
+	POLLRDHUP    = 0x4000
 )
 
 type CapRights struct {
diff --git a/vendor/golang.org/x/sys/unix/ztypes_freebsd_arm.go b/vendor/golang.org/x/sys/unix/ztypes_freebsd_arm.go
index 422107ee8b133..3f863d898dd85 100644
--- a/vendor/golang.org/x/sys/unix/ztypes_freebsd_arm.go
+++ b/vendor/golang.org/x/sys/unix/ztypes_freebsd_arm.go
@@ -616,6 +616,7 @@ const (
 	POLLRDNORM   = 0x40
 	POLLWRBAND   = 0x100
 	POLLWRNORM   = 0x4
+	POLLRDHUP    = 0x4000
 )
 
 type CapRights struct {
diff --git a/vendor/golang.org/x/sys/unix/ztypes_freebsd_arm64.go b/vendor/golang.org/x/sys/unix/ztypes_freebsd_arm64.go
index 505a12acfd9d2..61c7293106616 100644
--- a/vendor/golang.org/x/sys/unix/ztypes_freebsd_arm64.go
+++ b/vendor/golang.org/x/sys/unix/ztypes_freebsd_arm64.go
@@ -610,6 +610,7 @@ const (
 	POLLRDNORM   = 0x40
 	POLLWRBAND   = 0x100
 	POLLWRNORM   = 0x4
+	POLLRDHUP    = 0x4000
 )
 
 type CapRights struct {
diff --git a/vendor/golang.org/x/sys/unix/ztypes_freebsd_riscv64.go b/vendor/golang.org/x/sys/unix/ztypes_freebsd_riscv64.go
index cc986c7900660..b5d17414f039f 100644
--- a/vendor/golang.org/x/sys/unix/ztypes_freebsd_riscv64.go
+++ b/vendor/golang.org/x/sys/unix/ztypes_freebsd_riscv64.go
@@ -612,6 +612,7 @@ const (
 	POLLRDNORM   = 0x40
 	POLLWRBAND   = 0x100
 	POLLWRNORM   = 0x4
+	POLLRDHUP    = 0x4000
 )
 
 type CapRights struct {
diff --git a/vendor/golang.org/x/sys/unix/ztypes_linux.go b/vendor/golang.org/x/sys/unix/ztypes_linux.go
index 7f1961b907a2b..9f2550dc3120d 100644
--- a/vendor/golang.org/x/sys/unix/ztypes_linux.go
+++ b/vendor/golang.org/x/sys/unix/ztypes_linux.go
@@ -2486,7 +2486,7 @@ type XDPMmapOffsets struct {
 type XDPUmemReg struct {
 	Addr            uint64
 	Len             uint64
-	Chunk_size      uint32
+	Size            uint32
 	Headroom        uint32
 	Flags           uint32
 	Tx_metadata_len uint32
diff --git a/vendor/golang.org/x/sys/unix/ztypes_linux_riscv64.go b/vendor/golang.org/x/sys/unix/ztypes_linux_riscv64.go
index 15adc04142f2b..ad05b51a60364 100644
--- a/vendor/golang.org/x/sys/unix/ztypes_linux_riscv64.go
+++ b/vendor/golang.org/x/sys/unix/ztypes_linux_riscv64.go
@@ -727,6 +727,37 @@ const (
 	RISCV_HWPROBE_EXT_ZBA                = 0x8
 	RISCV_HWPROBE_EXT_ZBB                = 0x10
 	RISCV_HWPROBE_EXT_ZBS                = 0x20
+	RISCV_HWPROBE_EXT_ZICBOZ             = 0x40
+	RISCV_HWPROBE_EXT_ZBC                = 0x80
+	RISCV_HWPROBE_EXT_ZBKB               = 0x100
+	RISCV_HWPROBE_EXT_ZBKC               = 0x200
+	RISCV_HWPROBE_EXT_ZBKX               = 0x400
+	RISCV_HWPROBE_EXT_ZKND               = 0x800
+	RISCV_HWPROBE_EXT_ZKNE               = 0x1000
+	RISCV_HWPROBE_EXT_ZKNH               = 0x2000
+	RISCV_HWPROBE_EXT_ZKSED              = 0x4000
+	RISCV_HWPROBE_EXT_ZKSH               = 0x8000
+	RISCV_HWPROBE_EXT_ZKT                = 0x10000
+	RISCV_HWPROBE_EXT_ZVBB               = 0x20000
+	RISCV_HWPROBE_EXT_ZVBC               = 0x40000
+	RISCV_HWPROBE_EXT_ZVKB               = 0x80000
+	RISCV_HWPROBE_EXT_ZVKG               = 0x100000
+	RISCV_HWPROBE_EXT_ZVKNED             = 0x200000
+	RISCV_HWPROBE_EXT_ZVKNHA             = 0x400000
+	RISCV_HWPROBE_EXT_ZVKNHB             = 0x800000
+	RISCV_HWPROBE_EXT_ZVKSED             = 0x1000000
+	RISCV_HWPROBE_EXT_ZVKSH              = 0x2000000
+	RISCV_HWPROBE_EXT_ZVKT               = 0x4000000
+	RISCV_HWPROBE_EXT_ZFH                = 0x8000000
+	RISCV_HWPROBE_EXT_ZFHMIN             = 0x10000000
+	RISCV_HWPROBE_EXT_ZIHINTNTL          = 0x20000000
+	RISCV_HWPROBE_EXT_ZVFH               = 0x40000000
+	RISCV_HWPROBE_EXT_ZVFHMIN            = 0x80000000
+	RISCV_HWPROBE_EXT_ZFA                = 0x100000000
+	RISCV_HWPROBE_EXT_ZTSO               = 0x200000000
+	RISCV_HWPROBE_EXT_ZACAS              = 0x400000000
+	RISCV_HWPROBE_EXT_ZICOND             = 0x800000000
+	RISCV_HWPROBE_EXT_ZIHINTPAUSE        = 0x1000000000
 	RISCV_HWPROBE_KEY_CPUPERF_0          = 0x5
 	RISCV_HWPROBE_MISALIGNED_UNKNOWN     = 0x0
 	RISCV_HWPROBE_MISALIGNED_EMULATED    = 0x1
@@ -734,4 +765,6 @@ const (
 	RISCV_HWPROBE_MISALIGNED_FAST        = 0x3
 	RISCV_HWPROBE_MISALIGNED_UNSUPPORTED = 0x4
 	RISCV_HWPROBE_MISALIGNED_MASK        = 0x7
+	RISCV_HWPROBE_KEY_ZICBOZ_BLOCK_SIZE  = 0x6
+	RISCV_HWPROBE_WHICH_CPUS             = 0x1
 )
diff --git a/vendor/golang.org/x/sys/windows/syscall_windows.go b/vendor/golang.org/x/sys/windows/syscall_windows.go
index 1fa34fd17c5cd..5cee9a3143fd5 100644
--- a/vendor/golang.org/x/sys/windows/syscall_windows.go
+++ b/vendor/golang.org/x/sys/windows/syscall_windows.go
@@ -313,6 +313,10 @@ func NewCallbackCDecl(fn interface{}) uintptr {
 //sys	SetConsoleMode(console Handle, mode uint32) (err error) = kernel32.SetConsoleMode
 //sys	GetConsoleScreenBufferInfo(console Handle, info *ConsoleScreenBufferInfo) (err error) = kernel32.GetConsoleScreenBufferInfo
 //sys	setConsoleCursorPosition(console Handle, position uint32) (err error) = kernel32.SetConsoleCursorPosition
+//sys	GetConsoleCP() (cp uint32, err error) = kernel32.GetConsoleCP
+//sys	GetConsoleOutputCP() (cp uint32, err error) = kernel32.GetConsoleOutputCP
+//sys	SetConsoleCP(cp uint32) (err error) = kernel32.SetConsoleCP
+//sys	SetConsoleOutputCP(cp uint32) (err error) = kernel32.SetConsoleOutputCP
 //sys	WriteConsole(console Handle, buf *uint16, towrite uint32, written *uint32, reserved *byte) (err error) = kernel32.WriteConsoleW
 //sys	ReadConsole(console Handle, buf *uint16, toread uint32, read *uint32, inputControl *byte) (err error) = kernel32.ReadConsoleW
 //sys	resizePseudoConsole(pconsole Handle, size uint32) (hr error) = kernel32.ResizePseudoConsole
diff --git a/vendor/golang.org/x/sys/windows/types_windows.go b/vendor/golang.org/x/sys/windows/types_windows.go
index 3f03b3d57cc35..7b97a154c9573 100644
--- a/vendor/golang.org/x/sys/windows/types_windows.go
+++ b/vendor/golang.org/x/sys/windows/types_windows.go
@@ -1060,6 +1060,7 @@ const (
 	SIO_GET_EXTENSION_FUNCTION_POINTER = IOC_INOUT | IOC_WS2 | 6
 	SIO_KEEPALIVE_VALS                 = IOC_IN | IOC_VENDOR | 4
 	SIO_UDP_CONNRESET                  = IOC_IN | IOC_VENDOR | 12
+	SIO_UDP_NETRESET                   = IOC_IN | IOC_VENDOR | 15
 
 	// cf. http://support.microsoft.com/default.aspx?scid=kb;en-us;257460
 
diff --git a/vendor/golang.org/x/sys/windows/zsyscall_windows.go b/vendor/golang.org/x/sys/windows/zsyscall_windows.go
index 9bb979a3e4726..4c2e1bdc01ed3 100644
--- a/vendor/golang.org/x/sys/windows/zsyscall_windows.go
+++ b/vendor/golang.org/x/sys/windows/zsyscall_windows.go
@@ -247,7 +247,9 @@ var (
 	procGetCommandLineW                                      = modkernel32.NewProc("GetCommandLineW")
 	procGetComputerNameExW                                   = modkernel32.NewProc("GetComputerNameExW")
 	procGetComputerNameW                                     = modkernel32.NewProc("GetComputerNameW")
+	procGetConsoleCP                                         = modkernel32.NewProc("GetConsoleCP")
 	procGetConsoleMode                                       = modkernel32.NewProc("GetConsoleMode")
+	procGetConsoleOutputCP                                   = modkernel32.NewProc("GetConsoleOutputCP")
 	procGetConsoleScreenBufferInfo                           = modkernel32.NewProc("GetConsoleScreenBufferInfo")
 	procGetCurrentDirectoryW                                 = modkernel32.NewProc("GetCurrentDirectoryW")
 	procGetCurrentProcessId                                  = modkernel32.NewProc("GetCurrentProcessId")
@@ -347,8 +349,10 @@ var (
 	procSetCommMask                                          = modkernel32.NewProc("SetCommMask")
 	procSetCommState                                         = modkernel32.NewProc("SetCommState")
 	procSetCommTimeouts                                      = modkernel32.NewProc("SetCommTimeouts")
+	procSetConsoleCP                                         = modkernel32.NewProc("SetConsoleCP")
 	procSetConsoleCursorPosition                             = modkernel32.NewProc("SetConsoleCursorPosition")
 	procSetConsoleMode                                       = modkernel32.NewProc("SetConsoleMode")
+	procSetConsoleOutputCP                                   = modkernel32.NewProc("SetConsoleOutputCP")
 	procSetCurrentDirectoryW                                 = modkernel32.NewProc("SetCurrentDirectoryW")
 	procSetDefaultDllDirectories                             = modkernel32.NewProc("SetDefaultDllDirectories")
 	procSetDllDirectoryW                                     = modkernel32.NewProc("SetDllDirectoryW")
@@ -2162,6 +2166,15 @@ func GetComputerName(buf *uint16, n *uint32) (err error) {
 	return
 }
 
+func GetConsoleCP() (cp uint32, err error) {
+	r0, _, e1 := syscall.Syscall(procGetConsoleCP.Addr(), 0, 0, 0, 0)
+	cp = uint32(r0)
+	if cp == 0 {
+		err = errnoErr(e1)
+	}
+	return
+}
+
 func GetConsoleMode(console Handle, mode *uint32) (err error) {
 	r1, _, e1 := syscall.Syscall(procGetConsoleMode.Addr(), 2, uintptr(console), uintptr(unsafe.Pointer(mode)), 0)
 	if r1 == 0 {
@@ -2170,6 +2183,15 @@ func GetConsoleMode(console Handle, mode *uint32) (err error) {
 	return
 }
 
+func GetConsoleOutputCP() (cp uint32, err error) {
+	r0, _, e1 := syscall.Syscall(procGetConsoleOutputCP.Addr(), 0, 0, 0, 0)
+	cp = uint32(r0)
+	if cp == 0 {
+		err = errnoErr(e1)
+	}
+	return
+}
+
 func GetConsoleScreenBufferInfo(console Handle, info *ConsoleScreenBufferInfo) (err error) {
 	r1, _, e1 := syscall.Syscall(procGetConsoleScreenBufferInfo.Addr(), 2, uintptr(console), uintptr(unsafe.Pointer(info)), 0)
 	if r1 == 0 {
@@ -3038,6 +3060,14 @@ func SetCommTimeouts(handle Handle, timeouts *CommTimeouts) (err error) {
 	return
 }
 
+func SetConsoleCP(cp uint32) (err error) {
+	r1, _, e1 := syscall.Syscall(procSetConsoleCP.Addr(), 1, uintptr(cp), 0, 0)
+	if r1 == 0 {
+		err = errnoErr(e1)
+	}
+	return
+}
+
 func setConsoleCursorPosition(console Handle, position uint32) (err error) {
 	r1, _, e1 := syscall.Syscall(procSetConsoleCursorPosition.Addr(), 2, uintptr(console), uintptr(position), 0)
 	if r1 == 0 {
@@ -3054,6 +3084,14 @@ func SetConsoleMode(console Handle, mode uint32) (err error) {
 	return
 }
 
+func SetConsoleOutputCP(cp uint32) (err error) {
+	r1, _, e1 := syscall.Syscall(procSetConsoleOutputCP.Addr(), 1, uintptr(cp), 0, 0)
+	if r1 == 0 {
+		err = errnoErr(e1)
+	}
+	return
+}
+
 func SetCurrentDirectory(path *uint16) (err error) {
 	r1, _, e1 := syscall.Syscall(procSetCurrentDirectoryW.Addr(), 1, uintptr(unsafe.Pointer(path)), 0, 0)
 	if r1 == 0 {
diff --git a/vendor/golang.org/x/term/term_windows.go b/vendor/golang.org/x/term/term_windows.go
index 465f560604e41..df6bf948e1407 100644
--- a/vendor/golang.org/x/term/term_windows.go
+++ b/vendor/golang.org/x/term/term_windows.go
@@ -26,6 +26,7 @@ func makeRaw(fd int) (*State, error) {
 		return nil, err
 	}
 	raw := st &^ (windows.ENABLE_ECHO_INPUT | windows.ENABLE_PROCESSED_INPUT | windows.ENABLE_LINE_INPUT | windows.ENABLE_PROCESSED_OUTPUT)
+	raw |= windows.ENABLE_VIRTUAL_TERMINAL_INPUT
 	if err := windows.SetConsoleMode(windows.Handle(fd), raw); err != nil {
 		return nil, err
 	}
diff --git a/vendor/modules.txt b/vendor/modules.txt
index b49a55d88caed..5bd662067c5c5 100644
--- a/vendor/modules.txt
+++ b/vendor/modules.txt
@@ -209,7 +209,7 @@ github.com/DmitriyVTitov/size
 # github.com/IBM/go-sdk-core/v5 v5.17.5
 ## explicit; go 1.20
 github.com/IBM/go-sdk-core/v5/core
-# github.com/IBM/ibm-cos-sdk-go v1.11.0
+# github.com/IBM/ibm-cos-sdk-go v1.11.1
 ## explicit; go 1.19
 github.com/IBM/ibm-cos-sdk-go/aws
 github.com/IBM/ibm-cos-sdk-go/aws/arn
@@ -476,7 +476,7 @@ github.com/aws/smithy-go/transport/http/internal/io
 # github.com/axiomhq/hyperloglog v0.0.0-20240507144631-af9851f82b27
 ## explicit; go 1.12
 github.com/axiomhq/hyperloglog
-# github.com/baidubce/bce-sdk-go v0.9.189
+# github.com/baidubce/bce-sdk-go v0.9.192
 ## explicit; go 1.11
 github.com/baidubce/bce-sdk-go/auth
 github.com/baidubce/bce-sdk-go/bce
@@ -730,7 +730,7 @@ github.com/facette/natsort
 # github.com/fatih/color v1.16.0
 ## explicit; go 1.17
 github.com/fatih/color
-# github.com/felixge/fgprof v0.9.4
+# github.com/felixge/fgprof v0.9.5
 ## explicit; go 1.14
 github.com/felixge/fgprof
 # github.com/felixge/httpsnoop v1.0.4
@@ -980,7 +980,7 @@ github.com/gorilla/websocket
 # github.com/grafana/cloudflare-go v0.0.0-20230110200409-c627cf6792f2
 ## explicit; go 1.17
 github.com/grafana/cloudflare-go
-# github.com/grafana/dskit v0.0.0-20240925193654-7c41a4057319
+# github.com/grafana/dskit v0.0.0-20240930165212-f52de24af9bc
 ## explicit; go 1.21
 github.com/grafana/dskit/aws
 github.com/grafana/dskit/backoff
@@ -1817,7 +1817,7 @@ go.uber.org/zap/zapgrpc
 # go4.org/netipx v0.0.0-20230125063823-8449b0a6169f
 ## explicit; go 1.18
 go4.org/netipx
-# golang.org/x/crypto v0.26.0
+# golang.org/x/crypto v0.27.0
 ## explicit; go 1.20
 golang.org/x/crypto/argon2
 golang.org/x/crypto/bcrypt
@@ -1844,7 +1844,7 @@ golang.org/x/exp/slices
 # golang.org/x/mod v0.19.0
 ## explicit; go 1.18
 golang.org/x/mod/semver
-# golang.org/x/net v0.28.0
+# golang.org/x/net v0.29.0
 ## explicit; go 1.18
 golang.org/x/net/bpf
 golang.org/x/net/context
@@ -1884,7 +1884,7 @@ golang.org/x/oauth2/jwt
 golang.org/x/sync/errgroup
 golang.org/x/sync/semaphore
 golang.org/x/sync/singleflight
-# golang.org/x/sys v0.24.0
+# golang.org/x/sys v0.25.0
 ## explicit; go 1.18
 golang.org/x/sys/cpu
 golang.org/x/sys/plan9
@@ -1892,10 +1892,10 @@ golang.org/x/sys/unix
 golang.org/x/sys/windows
 golang.org/x/sys/windows/registry
 golang.org/x/sys/windows/svc/eventlog
-# golang.org/x/term v0.23.0
+# golang.org/x/term v0.24.0
 ## explicit; go 1.18
 golang.org/x/term
-# golang.org/x/text v0.17.0
+# golang.org/x/text v0.18.0
 ## explicit; go 1.18
 golang.org/x/text/cases
 golang.org/x/text/encoding