diff --git a/dialyzer.ignore-warnings b/dialyzer.ignore-warnings
deleted file mode 100644
index 276a202e..00000000
--- a/dialyzer.ignore-warnings
+++ /dev/null
@@ -1,6 +0,0 @@
-lib/appsignal/json.ex:31: Unknown function 'Elixir.Jason':encode/1
-lib/appsignal/json.ex:32: Unknown function 'Elixir.Jason':'encode!'/1
-lib/appsignal/json.ex:33: Unknown function 'Elixir.Jason':decode/1
-lib/appsignal/json.ex:34: Unknown function 'Elixir.Jason':'decode!'/1
-lib/mix/tasks/appsignal.install.ex:175: The pattern {'error', _reason@1} can never match the type 'ok'
-lib/mix/tasks/appsignal.install.ex:190
diff --git a/lib/mix/tasks/appsignal.install.ex b/lib/mix/tasks/appsignal.install.ex
index 02c0528f..5b62fa62 100644
--- a/lib/mix/tasks/appsignal.install.ex
+++ b/lib/mix/tasks/appsignal.install.ex
@@ -183,7 +183,7 @@ defmodule Mix.Tasks.Appsignal.Install do
 
     case File.open(appsignal_config_file_path(), [:write]) do
       {:ok, file} ->
-        case IO.binwrite(file, appsignal_config_file_contents(config)) do
+        case binwrite_with_result(file, appsignal_config_file_contents(config)) do
           :ok ->
             IO.puts("Success!")
 
@@ -200,6 +200,18 @@ defmodule Mix.Tasks.Appsignal.Install do
     end
   end
 
+  if Version.match?(System.version(), ">= 1.16.0") do
+    defp binwrite_with_result(path, contents) do
+      try do
+        IO.binwrite(path, contents)
+      catch
+        {:error, reason} -> {:error, reason}
+      end
+    end
+  else
+    defdelegate binwrite_with_result(path, contents), to: IO, as: :binwrite
+  end
+
   # Link the config/appsignal.exs config file to the config/config.exs file.
   # If already linked, it's ignored.
   defp link_config_file do
diff --git a/mix.exs b/mix.exs
index 3b8310b1..5716afc0 100644
--- a/mix.exs
+++ b/mix.exs
@@ -37,7 +37,6 @@ defmodule Appsignal.Mixfile do
         extras: ["README.md", "CHANGELOG.md"]
       ],
       dialyzer: [
-        ignore_warnings: "dialyzer.ignore-warnings",
         plt_file: {:no_warn, "priv/plts/dialyzer.plt"},
         plt_add_apps: [:mix]
       ]