a bunch more tests for exposure logging & evals

lib/statsig_ex.ex

@@ -83,18 +83,36 @@ defmodule StatsigEx do
   # shouldn't log anything...
   defp log_exposures(_user, [], _type), do: :ok
 
-  defp log_exposures(user, [primary | secondary], _type) do
+  defp log_exposures(user, [%{"gate" => c, "ruleID" => r} | secondary], :config) do
+    primary = %{
+      "config" => c,
+      "ruleID" => r
+    }
+
+    event =
+      base_event(user, secondary, :config)
+      |> Map.put("metadata", primary)
+
+    GenServer.call(__MODULE__, {:log, event})
+  end
+
+  defp log_exposures(user, [primary | secondary], type) do
+    event =
+      base_event(user, secondary, type)
+      |> Map.put("metadata", primary)
+
+    GenServer.call(__MODULE__, {:log, event})
+  end
+
+  defp base_event(user, secondary, type) do
     user = Utils.sanitize_user(user)
 
     event = %{
-      "eventName" => "statsig::gate_exposure",
-      "metadata" => primary,
+      "eventName" => "statsig::#{type}_exposure",
       "secondaryExposures" => secondary,
       "time" => DateTime.utc_now() |> DateTime.to_unix(:millisecond),
       "user" => user
     }
-
-    GenServer.call(__MODULE__, {:log, event})
   end
 
   defp reload_configs(api_key, since) do

lib/statsig_ex/evaluator.ex

@@ -18,6 +18,7 @@ defmodule StatsigEx.Evaluator do
 
   defmodule Result do
     defstruct exposures: [],
+              secondary_exposures: [],
               final: false,
               raw_result: false,
               result: false,
@@ -30,7 +31,21 @@ defmodule StatsigEx.Evaluator do
   def eval(user, name, type) do
     case StatsigEx.lookup(name, type) do
       [{_key, spec}] ->
-        do_eval(user, spec)
+        result = do_eval(user, spec)
+        # should we actually add the top-level exposure here...?
+        %Result{
+          result
+          | exposures: [
+              %{
+                "gate" => name,
+                "gateValue" => to_string(result.result),
+                "ruleID" => Map.get(result.rule, "id")
+              }
+              | Enum.reverse(result.exposures)
+            ]
+        }
+
+      # |> IO.inspect()
 
       _other ->
         %Result{
@@ -39,71 +54,63 @@ defmodule StatsigEx.Evaluator do
             %{"gate" => name, "gateValue" => to_string(false), "ruleID" => "Unrecognized"}
           ]
         }
-
-        # {false, false, %{}, %{"id" => "Unrecognized"},
-        #  [
-        #    %{"gate" => name, "gateValue" => to_string(false), "ruleID" => "Unrecognized"}
-        #  ]}
     end
   end
 
   # erlang client doesn't log an exposure for disabled flags, so neither will I
   defp do_eval(_user, %{"enabled" => false, "defaultValue" => default}),
-    # {false, false, default, %{}, []}
-    do: %Result{value: default}
+    do: %Result{value: default, rule: %{"id" => "disabled"}}
 
   defp do_eval(user, %{"rules" => rules} = spec), do: eval_rules(user, rules, spec, [])
 
-  defp eval_rules(_user, [], %{"defaultValue" => default, "name" => name}, results) do
-    # combine all the exposures and calculate result
-    # only one rule needs to pass, but we should bail when we get to a "final" result
+  defp eval_rules(_user, [], %{"defaultValue" => default}, results) do
+    # calculate the combined result. only one rule needs to pass
     Enum.reduce(results, %Result{result: false, raw_result: true}, fn curr, acc ->
       r =
-        case Map.keys(acc.rule) do
-          [] -> curr.rule
-          _ -> acc.rule
+        case curr.raw_result do
+          false -> acc.rule
+          _ -> if Enum.empty?(acc.rule), do: curr.rule, else: acc.rule
         end
 
       %Result{
         result: curr.result || acc.result,
         raw_result: curr.raw_result && acc.raw_result,
-        value: acc.value && curr.value,
+        value: acc.value || curr.value,
         rule: r,
         exposures: curr.exposures ++ acc.exposures
       }
     end)
     |> case do
       # in this case, we apparently want to list the rule_id as "default",
       # because we are falling back to the default
-      # (at least, that's what the erlang client does :shrug:)
-      # only add an exposure if there isn't already one, I guess?
-      %{result: false, exposures: []} ->
-        # do we always log the default exposure?
-        # so...should this be true?
+      %{result: false, rule: r, exposures: []} ->
         %Result{
           result: false,
           raw_result: true,
           value: default,
-          rule: %{"id" => "default"},
-          exposures: [
-            %{
-              "gate" => name,
-              "gateValue" => to_string(false),
-              "ruleID" => "default"
-            }
-          ]
+          rule: %{"id" => Map.get(r, "id", "default")}
+          # I don't know why this wouldn't be "default"
+          # rule: %{"id" => "default"}
         }
 
-      %{result: false} = r ->
-        %{r | raw_result: true, value: default, rule: %{"id" => "default"}}
+      # %{result: false} = r ->
+      #   %Result{
+      #     r
+      #     | value: default,
+      #       rule: %{"id" => "default"}
+      #   }
 
       pass ->
         pass
     end
   end
 
-  defp eval_rules(user, [%{"id" => id} = rule | rest], %{"name" => name} = spec, acc) do
-    # eval rules, and then
+  defp eval_rules(
+         user,
+         [%{"id" => id} = rule | rest],
+         %{"name" => name} = spec,
+         acc
+       ) do
     eval_one_rule(user, rule, spec)
     |> case do
       # once we find a passing rule, we bail
@@ -112,19 +119,20 @@ defmodule StatsigEx.Evaluator do
         final_result = eval_pass_percent(user, rule, spec)
 
         eval_rules(user, [], spec, [
-          %{
+          %Result{
             result
             | result: final_result,
               value: Map.get(rule, "returnValue"),
-              exposures: [
-                %{
-                  "gate" => name,
-                  "ruleID" => id,
-                  # not sure if this should be raw or just true?
-                  "gateValue" => to_string(final_result)
-                }
-                | exp
-              ]
+              exposures: exp
+              # [
+              # %{
+              #   "gate" => name,
+              #   "ruleID" => id,
+              #   # not sure if this should be raw or just true?
+              #   "gateValue" => to_string(final_result)
+              # }
+              # | exp
+              # ]
           }
           | acc
         ])
@@ -160,17 +168,23 @@ defmodule StatsigEx.Evaluator do
 
   # public conditions are final, so short-circuit this and return
   # gotta figure out how to make these final when they happen via pass/fail_gate
-  defp eval_conditions(_user, [%{"type" => "public"} | _rest], rule, _spec, acc),
-    do: [
-      %Result{
-        result: true,
-        raw_result: true,
-        value: Map.get(rule, "returnValue"),
-        rule: rule,
-        final: true
-      }
-      | acc
-    ]
+  defp eval_conditions(
+         _user,
+         [%{"type" => "public"} | _rest],
+         rule,
+         _spec,
+         acc
+       ),
+       do: [
+         %Result{
+           result: true,
+           raw_result: true,
+           value: Map.get(rule, "returnValue"),
+           rule: rule,
+           final: true
+         }
+         | acc
+       ]
 
   defp eval_conditions(
          user,
@@ -229,6 +243,7 @@ defmodule StatsigEx.Evaluator do
             res
             | result: false,
               raw_result: false,
+              # is this right? or should it be the default value?
               value: Map.get(rule, "returnValue"),
               rule: rule
           }

statsig_erl/src/test/test_network.erl

@@ -7,7 +7,7 @@ request(_APIKey, "rgstr", _Input) ->
 request(_APIKey, _Endpoint, _Input) ->
     erlang:display("using the test network client"),
     % this only works when running tests from the root statsig_ex directory
-    Path = filename:absname("test/data/simple_config.json"),
+    Path = filename:absname("test/data/rulesets_e2e_config.json"),
     case file:read_file(Path) of
         {ok, Body} ->
             Body;

test/consistency_test.exs

@@ -1,26 +1,38 @@
 defmodule StatsigEx.ConsistencyTest do
   use ExUnit.Case
   import StatsigEx.TestGenerator
+  import StatsigEx.PressureTest
   alias StatsigEx.Evaluator
 
-  def filter_unsupported([], acc), do: acc
-
-  def filter_unsupported([suite | rest], acc) do
-    filtered_gates =
-      Enum.filter(Map.get(suite, "feature_gates_v2"), fn gate ->
-        !all_conditions_supported?(gate, :gate)
-      end)
-
-    filter_unsupported([Map.put(suite, "feature_gates_v2", filtered_gates) | acc])
-  end
-
   "test/data/rulesets_e2e_expected_results.json"
   |> Path.expand()
   |> File.read!()
   |> Jason.decode!()
   |> Map.get("data")
   |> generate_tests()
 
+  test "one test" do
+    assert StatsigEx.Evaluator.eval(
+             %{
+               "appVersion" => "1.3",
+               "custom" => %{"bigInt" => 9_223_372_036_854_776_000},
+               "ip" => "1.0.0.0",
+               "locale" => "en_US",
+               "userAgent" =>
+                 "Mozilla/5.0 (iPhone; CPU iPhone OS 10_3_1 like Mac OS X) AppleWebKit/603.1.30 (KHTML, like Gecko) Version/10.0 Mobile/14E304 Safari/602.1",
+               "userID" => "123"
+             },
+             "operating_system_config",
+             :config
+           ).value() == %{
+             "arr" => ["hi", "there"],
+             "bool" => true,
+             "num" => 13,
+             "obj" => %{"a" => "bc"},
+             "str" => "hello"
+           }
+  end
+
   defp run_tests(
          %{
            "user" => user,
@@ -61,24 +73,4 @@ defmodule StatsigEx.ConsistencyTest do
   end
 
   # for now, just skip these, because we don't pull ID lists yet
-  defp all_conditions_supported?(gate, :gate)
-       when gate in ["test_not_in_id_list", "test_id_list"],
-       do: false
-
-  defp all_conditions_supported?(gate, type) do
-    case StatsigEx.lookup(gate, type) do
-      [{_key, spec}] ->
-        # IO.inspect(spec, label: Map.get(spec, "name"))
-
-        Enum.reduce(Map.get(spec, "rules"), true, fn %{"conditions" => c}, acc ->
-          acc &&
-            Enum.reduce(c, true, fn %{"type" => type, "operator" => op}, c_acc ->
-              c_acc && !Enum.any?(["ip_based"], fn n -> n == type end)
-            end)
-        end)
-
-      _ ->
-        false
-    end
-  end
 end

test/exposure_logging_test.exs

@@ -1,33 +1,43 @@
 defmodule StatsigEx.ExposureLoggingTest do
   use ExUnit.Case
+  import StatsigEx.PressureTest
+  alias StatsigEx.Evaluator
 
-  # ------------------------------------------------------------------------
-  # NOTE: these will all currently fail because the erlang & elixir clients
-  #       are loading different configs
-  # ------------------------------------------------------------------------
-  test "exposure logging on simple gate" do
-    compare_logs(%{}, "public", :gate)
-  end
-
-  test "exposure logging on more complex gate" do
-    compare_logs(%{"userID" => "123"}, "multiple_conditions_per_rule", :gate)
-  end
+  test "all existing configs" do
+    data =
+      "test/data/rulesets_e2e_expected_results.json"
+      |> Path.expand()
+      |> File.read!()
+      |> Jason.decode!()
+      |> Map.get("data")
+      |> Enum.each(fn %{"user" => user, "dynamic_configs" => configs} ->
+        Enum.each(configs, fn {name, %{"secondary_exposures" => sec}} ->
+          IO.inspect(user)
+          IO.inspect(name)
 
-  test "exposure logging on pass gate" do
-    compare_logs(%{"userID" => "123"}, "pass-gate", :gate)
+          if all_conditions_supported?(name, :config) do
+            [_ | exp] = Evaluator.eval(user, name, :config).exposures
+            assert Enum.sort(sec) == Enum.sort(exp)
+            compare_logs(user, name, :config)
+          end
+        end)
+      end)
   end
 
-  test "exposure logging on more complex, multi-rule gate" do
-    compare_logs(%{"userID" => "lkjlk"}, "complex-gate", :gate)
-  end
-
-  test "exposure logging on non-existent gate" do
-    compare_logs(%{"userID" => "123"}, "xxxxxxxxx", :gate)
-  end
+  test "one config" do
+    user = %{
+      "appVersion" => "1.2.3-alpha",
+      "ip" => "1.0.0.0",
+      "locale" => "en_US",
+      "userAgent" =>
+        "Mozilla/5.0 (iPhone; CPU iPhone OS 10_3_1 like Mac OS X) AppleWebKit/603.1.30 (KHTML, like Gecko) Version/10.0 Mobile/14E304 Safari/602.1",
+      "userID" => "123"
+    }
 
-  test "private attributes are properly dropped from logs" do
-    user = %{"userID" => "123", "privateAttributes" => %{"secret" => "key"}}
-    compare_logs(user, "complex-gate", :gate)
+    name = "operating_system_config"
+    r = Evaluator.eval(user, name, :config)
+    IO.inspect(r, label: :result)
+    compare_logs(user, name, :config)
   end
 
   defp compare_logs(user, id, type) do