diff --git a/lipo/optimizer.py b/lipo/optimizer.py
index 383d2a8..eae2e3c 100644
--- a/lipo/optimizer.py
+++ b/lipo/optimizer.py
@@ -108,6 +108,9 @@ def __init__(
         self.lower_bounds = lower_bounds
         self.upper_bounds = upper_bounds
         for name, cats in self.categories.items():
+            # we dont need to check both, lower and upper bounds
+            if name in lower_bounds:
+                continue
             self.lower_bounds[name] = 0
             self.upper_bounds[name] = len(cats) - 1