Added a config parsing utility module to support overriding a config …

…file if a flask header key/value is present that points to a base64-encoded version of the config.

PiperOrigin-RevId: 468115534

shoptimizer_api/main.py

@@ -123,7 +123,29 @@ def optimize() -> Tuple[str, http.HTTPStatus]:
 
   app.config['DRIVE_CONFIG_OVERRIDES'] = {
       'adult_optimizer_config_override':
-          flask.request.headers.get('adult_optimizer_config_override', '')
+          flask.request.headers.get('adult_optimizer_config_override', ''),
+      'brand_blocklist_override':
+          flask.request.headers.get('brand_blocklist_override', ''),
+      'color_optimizer_config_override':
+          flask.request.headers.get('color_optimizer_config_override', ''),
+      'condition_optimizer_config_override':
+          flask.request.headers.get('condition_optimizer_config_override', ''),
+      'free_shipping_optimizer_config_override':
+          flask.request.headers.get(
+              'free_shipping_optimizer_config_override', ''),
+      'gender_optimizer_config_override':
+          flask.request.headers.get('gender_optimizer_config_override', ''),
+      'promo_text_removal_optimizer_config_override':
+          flask.request.headers.get(
+              'promo_text_removal_optimizer_config_override', ''),
+      'shopping_exclusion_optimizer_config_override':
+          flask.request.headers.get(
+              'shopping_exclusion_optimizer_config_override', ''),
+      'title_word_order_optimizer_config_override':
+          flask.request.headers.get(
+              'title_word_order_optimizer_config_override', ''),
+      'title_word_order_blocklist_override':
+          flask.request.headers.get('title_word_order_blocklist_override', '')
   }
 
   lang_url_parameter = flask.request.args.get('lang',

shoptimizer_api/optimizers_builtin/adult_optimizer.py

@@ -21,17 +21,17 @@
 the "adult" attribute to be set to True on the Content API request, this
 optimizer will set the "adult" attribute to True for that product.
 """
-import ast
-import base64
 import logging
 from typing import AbstractSet, Any, Mapping, Optional, Sequence
-from flask import current_app
 
 from models import optimization_result_counts
 from optimizers_abstract import base_optimizer
+from util import config_parser
 from util import gpc_id_to_string_converter
 from util import optimization_util
 
+_ADULT_OPTIMIZER_CONFIG_FILE_NAME = 'adult_optimizer_config_{}'
+_ADULT_OPTIMIZER_CONFIG_OVERRIDE_KEY = 'adult_optimizer_config_override'
 _GPC_STRING_TO_ID_MAPPING_CONFIG_FILE_NAME: str = 'gpc_string_to_id_mapping_{}'
 
 
@@ -61,16 +61,10 @@ def _optimize(
     Returns:
       The number of products affected by this optimization.
     """
-    adult_optimizer_config_override = current_app.config.get(
-        'DRIVE_CONFIG_OVERRIDES', {}).get('adult_optimizer_config_override', '')
-    if adult_optimizer_config_override:
-      self._adult_config = ast.literal_eval(
-          base64.b64decode(adult_optimizer_config_override).decode(
-              'UTF-8').lstrip('\n').rstrip('\n').lstrip(' ').rstrip(' '))
-      logging.info('OVERRIDDEN CONFIG CONTENTS: %s', str(self._adult_config))
-    else:
-      self._adult_config = current_app.config.get('CONFIGS', {}).get(
-          f'adult_optimizer_config_{language}', {})
+    self._adult_config = config_parser.get_config_contents(
+        _ADULT_OPTIMIZER_CONFIG_OVERRIDE_KEY,
+        _ADULT_OPTIMIZER_CONFIG_FILE_NAME.format(language))
+
     self._adult_types = frozenset(
         self._adult_config.get('adult_product_types', []))
     self._gpc_id_to_string_converter = gpc_id_to_string_converter.GPCConverter(

shoptimizer_api/optimizers_builtin/adult_optimizer_test.py

@@ -16,7 +16,7 @@
 """Unit tests for adult_optimizer.py."""
 
 from absl.testing import parameterized
-import unittest.mock as mock
+from unittest import mock
 
 from optimizers_builtin import adult_optimizer
 from test_data import requests_bodies

shoptimizer_api/optimizers_builtin/condition_optimizer.py

@@ -24,13 +24,14 @@
 import logging
 from typing import Any, Dict, List, Optional, Set
 
-from flask import current_app
-
 from models import optimization_result_counts
 from optimizers_abstract import base_optimizer
+from util import config_parser
 from util import gpc_id_to_string_converter
 from util import optimization_util
 
+_CONDITION_OPTIMIZER_CONFIG_FILENAME = 'condition_optimizer_config_{}'
+_CONDITION_OPTIMIZER_CONFIG_OVERRIDE_KEY = 'condition_optimizer_config_override'
 _GPC_STRING_TO_ID_MAPPING_CONFIG_FILE_NAME: str = 'gpc_string_to_id_mapping_{}'
 _NEW = 'new'
 _USED = 'used'
@@ -64,9 +65,9 @@ def _optimize(
     num_of_products_optimized = 0
     num_of_products_excluded = 0
 
-    self._condition_config = current_app.config.get('CONFIGS', {}).get(
-        f'condition_optimizer_config_{language}', {})
-
+    self._condition_config = config_parser.get_config_contents(
+        _CONDITION_OPTIMIZER_CONFIG_OVERRIDE_KEY,
+        _CONDITION_OPTIMIZER_CONFIG_FILENAME.format(language))
     self._gpc_id_to_string_converter = gpc_id_to_string_converter.GPCConverter(
         _GPC_STRING_TO_ID_MAPPING_CONFIG_FILE_NAME.format(language))
 

shoptimizer_api/optimizers_builtin/description_optimizer_test.py

@@ -14,7 +14,7 @@
 # limitations under the License.
 
 """Unit tests for description_optimizer.py."""
-import unittest.mock as mock
+from unittest import mock
 
 from absl.testing import parameterized
 import constants

shoptimizer_api/optimizers_builtin/free_shipping_optimizer.py

@@ -33,12 +33,15 @@
 import re
 from typing import Any, Dict, List
 
-import flask
-
 from models import optimization_result_counts
 from optimizers_abstract import base_optimizer
+from util import config_parser
 from util import optimization_util
 
+_FREE_SHIPPING_OPTIMIZER_CONFIG_NAME = 'free_shipping_optimizer_config_{}'
+_FREE_SHIPPING_OPTIMIZER_CONFIG_OVERRIDE_KEY = (
+    'free_shipping_optimizer_config_override')
+
 
 class FreeShippingOptimizer(base_optimizer.BaseOptimizer):
   """An optimizer that optimizes shipping field when shipping is free."""
@@ -62,8 +65,9 @@ def _optimize(
     num_of_products_optimized = 0
     num_of_products_excluded = 0
 
-    self._config = flask.current_app.config.get('CONFIGS', {}).get(
-        f'free_shipping_optimizer_config_{language}', {})
+    self._free_shipping_config = config_parser.get_config_contents(
+        _FREE_SHIPPING_OPTIMIZER_CONFIG_OVERRIDE_KEY,
+        _FREE_SHIPPING_OPTIMIZER_CONFIG_NAME.format(language))
 
     for entry in product_batch['entries']:
 
@@ -129,7 +133,7 @@ def _title_contains_pattern(self, product: Dict[str, Any],
       not.
     """
     title = product.get('title', '')
-    for pattern in self._config.get(config_key):
+    for pattern in self._free_shipping_config.get(config_key):
       if re.search(pattern, title):
         return True
     return False

shoptimizer_api/optimizers_builtin/promo_text_removal_optimizer_test.py

@@ -16,7 +16,7 @@
 """Unit tests for promo_text_removal_optimizer.py."""
 
 from absl.testing import parameterized
-import unittest.mock as mock
+from unittest import mock
 
 import constants
 import enums
@@ -25,8 +25,9 @@
 from util import app_util
 
 
-@mock.patch('util.promo_text_remover._PROMO_TEXT_REMOVAL_CONFIG_FILE_NAME',
-            'promo_text_removal_optimizer_config_{}_test')
+@mock.patch(
+    'util.promo_text_remover._PROMO_TEXT_REMOVAL_OPTIMIZER_CONFIG_FILE_NAME',
+    'promo_text_removal_optimizer_config_{}_test')
 class PromoTextRemovalOptimizerTest(parameterized.TestCase):
 
   def setUp(self):

shoptimizer_api/optimizers_builtin/shopping_exclusion_optimizer.py

@@ -22,10 +22,9 @@
 import logging
 from typing import Any, Dict, Optional, Set
 
-import flask
-
 from models import optimization_result_counts
 from optimizers_abstract import base_optimizer
+from util import config_parser
 from util import optimization_util
 
 _EXCLUDED_DESTINATIONS_KEY = 'excludedDestinations'
@@ -34,6 +33,11 @@
 _SHOPPING_ADS_DESTINATION = 'Shopping_ads'
 _FREE_LISTINGS_DESTINATION = 'Free_listings'
 
+_SHOPPING_EXCLUSION_OPTIMIZER_CONFIG_FILE_NAME = (
+    'shopping_exclusion_optimizer_config_{}')
+_SHOPPING_EXCLUSION_OPTIMIZER_CONFIG_OVERRIDE_KEY = (
+    'shopping_exclusion_optimizer_config_override')
+
 
 class ShoppingExclusionOptimizer(base_optimizer.BaseOptimizer):
   """An optimizer that detects and excludes products from Shopping Ads."""
@@ -59,9 +63,10 @@ def _optimize(
     num_of_products_optimized = 0
     num_of_products_excluded = 0
 
-    self._shopping_exclusion_config = flask.current_app.config.get(
-        'CONFIGS', {}).get(f'shopping_exclusion_optimizer_config_{language}',
-                           {})
+    self._shopping_exclusion_config = config_parser.get_config_contents(
+        _SHOPPING_EXCLUSION_OPTIMIZER_CONFIG_OVERRIDE_KEY,
+        _SHOPPING_EXCLUSION_OPTIMIZER_CONFIG_FILE_NAME.format(language))
+
     self.shopping_removal_patterns_exact_match = frozenset(
         self._shopping_exclusion_config.get(
             'shopping_exclusion_patterns_exact_match', []))

shoptimizer_api/optimizers_builtin/title_word_order_optimizer.py

@@ -41,17 +41,21 @@
 
 from models import optimization_result_counts
 from optimizers_abstract import base_optimizer
+from util import config_parser
 from util import gpc_id_to_string_converter
 from util import optimization_util
 from util import regex_util
 
 _MAX_KEYWORDS_PER_TITLE = 3
 _MAX_TITLE_LENGTH = 150
 
-_GPC_STRING_TO_ID_MAPPING_CONFIG_FILE_NAME: str = 'gpc_string_to_id_mapping_{}'
-_TITLE_WORD_ORDER_CONFIG_FILE_NAME: str = 'title_word_order_config_{}'
-_TITLE_WORD_ORDER_BLOCKLIST_FILE_NAME: str = 'title_word_order_blocklist_{}'
-_TITLE_WORD_ORDER_OPTIONS_FILE_NAME: str = 'title_word_order_options'
+_GPC_STRING_TO_ID_MAPPING_CONFIG_FILE_NAME = 'gpc_string_to_id_mapping_{}'
+_TITLE_WORD_ORDER_OPTIMIZER_CONFIG_FILE_NAME = 'title_word_order_config_{}'
+_TITLE_WORD_ORDER_OPTIMIZER_CONFIG_OVERRIDE_KEY = (
+    'title_word_order_optimizer_config_override')
+_TITLE_WORD_ORDER_BLOCKLIST_FILE_NAME = 'title_word_order_blocklist_{}'
+_TITLE_WORD_ORDER_BLOCKLIST_OVERRIDE_KEY = 'title_word_order_blocklist_override'
+_TITLE_WORD_ORDER_OPTIONS_FILE_NAME = 'title_word_order_options'
 
 _KEYWORD_WEIGHTS_MAPPING_CONFIG_KEY = 'keyword_weights_by_gpc'
 _PHRASE_DICTIONARY_CONFIG_KEY = 'phrase_dictionary'
@@ -84,13 +88,13 @@ def _get_configs_from_environment(language: str):
         current_app.config.get('CONFIGS', {}).get(
             _GPC_STRING_TO_ID_MAPPING_CONFIG_FILE_NAME.format(language), {}))
 
-    title_word_order_config = (
-        current_app.config.get('CONFIGS', {}).get(
-            _TITLE_WORD_ORDER_CONFIG_FILE_NAME.format(language), {}))
+    title_word_order_config = config_parser.get_config_contents(
+        _TITLE_WORD_ORDER_OPTIMIZER_CONFIG_OVERRIDE_KEY,
+        _TITLE_WORD_ORDER_OPTIMIZER_CONFIG_FILE_NAME.format(language))
 
-    blocklist_config = (
-        current_app.config.get('CONFIGS', {}).get(
-            _TITLE_WORD_ORDER_BLOCKLIST_FILE_NAME.format(language), {}))
+    blocklist_config = config_parser.get_config_contents(
+        _TITLE_WORD_ORDER_BLOCKLIST_OVERRIDE_KEY,
+        _TITLE_WORD_ORDER_BLOCKLIST_FILE_NAME.format(language))
 
     title_word_order_options = (
         current_app.config.get('CONFIGS',

shoptimizer_api/optimizers_builtin/title_word_order_optimizer_test.py

@@ -16,7 +16,7 @@
 """Unit tests for title_word_order_optimizer."""
 
 from absl.testing import parameterized
-import unittest.mock as mock
+from unittest import mock
 
 from optimizers_builtin import title_word_order_optimizer
 from test_data import requests_bodies
@@ -34,13 +34,14 @@
 _GPC_CATEGORY_LEVEL_4_JA = ('ファッション・アクセサリー > ' '衣料品 > アウター > ' 'コート・ジャケット')
 
 
-@mock.patch('util.promo_text_remover._PROMO_TEXT_REMOVAL_CONFIG_FILE_NAME',
-            'promo_text_removal_optimizer_config_{}_test')
+@mock.patch(
+    'util.promo_text_remover._PROMO_TEXT_REMOVAL_OPTIMIZER_CONFIG_FILE_NAME',
+    'promo_text_removal_optimizer_config_{}_test')
 @mock.patch(
     'optimizers_builtin.title_word_order_optimizer._GPC_STRING_TO_ID_MAPPING_CONFIG_FILE_NAME',
     'gpc_string_to_id_mapping_{}_test')
 @mock.patch(
-    'optimizers_builtin.title_word_order_optimizer._TITLE_WORD_ORDER_CONFIG_FILE_NAME',
+    'optimizers_builtin.title_word_order_optimizer._TITLE_WORD_ORDER_OPTIMIZER_CONFIG_FILE_NAME',
     'title_word_order_config_{}_test')
 @mock.patch(
     'optimizers_builtin.title_word_order_optimizer._TITLE_WORD_ORDER_BLOCKLIST_FILE_NAME',

shoptimizer_api/util/attribute_miner.py

@@ -26,9 +26,12 @@
 
 import original_types
 from util import color_miner
+from util import config_parser
 from util import gender_miner
 from util import size_miner
 
+_BRAND_BLOCKLIST_CONFIG_NAME = 'brand_blocklist'
+_BRAND_BLOCKLIST_OVERRIDE_KEY = 'brand_blocklist_override'
 _MAX_BRAND_LENGTH: int = 70
 
 
@@ -49,8 +52,8 @@ def __init__(self, language: str, country: str) -> None:
       country: The configured country code.
     """
     super(AttributeMiner, self).__init__()
-    brand_blocklist_config = current_app.config.get('CONFIGS', {}).get(
-        'brand_blocklist', {})
+    brand_blocklist_config = config_parser.get_config_contents(
+        _BRAND_BLOCKLIST_OVERRIDE_KEY, _BRAND_BLOCKLIST_CONFIG_NAME)
     self._brand_blocklist = set(
         [brand.lower() for brand in brand_blocklist_config])
 

shoptimizer_api/util/color_miner.py

@@ -27,15 +27,16 @@
 import logging
 import subprocess
 from typing import Any, Dict, List, Optional, Tuple
-from flask import current_app
 
 import MeCab
 
 import constants
+from util import config_parser
 from util import gpc_id_to_string_converter
 from util import optimization_util
 
 _COLOR_OPTIMIZER_CONFIG_FILE_NAME: str = 'color_optimizer_config_{}'
+_COLOR_OPTIMIZER_CONFIG_OVERRIDE_KEY = 'color_optimizer_config_override'
 _COLOR_TERMS_CONFIG_KEY: str = 'color_terms'
 _GPC_STRING_TO_ID_MAPPING_CONFIG_FILE_NAME: str = 'gpc_string_to_id_mapping_{}'
 
@@ -57,8 +58,11 @@ def __init__(self, language: str) -> None:
     """
     super(ColorMiner, self).__init__()
     self._language = language
-    self._color_config = current_app.config.get('CONFIGS', {}).get(
-        _COLOR_OPTIMIZER_CONFIG_FILE_NAME.format(language), {})
+
+    self._color_config = config_parser.get_config_contents(
+        _COLOR_OPTIMIZER_CONFIG_OVERRIDE_KEY,
+        _COLOR_OPTIMIZER_CONFIG_FILE_NAME.format(language))
+
     self._gpc_id_to_string_converter = gpc_id_to_string_converter.GPCConverter(
         _GPC_STRING_TO_ID_MAPPING_CONFIG_FILE_NAME.format(language))
     if self._language == constants.LANGUAGE_CODE_JA:

shoptimizer_api/util/color_miner_test.py

@@ -16,7 +16,7 @@
 """Unit tests for color_miner.py."""
 
 from typing import Any, Dict, List, Optional
-import unittest.mock as mock
+from unittest import mock
 
 from absl.testing import parameterized