Merge pull request #1136 from Drakkar-Software/dev

Dev merge

Evaluator/TA/ai_evaluator/ai.py

@@ -48,7 +48,9 @@ class GPTEvaluator(evaluators.TAEvaluator):
         "Detrended Price Oscillator": tulipy.dpo,
     }
     SOURCES = ["Open", "High", "Low", "Close", "Volume", "Full candle (For no indicator only)"]
+    ALLOW_GPT_REEVALUATION_ENV = "ALLOW_GPT_REEVALUATIONS"
     GPT_MODELS = []
+    ALLOW_TOKEN_LIMIT_UPDATE = False
 
     def __init__(self, tentacles_setup_config):
         super().__init__(tentacles_setup_config)
@@ -67,7 +69,8 @@ def __init__(self, tentacles_setup_config):
                     commons_enums.TimeFramesMinutes[commons_enums.TimeFrames(self.min_allowed_timeframe)]
         except ValueError:
             self.logger.error(f"Invalid timeframe configuration: unknown timeframe: '{self.min_allowed_timeframe}'")
-        self.allow_reevaluations = os_util.parse_boolean_environment_var("ALLOW_GPT_REEVALUATIONS", "True")
+        self.allow_reevaluations = os_util.parse_boolean_environment_var(self.ALLOW_GPT_REEVALUATION_ENV, "True")
+        self.gpt_tokens_limit = gpt_service.GPTService.NO_TOKEN_LIMIT_VALUE
         self.services_config = None
 
     def enable_reevaluation(self) -> bool:
@@ -122,6 +125,21 @@ def init_user_inputs(self, inputs: dict) -> None:
                 inputs, options=list(self.GPT_MODELS),
                 title="GPT Model: the GPT model to use."
             )
+        if os_util.parse_boolean_environment_var(self.ALLOW_GPT_REEVALUATION_ENV, "True"):
+            self.allow_reevaluations = self.UI.user_input(
+                "allow_reevaluation", enums.UserInputTypes.BOOLEAN, self.allow_reevaluations,
+                inputs,
+                title="Allow Reevaluation: send a ChatGPT request when realtime evaluators trigger a "
+                      "global reevaluation Use latest available value otherwise. "
+                      "Warning: enabling this can lead to a large amount of GPT requests and consumed tokens."
+            )
+        if self.ALLOW_TOKEN_LIMIT_UPDATE:
+            self.gpt_tokens_limit = self.UI.user_input(
+                "max_gpt_tokens", enums.UserInputTypes.INT,
+                self.gpt_tokens_limit, inputs, min_val=gpt_service.GPTService.NO_TOKEN_LIMIT_VALUE,
+                title=f"OpenAI token limit: maximum daily number of tokens to consume with a given OctoBot instance. "
+                      f"Use {gpt_service.GPTService.NO_TOKEN_LIMIT_VALUE} to remove the limit."
+            )
 
     async def _init_GPT_models(self):
         if not self.GPT_MODELS:
@@ -132,6 +150,7 @@ async def _init_GPT_models(self):
                         gpt_service.GPTService, self.is_backtesting, self.services_config
                     )
                     self.GPT_MODELS = service.models
+                    self.ALLOW_TOKEN_LIMIT_UPDATE = service.allow_token_limit_update()
                 except Exception as err:
                     self.logger.exception(err, True, f"Impossible to fetch GPT models: {err}")
 
@@ -169,7 +188,10 @@ async def evaluate(self, cryptocurrency, symbol, time_frame, candle_data, candle
                 except services_errors.InvalidRequestError as e:
                     self.logger.error(f"Invalid GPT request: {e}")
                 except services_errors.RateLimitError as e:
-                    self.logger.error(f"Too many requests: {e}")
+                    self.logger.error(f"Impossible to get ChatGPT evaluation for {symbol} on {time_frame}: "
+                                      f"No remaining free tokens for today : {e}. To prevent this, you can reduce the "
+                                      f"amount of traded pairs, use larger time frames or increase the maximum "
+                                      f"allowed tokens.")
                 except services_errors.UnavailableInBacktestingError:
                     # error already logged error for backtesting in use_backtesting_init_timeout
                     pass
@@ -201,6 +223,7 @@ async def ask_gpt(self, preprompt, inputs, symbol, time_frame, candle_time) -> s
                 self.is_backtesting,
                 {} if self.is_backtesting else self.services_config
             )
+            service.apply_daily_token_limit_if_possible(self.gpt_tokens_limit)
             resp = await service.get_chat_completion(
                 [
                     service.create_message("system", preprompt),

Evaluator/TA/ai_evaluator/config/GPTEvaluator.json

@@ -1,5 +1,8 @@
 {
-    "indicator": "No indicator: the raw value of the selected source",
+    "indicator": "No indicator: raw candles price data",
     "period": 2,
-    "source": "Close"
+    "source": "Close",
+    "min_confidence_threshold": 100,
+    "allow_reevaluation": false,
+    "max_gpt_tokens": -1
 }

Evaluator/TA/ai_evaluator/resources/GPTEvaluator.md

@@ -2,6 +2,25 @@ Uses [Chat GPT](https://chat.openai.com/) to predict the next moves of the marke
 
 Evaluates between -1 to 1 according to ChatGPT's prediction of the selected data and its confidence.
 
+Learn more about ChatGPT trading strategies from our 
+[Trading with ChatGPT blog article](https://www.octobot.cloud/en/blog/trading-using-chat-gpt?utm_source=octobot&utm_medium=dk&utm_campaign=regular_open_source_content&utm_content=GPTEvaluator).
+
+<div class="text-center">
+    <div>
+
+    <iframe width="560" height="315" src="https://www.youtube.com/embed/P23oiE8gW4Y?si=6p4a25VOx74DB2Kh" 
+    title="Build your own Smart DCA strategy" frameborder="0" allow="accelerometer; autoplay; 
+    clipboard-write; encrypted-media; gyroscope; picture-in-picture" allowfullscreen></iframe>
+
+
+    </div>
+    Example of a trading strategy using ChatGPT and the ChatGPTEvaluator
+</div>
+
+Any question ? Checkout our [ChatGPT guide](https://www.octobot.cloud/en/guides/octobot-interfaces/chatgpt?utm_source=octobot&utm_medium=dk&utm_campaign=regular_open_source_content&utm_content=GPTEvaluator) to configure your OctoBot 
+to use ChatGPT.
+
 Note: this evaluator can only be used in backtesting for markets where historical ChatGPT data are available.
+Find the full list of supported historical markets on our [ChatGPT page](https://www.octobot.cloud/features/chatgpt-trading?utm_source=octobot&utm_medium=dk&utm_campaign=regular_open_source_content&utm_content=GPTEvaluator).
+
 
-Find the full list of supported historical markets on https://www.octobot.cloud/features/chatgpt-trading

Services/Interfaces/telegram_bot_interface/telegram_bot.py

@@ -177,7 +177,7 @@ async def command_pause_resume(self, update: telegram.Update, _: telegram.ext.Co
                 await TelegramBotInterface._send_message(
                     update, f"_Pausing..._{interfaces_bots.EOL}`I'm cancelling my orders.`"
                 )
-                self.set_command_pause()
+                await self.set_command_pause()
 
     @staticmethod
     async def command_ping(update: telegram.Update, _: telegram.ext.ContextTypes.DEFAULT_TYPE):
@@ -200,8 +200,9 @@ async def command_risk(update: telegram.Update, context: telegram.ext.ContextTyp
     @staticmethod
     async def command_profitability(update: telegram.Update, _: telegram.ext.ContextTypes.DEFAULT_TYPE):
         if TelegramBotInterface._is_valid_user(update):
-            await TelegramBotInterface._send_message(update, interfaces_bots.AbstractBotInterface.get_command_profitability(
-                markdown=True))
+            await TelegramBotInterface._send_message(
+                update, interfaces_bots.AbstractBotInterface.get_command_profitability(markdown=True)
+            )
 
     @staticmethod
     async def command_fees(update: telegram.Update, _: telegram.ext.ContextTypes.DEFAULT_TYPE):
@@ -214,7 +215,7 @@ async def command_fees(update: telegram.Update, _: telegram.ext.ContextTypes.DEF
     async def command_sell_all_currencies(update: telegram.Update, _: telegram.ext.ContextTypes.DEFAULT_TYPE):
         if TelegramBotInterface._is_valid_user(update):
             await TelegramBotInterface._send_message(
-                update, f"`{interfaces_bots.AbstractBotInterface.get_command_sell_all_currencies()}`"
+                update, f"`{await interfaces_bots.AbstractBotInterface.get_command_sell_all_currencies()}`"
             )
 
     @staticmethod
@@ -225,7 +226,7 @@ async def command_sell_all(update: telegram.Update, context: telegram.ext.Contex
                 await TelegramBotInterface._send_message(update, "`Require a currency in parameter of this command.`")
             else:
                 await TelegramBotInterface._send_message(
-                    update, f"`{interfaces_bots.AbstractBotInterface.get_command_sell_all(currency)}`"
+                    update, f"`{await interfaces_bots.AbstractBotInterface.get_command_sell_all(currency)}`"
                 )
 
     @staticmethod
@@ -254,7 +255,7 @@ async def command_portfolio_refresh(update: telegram.Update, _: telegram.ext.Con
         if TelegramBotInterface._is_valid_user(update):
             result = "Refresh"
             try:
-                interfaces_bots.AbstractBotInterface.set_command_portfolios_refresh()
+                await interfaces_bots.AbstractBotInterface.set_command_portfolios_refresh()
                 await TelegramBotInterface._send_message(update, f"`{result} successful`")
             except Exception as e:
                 await TelegramBotInterface._send_message(update, f"`{result} failure: {e}`")

Services/Interfaces/telegram_bot_interface/tests/test_bot_interface.py

@@ -74,8 +74,10 @@ async def test_all_commands():
         assert len(bot_interface.get_command_open_orders()) > 50
         assert len(bot_interface.get_command_fees()) > 50
         assert "Decimal" not in bot_interface.get_command_fees()
-        assert "Nothing to sell" in bot_interface.get_command_sell_all_currencies()
-        assert "Nothing to sell for BTC" in bot_interface.get_command_sell_all("BTC")
+        assert "Nothing to sell" in await bot_interface.get_command_sell_all_currencies()
+        assert "Nothing to sell for BTC" in await bot_interface.get_command_sell_all("BTC")
+        with pytest.raises(RuntimeError):
+            await bot_interface.set_command_portfolios_refresh()
         assert len(bot_interface.get_command_portfolio()) > 50
         assert "Decimal" not in bot_interface.get_command_portfolio()
         assert len(bot_interface.get_command_profitability()) > 50
@@ -85,3 +87,4 @@ async def test_all_commands():
                    for elem in
                    [interfaces.AbstractInterface.project_name, interfaces.AbstractInterface.project_version])
         assert "Hello, I'm OctoBot" in bot_interface.get_command_start()
+        assert await bot_interface.set_command_pause() is None

Services/Interfaces/web_interface/controllers/community.py

@@ -54,6 +54,7 @@ def community():
     @blueprint.route("/community_metrics")
     @login.login_required_when_activated
     def community_metrics():
+        return flask.redirect("/")
         can_get_metrics = models.can_get_community_metrics()
         display_metrics = models.get_community_metrics_to_display() if can_get_metrics else None
         return flask.render_template('community_metrics.html',

Services/Interfaces/web_interface/models/configuration.py

@@ -14,6 +14,7 @@
 #  You should have received a copy of the GNU Lesser General Public
 #  License along with this library.
 import asyncio
+import logging
 import os.path as path
 import ccxt
 import ccxt.async_support
@@ -922,6 +923,7 @@ async def _load_market(exchange, results):
                 symbols = client.symbols
         else:
             async with getattr(ccxt.async_support, exchange)({'verbose': False}) as client:
+                client.logger.setLevel(logging.INFO)    # prevent log of each request (huge on market statuses)
                 await client.load_markets()
                 symbols = client.symbols
         # filter symbols with a "." or no "/" because bot can't handle them for now

Services/Interfaces/web_interface/templates/components/community/bots_stats.html

@@ -2,18 +2,15 @@
 <div class="bg-darker text-light">
     <div>
         <div class="d-flex flex-wrap justify-content-center py-2 mt-0 mb-4 w-100">
-            <div class="community-bot-stats-label my-auto py-1 text-center mx-2 mx-md-4">
-                <span class="badge community-bot-stats" id="monthly-bots">{{ stats["monthly"] }}</span> Active OctoBots this month
-            </div>
-            <div class="community-bot-stats-label my-auto py-1 mx-2 mx-md-4">
-                <span class="badge community-bot-stats" id="daily-bots">{{ stats["daily"] }}</span> Currently trading OctoBots
-            </div>
-            <div class="community-bot-stats-label my-auto py-1 mx-2 mx-md-4">
-                <span class="badge community-bot-stats" id="all-bots">{{ stats["all"] }}</span> Already installed OctoBots
-            </div>
+
             <div class="community-bot-stats-label my-auto py-1 mx-2 mx-md-4">
-                <a class="btn btn-outline-danger waves-effect" href="{{url_for('community_metrics')}}">See more</a>
+                Welcome to the OctoBot community and its
+                <span class="badge community-bot-stats" id="all-bots">{{ stats["total_bots"] }}</span>
+                already installed OctoBots
             </div>
+<!--            <div class="community-bot-stats-label my-auto py-1 mx-2 mx-md-4">-->
+<!--                <a class="btn btn-outline-danger waves-effect" href="{{url_for('community_metrics')}}">See more</a>-->
+<!--            </div>-->
         </div>
     </div>
 </div>

Services/Interfaces/web_interface/templates/components/config/evaluator_card.html

@@ -18,8 +18,8 @@
         {% endif %}
         {{ evaluator_name }}
         <span class="float-right">
-            <span class="badge {{'badge-warning' if startup_config[evaluator_name] != info['activation'] else ('badge-success' if info['activation'] else 'badge-secondary')}}">
-                {{('Activation pending restart' if info['activation'] else 'Deactivation pending restart') if startup_config[evaluator_name] != info['activation'] else ('Activated' if info['activation'] else 'Deactivated')}}
+            <span class="badge {{'badge-warning' if (evaluator_name in startup_config and startup_config[evaluator_name] != info['activation']) else ('badge-success' if info['activation'] else 'badge-secondary')}}">
+                {{('Activation pending restart' if info['activation'] else 'Deactivation pending restart') if (evaluator_name in startup_config and startup_config[evaluator_name] != info['activation']) else ('Activated' if info['activation'] else 'Deactivated')}}
             </span>
             <button class="btn btn-outline-primary btn-md waves-effect" data-toggle="modal" data-target="#{{evaluator_name}}Modal" no-activation-click="true"><i class="fa fa-info-circle" no-activation-click="true"></i></button>
         </span>

Services/Interfaces/web_interface/templates/profile.html

@@ -98,7 +98,9 @@
         </div>
     </nav>
     <main role="main" class="col-md-9 col-lg-10 col-11 ml-auto px-4">
-        {{ m_flash_messages.flash_messages() }}
+        <div class="bg-light">
+            {{ m_flash_messages.flash_messages() }}
+        </div>
         {% if not strategy_config["trading-modes"] %}
         <div class="alert alert-danger" role="alert">
             <h4>Configuration issue</h4>

Services/Services_bases/gpt_service/gpt.py

@@ -36,6 +36,7 @@
 class GPTService(services.AbstractService):
     BACKTESTING_ENABLED = True
     DEFAULT_MODEL = "gpt-3.5-turbo"
+    NO_TOKEN_LIMIT_VALUE = -1
 
     def get_fields_description(self):
         if self._env_secret_key is None:
@@ -58,7 +59,11 @@ def __init__(self):
         self.model = os.getenv(services_constants.ENV_GPT_MODEL, self.DEFAULT_MODEL)
         self.stored_signals: tree.BaseTree = tree.BaseTree()
         self.models = []
-        self.daily_tokens_limit = int(os.getenv(services_constants.ENV_GPT_DAILY_TOKENS_LIMIT, 0))
+        self._env_daily_token_limit = int(os.getenv(
+            services_constants.ENV_GPT_DAILY_TOKENS_LIMIT,
+            self.NO_TOKEN_LIMIT_VALUE)
+        )
+        self._daily_tokens_limit = self._env_daily_token_limit
         self.consumed_daily_tokens = 1
         self.last_consumed_token_date = None
 
@@ -227,17 +232,27 @@ async def fetch_gpt_history(
     def clear_signal_history(self):
         self.stored_signals.clear()
 
+    def allow_token_limit_update(self):
+        return self._env_daily_token_limit == self.NO_TOKEN_LIMIT_VALUE
+
+    def apply_daily_token_limit_if_possible(self, updated_limit: int):
+        # do not allow updating daily_tokens_limit when set from environment variables
+        if self.allow_token_limit_update():
+            self._daily_tokens_limit = updated_limit
+
     def _supported_history_url(self):
         return f"{community.IdentifiersProvider.COMMUNITY_LANDING_URL}/features/chatgpt-trading"
 
     def _ensure_rate_limit(self):
         if self.last_consumed_token_date != datetime.date.today():
             self.consumed_daily_tokens = 0
             self.last_consumed_token_date = datetime.date.today()
-        if not self.daily_tokens_limit:
+        if self._daily_tokens_limit == self.NO_TOKEN_LIMIT_VALUE:
             return
-        if self.consumed_daily_tokens >= self.daily_tokens_limit:
-            raise errors.RateLimitError("Daily rate limit reached")
+        if self.consumed_daily_tokens >= self._daily_tokens_limit:
+            raise errors.RateLimitError(
+                f"Daily rate limit reached (used {self.consumed_daily_tokens} out of {self._daily_tokens_limit})"
+            )
 
     def _update_token_usage(self, consumed_tokens):
         self.consumed_daily_tokens += consumed_tokens

Services/Services_feeds/google_service_feed/google_feed.py

@@ -33,6 +33,10 @@ class GoogleServiceFeedChannel(services_channel.AbstractServiceFeedChannel):
 class TrendTopic:
     def __init__(self, refresh_time, keywords, category=0, time_frame="today 5-y", geo="", grop=""):
         self.keywords = keywords
+        self.sanitized_keywords = [
+            keyword.replace(" ", "+")
+            for keyword in keywords
+        ]
         self.category = category
         self.time_frame = time_frame
         self.geo = geo
@@ -72,7 +76,8 @@ def _get_sleep_time_before_next_wakeup(self):
         return max(0, closest_wakeup - time.time())
 
     async def _get_topic_trend(self, topic):
-        await self.trends_req_builder.async_build_payload(kw_list=topic.keywords,
+        self.logger.debug(f"Fetching trend on {topic.keywords} over {topic.time_frame}")
+        await self.trends_req_builder.async_build_payload(kw_list=topic.sanitized_keywords,
                                                           cat=topic.category,
                                                           timeframe=topic.time_frame,
                                                           geo=topic.geo,