From 80e4e210c1e443c19fb1b5fcb55334998d65e6e7 Mon Sep 17 00:00:00 2001 From: connorsanders Date: Tue, 12 Dec 2023 22:17:01 -0600 Subject: [PATCH] Refactored session management system to handle cookie and crumbs better. Added fixes for #167, #166, #160. --- yahoofinancials/etl.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/yahoofinancials/etl.py b/yahoofinancials/etl.py index c668782..a62fb28 100644 --- a/yahoofinancials/etl.py +++ b/yahoofinancials/etl.py @@ -197,11 +197,18 @@ def _request_handler(self, url, res_field=""): open_session = False session, crumb = _init_session(None, proxies=self._get_proxy(), timeout=self.timeout) crumb_url = cur_url + "&crumb=" + str(crumb) - response = urlopener.get_data(session, crumb_url, proxy=self._get_proxy(), timeout=self.timeout) + try: + response = urlopener.get_data(session, crumb_url, proxy=self._get_proxy(), timeout=self.timeout) + except: + continue else: - response = urlopener.open(cur_url, proxy=self._get_proxy(), timeout=self.timeout) - if response.status_code == 401: + try: + response = urlopener.open(cur_url, proxy=self._get_proxy(), timeout=self.timeout) + if response.status_code == 401: + open_session = True + except AttributeError: open_session = True + continue if response.status_code != 200: time.sleep(random.randrange(1, 5)) response.close()