diff --git a/docs/src/examples/snippets.rst b/docs/src/examples/snippets.rst index 2b02079a..256f4953 100644 --- a/docs/src/examples/snippets.rst +++ b/docs/src/examples/snippets.rst @@ -5,11 +5,11 @@ Snippets ================== Here are some snippets showcasing how the library can be used. -- `Authenticating using OAuth2`_ - `All the songs of an artist`_ - `Artist's least popular song`_ -- `Getting songs that have a tag` +- `Authenticating using OAuth2`_ - `Getting song lyrics by URL or ID`_ +- `Getting songs by tag (genre)`_ - `Getting the lyrics for all songs of a search`_ - `Searching for a song by lyrics`_ - `YouTube URL of artist's songs`_ @@ -23,7 +23,7 @@ Getting song lyrics by URL or ID # Using Song URL url = "https://genius.com/Andy-shauf-begin-again-lyrics" - genius.lyrics(url) + genius.lyrics(song_url=url) # Using Song ID # Requires an extra request to get song URL @@ -101,8 +101,8 @@ Using :meth:`search_all `: for hit in request['sections'][2]['hits']: print(hit['result']['title']) -Getting songs that have a tag ------------------------------ +Getting songs by tag (genre) +---------------------------- Genius has the following main tags: ``rap``, ``pop``, ``r-b``, ``rock``, ``country``, ``non-music`` To discover more tags, visit the `Genius Tags`_ page. @@ -121,7 +121,7 @@ Genius probably has more than 1000 songs with the pop tag. while page: res = genius.tag('pop', page=page) for hit in res['hits']: - song_lyrics = genius.lyrics(hit['url']) + song_lyrics = genius.lyrics(song_url=hit['url']) lyrics.append(song_lyrics) page = res['next_page'] @@ -135,7 +135,7 @@ Getting the lyrics for all songs of a search songs = genius.search_songs('Begin Again Andy Shauf') for song in songs['hits']: url = song['result']['url'] - song_lyrics = genius.lyrics(url) + song_lyrics = genius.lyrics(song_url=url) # id = song['result']['id'] # song_lyrics = genius.lyrics(id) lyrics.append(song_lyrics) @@ -170,7 +170,6 @@ URI will work (for example ``http://example.com/callback``) from lyricsgenius import OAuth2, Genius - # you can also use OAuth2.full_code_exchange() auth = OAuth2.client_only_app( 'my_client_id', 'my_redirect_uri', @@ -199,14 +198,18 @@ Authenticating another user 'my_client_id', 'my_redirect_uri', 'my_client_secret', - scope='all' + scope='all', + state='some_unique_value' ) # this part is the same url_for_user = auth.url print('Redirecting you to ' + url_for_user) - redirected_url = 'https://example.com/?code=some_code' - token = auth.get_user_token(redirected_url) + + # If we were using Flask: + code = request.args.get('code') + state = request.args.get('state') + token = auth.get_user_token(code, state) genius = Genius(token) diff --git a/docs/src/reference/types.rst b/docs/src/reference/types.rst index bfcbfbbf..06ac0e2f 100644 --- a/docs/src/reference/types.rst +++ b/docs/src/reference/types.rst @@ -25,17 +25,67 @@ Classes :nosignatures: Stats + Track .. autoclass:: Stats :members: :member-order: bysource :no-show-inheritance: +.. autoclass:: Track + :members: + :member-order: bysource + :no-show-inheritance: + Album ------ An album from Genius that has the album's songs and their lyrics. +Attributes +^^^^^^^^^^ +.. list-table:: + :header-rows: 1 + + * - Attribute + - Type + + * - _type + - :obj:`str` + + * - api_path + - :obj:`str` + + * - artist + - :class:`Artist` + + * - cover_art_thumbnail_url + - :obj:`str` + + * - cover_art_url + - :obj:`str` + + * - full_title + - :obj:`str` + + * - id + - :obj:`int` + + * - name + - :obj:`str` + + * - name_with_artist + - :obj:`str` + + * - release_date_components + - :class:`datetime` + + * - tracks + - :obj:`list` of :class:`Track` + + * - url + - :obj:`str` + Methods ^^^^^^^^ @@ -59,6 +109,41 @@ Artist The Artist object which holds the details of the artist and the `Song`_ objects of that artist. +Attributes +^^^^^^^^^^ +.. list-table:: + :header-rows: 1 + + * - Attribute + - Type + + + * - api_path + - :obj:`str` + + * - header_image_url + - :obj:`str` + + * - id + - :obj:`int` + + * - image_url + - :obj:`str` + + * - is_meme_verified + - :obj:`bool` + + * - is_verified + - :obj:`bool` + + * - name + - :obj:`str` + + * - songs + - :obj:`list` + + * - url + - :obj:`str` Methods ^^^^^^^^ @@ -83,6 +168,72 @@ Song ---- This is the Song object which holds the details of the song. +Attributes +^^^^^^^^^^ +.. list-table:: + :header-rows: 1 + + * - Attribute + - Type + + + * - annotation_count + - :obj:`int` + + * - api_path + - :obj:`str` + + * - artist + - :obj:`str` + + * - full_title + - :obj:`str` + + * - header_image_thumbnail_url + - :obj:`str` + + * - header_image_url + - :obj:`str` + + * - id + - :obj:`int` + + * - lyrics + - :obj:`str` + + * - lyrics_owner_id + - :obj:`int` + + * - lyrics_state + - :obj:`str` + + * - path + - :obj:`str` + + * - primary_artist + - :class:`Artist` + + * - pyongs_count + - :obj:`int` + + * - song_art_image_thumbnail_url + - :obj:`str` + + * - song_art_image_url + - :obj:`str` + + * - stats + - :class:`Stats` + + * - title + - :obj:`str` + + * - title_with_featured + - :obj:`str` + + * - url + - :obj:`str` + Methods ^^^^^^^^ .. autosummary:: diff --git a/docs/src/release_notes.rst b/docs/src/release_notes.rst index 1222886a..926ff8aa 100644 --- a/docs/src/release_notes.rst +++ b/docs/src/release_notes.rst @@ -3,6 +3,56 @@ Release notes ============= +3.0.0 (2021-02-08) +------------------ +New +*** + +- All requests now go through the ``Sender`` object. This provides + features such as retries ``genius.retries`` and handling HTTP and + timeout errors. For more info have a look at the guide about `request + error handling`_. +- Added ``OAuth2`` class to help with OAuth2 authentication. +- Added ``PublicAPI`` class to allow accessing methods of the public + API (genius.com/api). Check `this page`_ for a list of available + methods. +- Added the ``Album`` type and the ``genius.search_album()`` method. +- Added the ``genius.tag()`` method to get songs by tag. +- All API endpoints are now supported (e.g. ``upvote_annotation``). +- New additions to the docs. + +Changed +******* + +- ``GENIUS_CLIENT_ACCESS_TOKEN`` env var has been renamed to + ``GENIUS_ACCESS_TOKEN``. +- ``genius.client_access_token`` has been renamed to + ``genius.access_token``. +- ``genius.search_song()`` will also accept ``song_id``. +- Lyrics won't be fetched for instrumental songs and their lyrics will + be set to ``""``. You can check to see if a song is instrumental + using ``Song.instrumental``. +- Renamed all interface methods to remove redundant ``get_`` + (``genius.get_song`` is now ``genius.song``). +- Renamed the lyrics method to ``genius.lyrics()`` to allow use by + users. It accepts song URLs and song IDs. +- Reformatted the types. Some attributes won't be available anymore. + More info on the `types page`_. +- ``save_lyrics()`` will save songs with ``utf8`` encoding when + ``extension='txt'``. +- Using ``Genius()`` will check for the env var + ``GENIUS_ACCESS_TOKEN``. + +Other (CI, etc) +*************** + +- Bumped ``Sphinx`` to 3.3.0 + +.. _request error handling: https://lyricsgenius.readthedocs.io/en/master/other_guides.html#request-errors +.. _this page: https://lyricsgenius.readthedocs.io/en/latest/reference/genius.html +.. _types page: https://lyricsgenius.readthedocs.io/en/latest/reference/types.html#types + + 2.0.2 (2020-09-26) ------------------ Added @@ -15,6 +65,7 @@ Added :meth:`Artist.save_lyrics ` and :meth:`Artist.to_json ` + 2.0.1 (2020-09-20) ------------------ Changed diff --git a/docs/src/usage.rst b/docs/src/usage.rst index 8a762a1e..73c4c057 100644 --- a/docs/src/usage.rst +++ b/docs/src/usage.rst @@ -87,7 +87,7 @@ Search for five songs by ‘The Beatles’ and save the lyrics: python3 -m lyricsgenius artist "The Beatles" --max-songs 5 --save -There also examples under the docs of some methods. +You might also like checking out the :ref:`snippets` page. .. toctree:: diff --git a/lyricsgenius/__init__.py b/lyricsgenius/__init__.py index cc059ada..d1e43053 100644 --- a/lyricsgenius/__init__.py +++ b/lyricsgenius/__init__.py @@ -14,4 +14,4 @@ __url__ = 'https://github.com/johnwmillr/LyricsGenius' __description__ = 'A Python wrapper around the Genius API' __license__ = 'MIT' -__version__ = '2.0.2' +__version__ = '3.0.0' diff --git a/lyricsgenius/api/api.py b/lyricsgenius/api/api.py index 26f15976..3827c62b 100644 --- a/lyricsgenius/api/api.py +++ b/lyricsgenius/api/api.py @@ -312,7 +312,7 @@ def artist_songs(self, artist_id, per_page=None, page=None, sort='title'): Args: artist_id (:obj:`int`): Genius artist ID sort (:obj:`str`, optional): Sorting preference. - Either based on 'title' or 'popularity'. + Either based on 'title', 'popularity' or 'release_date'. per_page (:obj:`int`, optional): Number of results to return per request. It can't be more than 50. page (:obj:`int`, optional): Paginated offset (number of the page). @@ -526,11 +526,17 @@ def __init__( retries=0, **kwargs ): + + # If PublicAPI was instantiated directly + # there is no need for a token anymore + public_api_constructor = False if self.__class__.__name__ == 'Genius' else True + # Genius PublicAPI Constructor super().__init__( response_format=response_format, timeout=timeout, sleep_time=sleep_time, retries=retries, + public_api_constructor=public_api_constructor, **kwargs ) diff --git a/lyricsgenius/api/base.py b/lyricsgenius/api/base.py index f58bd9be..31d73122 100644 --- a/lyricsgenius/api/base.py +++ b/lyricsgenius/api/base.py @@ -1,5 +1,6 @@ import time import os +from json.decoder import JSONDecodeError import requests from requests.exceptions import HTTPError, Timeout @@ -18,7 +19,8 @@ def __init__( response_format='plain', timeout=5, sleep_time=0.2, - retries=0 + retries=0, + public_api_constructor=False, ): self._session = requests.Session() self._session.headers = { @@ -27,8 +29,15 @@ def __init__( } if access_token is None: access_token = os.environ.get('GENIUS_ACCESS_TOKEN') - self.access_token = 'Bearer ' + access_token if access_token else None - self.authorization_header = {'authorization': self.access_token} + + if public_api_constructor: + self.authorization_header = {} + else: + if not access_token or not isinstance(access_token, str): + raise TypeError('Invalid token') + self.access_token = 'Bearer ' + access_token + self.authorization_header = {'authorization': self.access_token} + self.response_format = response_format.lower() self.timeout = timeout self.sleep_time = sleep_time @@ -95,7 +104,10 @@ def _make_request( def get_description(e): error = str(e) - res = e.response.json() + try: + res = e.response.json() + except JSONDecodeError: + res = {} description = (res['meta']['message'] if res.get('meta') else res.get('error_description')) diff --git a/lyricsgenius/api/public_methods/article.py b/lyricsgenius/api/public_methods/article.py index 7d007c5d..b4224da2 100644 --- a/lyricsgenius/api/public_methods/article.py +++ b/lyricsgenius/api/public_methods/article.py @@ -40,6 +40,7 @@ def article_comments(self, article_id, per_page=None, page=None, text_format=Non def latest_articles(self, per_page=None, page=None, text_format=None): """Gets the latest articles on the homepage. + This method will return the featured articles that are placed on top of the Genius.com page. diff --git a/lyricsgenius/api/public_methods/cover_art.py b/lyricsgenius/api/public_methods/cover_art.py index 6425c671..2d1d8a8d 100644 --- a/lyricsgenius/api/public_methods/cover_art.py +++ b/lyricsgenius/api/public_methods/cover_art.py @@ -3,6 +3,7 @@ class CoverArtMethods(object): def cover_arts(self, album_id=None, song_id=None, text_format=None): """Gets the cover arts of an album or a song. + You must supply one of :obj:`album_id` or :obj:`song_id`. Args: diff --git a/lyricsgenius/api/public_methods/leaderboard.py b/lyricsgenius/api/public_methods/leaderboard.py index 8c94672f..fd457cf2 100644 --- a/lyricsgenius/api/public_methods/leaderboard.py +++ b/lyricsgenius/api/public_methods/leaderboard.py @@ -11,7 +11,7 @@ def leaderboard(self, This method gets data of the community charts on the Genius.com page. Args: - time_period (:obj:`str`, optional): Time period of the results + time_period (:obj:`str`, optional): Time period of the results. ('day', 'week', 'month' or 'all_time'). per_page (:obj:`int`, optional): Number of results to return per request. It can't be more than 50. @@ -42,15 +42,19 @@ def charts(self, This method gets data of the chart on the Genius.com page. Args: - time_period (:obj:`str`, optional): Time period of the results + time_period (:obj:`str`, optional): Time period of the results. + The default is `all`. ('day', 'week', 'month' or 'all_time'). chart_genre (:obj:`str`, optional): The genre of the results. + The default value is ``all``. + ('all', 'rap', 'pop', 'rb', 'rock' or 'country') per_page (:obj:`int`, optional): Number of results to return per request. It can't be more than 50. page (:obj:`int`, optional): Paginated offset (number of the page). text_format (:obj:`str`, optional): Text format of the results ('dom', 'html', 'markdown' or 'plain'). type_ (:obj:`int`, optional): The type to get the charts for. + The default is ``songs``. ('songs', 'albums', 'artists' or 'referents'). Returns: @@ -59,7 +63,7 @@ def charts(self, .. Note:: The *referents* mentioned in the description of the :obj:`type_` argument is shown as *Lyrics* in the drop-down menu on Genius.com - where you can choose the *Type*. + where you choose the *Type*. """ endpoint = type_ + '/chart' diff --git a/lyricsgenius/api/public_methods/misc.py b/lyricsgenius/api/public_methods/misc.py index d72b6ffe..acd56d88 100644 --- a/lyricsgenius/api/public_methods/misc.py +++ b/lyricsgenius/api/public_methods/misc.py @@ -23,12 +23,94 @@ def line_item(self, line_item_id, text_format=None): params = {'text_format': text_format or self.response_format} return self._make_request(path=endpoint, params_=params, public_api=True) + def page_data(self, album=None, song=None, artist=None): + """Gets page data of an item. + + If you want the page data of a song, you must supply + song and artist. But if you want the page data of an album, + you only have to supply the album. + + Page data will return all possible values for the album/song and + the lyrics in HTML format if the item is a song! + Album page data will contian album info and tracks info as well. + + Args: + album (:obj:`str`, optional): Album path + (e.g. '/albums/Eminem/Music-to-be-murdered-by') + song (:obj:`str`, optional): Song path + (e.g. '/Sia-chandelier-lyrics') + artist (:obj:`str`, optional): Artist slug. (e.g. 'Andy-shauf') + + Returns: + :obj:`dict` + + Warning: + Some albums/songs either don't have page data or + their page data path can't be infered easily from + the artist slug and their API path. So make sure to + use this method with a try/except clause that catches + 404 errors. Check out the example below. + + + Examples: + Getting the lyrics of a song from its page data + + .. code:: python + + from lyricsgenius import Genius, PublicAPI + from bs4 import BeautifulSoup + from requests import HTTPError + + genius = Genius(token) + public = PublicAPI() + + # We need the PublicAPI to get artist's slug + artist = public.artist(1665) + artist_slug = artist['artist']['slug'] + + # The rest can be done using Genius + song = genius.song(4558484) + song_path = song['song']['path'] + + try: + page_data = genius.page_data(artist=artist_slug, song=song_path) + except HTTPError as e: + print("Couldn't find page data {}".format(e.status_code)) + page_data = None + + if page_data is not None: + lyrics_html = page_data['page_data']['lyrics_data']['body']['html'] + lyrics_text = BeautifulSoup(lyrics_html, 'html.parser').get_text() + + """ + assert any([album, song]), "You must pass either song or album." + if song: + assert all([song, artist]), "You must pass artist." + + if album: + endpoint = 'page_data/album' + page_type = 'albums' + item_path = album.replace('/albums/', '') + else: + endpoint = 'page_data/song' + page_type = 'songs' + + # item path becomes something like: Artist/Song + item_path = song[1:].replace(artist + '-', artist + '/').replace('-lyrics', '') + + page_path = '/{page_type}/{item_path}'.format(page_type=page_type, + item_path=item_path) + params = {'page_path': page_path} + + return self._make_request(endpoint, params_=params, public_api=True) + def voters(self, annotation_id=None, answer_id=None, article_id=None, comment_id=None): """Gets the voters of an item. + You must supply one of :obj:`annotation_id`, :obj:`answer_id`, :obj:`article_id` or :obj:`comment_id`. diff --git a/lyricsgenius/api/public_methods/question.py b/lyricsgenius/api/public_methods/question.py index f1ed0022..aab7d22f 100644 --- a/lyricsgenius/api/public_methods/question.py +++ b/lyricsgenius/api/public_methods/question.py @@ -9,6 +9,7 @@ def questions(self, state=None, text_format=None): """Gets the questions on an album or a song. + You must supply one of :obj:`album_id` or :obj:`song_id`. Args: diff --git a/lyricsgenius/api/public_methods/referent.py b/lyricsgenius/api/public_methods/referent.py index c2fd60ed..75bcfece 100644 --- a/lyricsgenius/api/public_methods/referent.py +++ b/lyricsgenius/api/public_methods/referent.py @@ -3,6 +3,7 @@ class ReferentMethods(object): def referent(self, referent_ids, text_format=None): """Gets data of one or more referents. + This method can get multiple referents in one call, thus increasing performance. diff --git a/lyricsgenius/api/public_methods/search.py b/lyricsgenius/api/public_methods/search.py index 3562d7bc..d772804c 100644 --- a/lyricsgenius/api/public_methods/search.py +++ b/lyricsgenius/api/public_methods/search.py @@ -186,6 +186,7 @@ def search_videos(self, search_term, per_page=None, page=None): def search_all(self, search_term, per_page=None, page=None): """Searches all types. + Including: albums, articles, lyrics, songs, users and videos. diff --git a/lyricsgenius/auth.py b/lyricsgenius/auth.py index f93ccf8f..165c610e 100644 --- a/lyricsgenius/auth.py +++ b/lyricsgenius/auth.py @@ -3,6 +3,7 @@ from .utils import parse_redirected_url from .api import Sender +from .errors import InvalidStateError class OAuth2(Sender): @@ -50,6 +51,7 @@ def __init__(self, client_id, redirect_uri, @property def url(self): """Returns the URL you redirect the user to. + You can use this property to get a URL that when opened on the user's device, shows Genius's authorization page where user clicks *Agree* to give your app access, and then Genius redirects user back to your @@ -67,21 +69,37 @@ def url(self): payload['state'] = self.state return OAuth2.auth_url + '?' + urlencode(payload) - def get_user_token(self, url, **kwargs): - """Gets a user token using the redirected URL. - This method will either get the value of the *token* - parameter in the redirected URL, or use the value of the - *code* parameter to request a token from Genius. + def get_user_token(self, code=None, url=None, state=None, **kwargs): + """Gets a user token using the url or the code parameter.. + + If you supply value for :obj:`code`, this method will use the value of the + :obj:`code` parameter to request a token from Genius. + + If you use the :method`client_only_app` and supplt the redirected URL, + it will already have the token. + You could pass the URL to this method or parse it yourself. + + If you provide a :obj:`state` the method will also compare + it to the initial state and will raise an exception if + they're not equal. Args: - url (:obj:`str`): 'code' parameter of redirected URL. + code (:obj:`str`): 'code' parameter of redirected URL. + url (:obj:`str`): Redirected URL (used in client-only apps) + state (:obj:`str`): state parameter of redirected URL (only + provide if you want to compare with initial :obj:`self.state`) **kwargs: keywords for the POST request. returns: :obj:`str`: User token. """ - if self.flow == 'code': - payload = {'code': parse_redirected_url(url, self.flow), + assert any([code, url]), "You must pass either `code` or `url`." + + if state is not None and self.state != state: + raise InvalidStateError('States do not match.') + + if code: + payload = {'code': code, 'client_id': self.client_id, 'client_secret': self.client_secret, 'redirect_uri': self.redirect_uri, @@ -89,16 +107,17 @@ def get_user_token(self, url, **kwargs): 'response_type': 'code'} url = OAuth2.token_url.replace('https://api.genius.com/', '') res = self._make_request(url, 'POST', data=payload, **kwargs) - return res['access_token'] - elif self.flow == 'token': - return parse_redirected_url(url, self.flow) + token = res['access_token'] + else: + token = parse_redirected_url(url, self.flow) + return token def prompt_user(self): """Prompts current user for authentication. Opens a web browser for you to log in with Genius. Prompts to paste the URL after logging in to parse the - *code* or *token* URL parameter. + *token* URL parameter. returns: :obj:`str`: User token. @@ -110,7 +129,13 @@ def prompt_user(self): webbrowser.open(url) redirected = input('Please paste redirect URL: ').strip() - return self.get_user_token(redirected) + if self.flow == 'token': + token = parse_redirected_url(redirected, self.flow) + else: + code = parse_redirected_url(redirected, self.flow) + token = self.get_user_token(code) + + return token @classmethod def client_only_app(cls, client_id, redirect_uri, scope=None, state=None): diff --git a/lyricsgenius/errors.py b/lyricsgenius/errors.py new file mode 100644 index 00000000..911a7002 --- /dev/null +++ b/lyricsgenius/errors.py @@ -0,0 +1,2 @@ +class InvalidStateError(Exception): + """Exception for non-matching states.""" diff --git a/lyricsgenius/genius.py b/lyricsgenius/genius.py index 6b8e707d..e9368f5a 100644 --- a/lyricsgenius/genius.py +++ b/lyricsgenius/genius.py @@ -13,7 +13,7 @@ from bs4 import BeautifulSoup from .api import API, PublicAPI -from .types import Album, Artist, Song +from .types import Album, Artist, Song, Track from .utils import clean_str, safe_unicode @@ -92,12 +92,14 @@ def __init__(self, access_token=None, self.excluded_terms = self.default_terms.copy() self.excluded_terms.extend(excluded_terms) - def lyrics(self, urlthing, remove_section_headers=False): + def lyrics(self, song_id=None, song_url=None, remove_section_headers=False): """Uses BeautifulSoup to scrape song info off of a Genius song URL + You must supply either `song_id` or song_url`. + Args: - urlthing (:obj:`str` | :obj:`int`): - Song ID or song URL. + song_id (:obj:`int`, optional): Song ID. + song_url (:obj:`str`, optional): Song URL. remove_section_headers (:obj:`bool`, optional): If `True`, removes [Chorus], [Bridge], etc. headers from lyrics. @@ -118,10 +120,12 @@ def lyrics(self, urlthing, remove_section_headers=False): :attr:`Genius.remove_section_headers` attribute. """ - if isinstance(urlthing, int): - path = self.song(urlthing)['song']['path'][1:] + msg = "You must supply either `song_id` or `song_url`." + assert any([song_id, song_url]), msg + if song_url: + path = song_url.replace("https://genius.com/", "") else: - path = urlthing.replace("https://genius.com/", "") + path = self.song(song_id)['song']['path'][1:] # Scrape the song lyrics from the HTML html = BeautifulSoup( @@ -165,7 +169,8 @@ def _result_is_lyrics(self, song): 'interview', 'skit', 'instrumental', and 'setlist'. """ - if song['lyrics_state'] != 'complete': + if (song['lyrics_state'] != 'complete' + or song.get('instrumental')): return False expression = r"".join(["({})|".format(term) for term in self.excluded_terms]) @@ -319,29 +324,36 @@ def search_album(self, name=None, artist="", album_id = album_info['id'] - songs = [] + tracks = [] next_page = 1 + + # It's unlikely for an album to have >=50 songs, + # but it's best to check while next_page: - tracks = self.album_tracks(album_id=album_id, - per_page=50, - page=next_page, - text_format=text_format) - for track in tracks['tracks']: + tracks_list = self.album_tracks( + album_id=album_id, + per_page=50, + page=next_page, + text_format=text_format + ) + for track in tracks_list['tracks']: song_info = track['song'] - if song_info['lyrics_state'] == 'complete': - song_lyrics = self.lyrics(song_info['url']) + if (song_info['lyrics_state'] == 'complete' + and not song_info.get('instrumental')): + song_lyrics = self.lyrics(song_url=song_info['url']) else: song_lyrics = "" - song = Song(self, song_info, song_lyrics) - songs.append(song) - next_page = tracks['next_page'] + track = Track(self, track, song_lyrics) + tracks.append(track) + + next_page = tracks_list['next_page'] if album_id is None and get_full_info is True: new_info = self.album(album_id, text_format=text_format)['album'] album_info.update(new_info) - return Album(self, album_info, songs) + return Album(self, album_info, tracks) def search_song(self, title=None, artist="", song_id=None, get_full_info=True): @@ -418,8 +430,10 @@ def search_song(self, title=None, artist="", song_id=None, if song_id is None and get_full_info is True: new_info = self.song(song_id)['song'] song_info.update(new_info) - if song_info['lyrics_state'] == 'complete': - lyrics = self.lyrics(song_info['url']) + + if (song_info['lyrics_state'] == 'complete' + and not song_info.get('instrumental')): + lyrics = self.lyrics(song_url=song_info['url']) else: lyrics = "" @@ -547,7 +561,7 @@ def find_artist_id(search_term): # Create the Song object from lyrics and metadata if song_info['lyrics_state'] == 'complete': - lyrics = self.lyrics(song_info['url']) + lyrics = self.lyrics(song_url=song_info['url']) else: lyrics = "" if get_full_info: @@ -673,7 +687,7 @@ def tag(self, name, page=None): while page: res = genius.tag('pop', page=page) for hit in res['hits']: - song_lyrics = genius.lyrics(hit['url']) + song_lyrics = genius.lyrics(song_url=hit['url']) lyrics.append(song_lyrics) page = res['next_page'] @@ -689,6 +703,7 @@ def tag(self, name, page=None): ul = soup.find('ul', class_='song_list') for li in ul.find_all('li'): url = li.a.attrs['href'] + # Genius uses \xa0 in the HTML to add spaces song = [x.replace('\xa0', ' ') for x in li.a.span.stripped_strings] title = song[0] @@ -709,6 +724,7 @@ def tag(self, name, page=None): res = {'hits': hits} page = page if page is not None else 1 + # Full pages contain 20 items res['next_page'] = page + 1 if len(hits) == 20 else None return res diff --git a/lyricsgenius/types/__init__.py b/lyricsgenius/types/__init__.py index 05dff346..752a3932 100644 --- a/lyricsgenius/types/__init__.py +++ b/lyricsgenius/types/__init__.py @@ -1,4 +1,4 @@ from .base import Stats -from .album import Album +from .album import Album, Track from .artist import Artist from .song import Song diff --git a/lyricsgenius/types/album.py b/lyricsgenius/types/album.py index 30cf23c4..e3ea3fdf 100644 --- a/lyricsgenius/types/album.py +++ b/lyricsgenius/types/album.py @@ -1,34 +1,19 @@ from ..utils import convert_to_datetime from .base import BaseEntity from .artist import Artist +from .song import Song class Album(BaseEntity): - """An album from the Genius.com database. - - Attributes: - _type (:obj:`str`) - api_path (:obj:`str`) - artist (:class:`Artist`) - cover_art_thumbnail_url (:obj:`str`) - cover_art_url (:obj:`str`) - full_title (:obj:`str`) - id (:obj:`int`) - name (:obj:`str`) - name_with_artist (:obj:`str`) - release_date_components (:class:`datetime`) - songs (:obj:`list`): - A list of :class:`Song` objects. - url (:obj:`str`) - """ - - def __init__(self, client, json_dict, songs): + """An album from the Genius.com database.""" + + def __init__(self, client, json_dict, tracks): body = json_dict super().__init__(body['id']) self._body = body self._client = client self.artist = Artist(client, body['artist']) - self.songs = songs + self.tracks = tracks self.release_date_components = convert_to_datetime( body['release_date_components'] ) @@ -44,7 +29,7 @@ def __init__(self, client, json_dict, songs): def to_dict(self): body = super().to_dict() - body['songs'] = [song.to_dict() for song in self.songs] + body['tracks'] = [track.to_dict() for track in self.tracks] return body def to_json(self, @@ -61,7 +46,7 @@ def to_json(self, def to_text(self, filename=None, sanitize=True): - data = ' '.join(song.lyrics for song in self.songs) + data = ' '.join(track.song.lyrics for track in self.tracks) return super().to_text(data=data, filename=filename, @@ -83,3 +68,62 @@ def save_lyrics(self, ensure_ascii=ensure_ascii, sanitize=sanitize, verbose=verbose) + + +class Track(BaseEntity): + """docstring for Track""" + + def __init__(self, client, json_dict, lyrics): + body = json_dict + super().__init__(body['song']['id']) + self._body = body + self.song = Song(client, body['song'], lyrics) + + self.number = body['number'] + + def to_dict(self): + body = super().to_dict() + body['song'] = self.song.to_dict() + return body + + def to_json(self, + filename=None, + sanitize=True, + ensure_ascii=True): + data = self.to_dict() + + return super().to_json(data=data, + filename=filename, + sanitize=sanitize, + ensure_ascii=ensure_ascii) + + def to_text(self, + filename=None, + sanitize=True): + data = self.song.lyrics + + return super().to_text(data=data, + filename=filename, + sanitize=sanitize) + + def save_lyrics(self, + filename=None, + extension='json', + overwrite=False, + ensure_ascii=True, + sanitize=True, + verbose=True): + if filename is None: + filename = 'Lyrics_{:02d}_{}'.format(self.number, self.song.title) + filename = filename.replace(' ', '') + + return super().save_lyrics(filename=filename, + extension=extension, + overwrite=overwrite, + ensure_ascii=ensure_ascii, + sanitize=sanitize, + verbose=verbose) + + def __repr__(self): + name = self.__class__.__name__ + return "{}(number, song)".format(name) diff --git a/lyricsgenius/types/artist.py b/lyricsgenius/types/artist.py index f3dce53c..0c4350da 100644 --- a/lyricsgenius/types/artist.py +++ b/lyricsgenius/types/artist.py @@ -9,22 +9,7 @@ class Artist(BaseEntity): - """An artist with songs from the Genius.com database. - - Attributes: - api_path (:obj:`str`) - header_image_url (:obj:`str`) - id (:obj:`int`) - image_url (:obj:`str`) - is_meme_verified (:obj:`bool`) - is_verified (:obj:`bool`) - name (:obj:`str`) - songs (:obj:`list`): - A list of :class:`Song` objects - or an empty list. - url (:obj:`str`) - - """ + """An artist with songs from the Genius.com database.""" def __init__(self, client, json_dict): # Artist Constructor diff --git a/lyricsgenius/types/song.py b/lyricsgenius/types/song.py index b7c8e8df..bb0e5a25 100644 --- a/lyricsgenius/types/song.py +++ b/lyricsgenius/types/song.py @@ -9,32 +9,7 @@ class Song(BaseEntity): - """A song from the Genius.com database. - - Attributes: - annotation_count (:obj:`int`) - api_path (:obj:`str`) - artist (:obj:`str`): - Primary artist's name - (Same as ``Song.primary_artist.name``) - full_title (:obj:`str`) - header_image_thumbnail_url (:obj:`str`) - header_image_url (:obj:`str`) - id (:obj:`int`) - lyrics (:obj:`str`) - lyrics_owner_id (:obj:`int`) - lyrics_state (:obj:`str`) - path (:obj:`str`) - primary_artist (:class:`Artist`) - pyongs_count (:obj:`int`) - song_art_image_thumbnail_url (:obj:`str`) - song_art_image_url (:obj:`str`) - stats (:class:`Stats`) - title (:obj:`str`) - title_with_featured (:obj:`str`) - url (:obj:`str`) - - """ + """A song from the Genius.com database.""" def __init__(self, client, json_dict, lyrics=""): body = json_dict diff --git a/lyricsgenius/utils.py b/lyricsgenius/utils.py index ba306326..a29995fe 100644 --- a/lyricsgenius/utils.py +++ b/lyricsgenius/utils.py @@ -72,7 +72,7 @@ def clean_str(s): """Cleans a string to help with string comparison. Removes punctuation and returns - a stripped, NFKD normalized string in lowercase. + a stripped, NFKC normalized string in lowercase. Args: s (:obj:`str`): A string. @@ -81,9 +81,9 @@ def clean_str(s): :obj:`str`: Cleaned string. """ - punctuation_ = punctuation + "’" + punctuation_ = punctuation + "’" + "\u200b" string = s.translate(str.maketrans('', '', punctuation_)).strip().lower() - return unicodedata.normalize("NFKD", string) + return unicodedata.normalize("NFKC", string) def parse_redirected_url(url, flow): diff --git a/requirements.txt b/requirements.txt deleted file mode 100644 index d915612c..00000000 --- a/requirements.txt +++ /dev/null @@ -1,2 +0,0 @@ -beautifulsoup4>=4.6.0 -requests>=2.20.0 diff --git a/setup.py b/setup.py index 7f9c012c..202928d3 100644 --- a/setup.py +++ b/setup.py @@ -26,7 +26,7 @@ extras_require = { 'docs': [ - 'sphinx~=3.2', + 'sphinx~=3.3', 'sphinx-rtd-theme', ], 'checks': [ diff --git a/tests/__init__.py b/tests/__init__.py index e69de29b..4567ee58 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -0,0 +1,12 @@ +import os + +from lyricsgenius import Genius + + +# Import client access token from environment variable +access_token = os.environ.get("GENIUS_ACCESS_TOKEN", None) +assert access_token is not None, ( + "Must declare environment variable: GENIUS_ACCESS_TOKEN") + +# Genius client +genius = Genius(access_token, sleep_time=1.0, timeout=15, retries=3) diff --git a/tests/test_album.py b/tests/test_album.py index e387e239..0e9c872e 100644 --- a/tests/test_album.py +++ b/tests/test_album.py @@ -1,10 +1,8 @@ import unittest import os +import warnings -try: - from .test_genius import genius -except ModuleNotFoundError: - from test_genius import genius +from . import genius from lyricsgenius.types import Album @@ -13,9 +11,10 @@ class TestAlbum(unittest.TestCase): @classmethod def setUpClass(cls): print("\n---------------------\nSetting up Album tests...\n") + warnings.simplefilter("ignore", ResourceWarning) cls.album_name = "The Party" cls.artist_name = "Andy Shauf" - cls.num_songs = 10 + cls.num_tracks = 10 cls.album = genius.search_album( cls.album_name, cls.artist_name @@ -30,8 +29,8 @@ def test_album_name(self): def test_album_artist(self): self.assertEqual(self.album.artist.name, self.artist_name) - def test_songs(self): - self.assertEqual(len(self.album.songs), self.num_songs) + def test_tracks(self): + self.assertEqual(len(self.album.tracks), self.num_tracks) def test_saving_json_file(self): print('\n') diff --git a/tests/test_api.py b/tests/test_api.py index 41170c9e..1b173d9f 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -1,9 +1,7 @@ import unittest -try: - from .test_genius import genius -except ModuleNotFoundError: - from test_genius import genius + +from . import genius class TestAPI(unittest.TestCase): diff --git a/tests/test_artist.py b/tests/test_artist.py index 82eafcba..42c852b3 100644 --- a/tests/test_artist.py +++ b/tests/test_artist.py @@ -1,10 +1,7 @@ import unittest import os -try: - from .test_genius import genius -except ModuleNotFoundError: - from test_genius import genius +from . import genius from lyricsgenius.types import Artist from lyricsgenius.utils import sanitize_filename @@ -14,6 +11,7 @@ class TestArtist(unittest.TestCase): @classmethod def setUpClass(cls): print("\n---------------------\nSetting up Artist tests...\n") + cls.artist_name = "The Beatles" cls.new_song = "Paperback Writer" cls.max_songs = 2 diff --git a/tests/test_auth.py b/tests/test_auth.py index fe359a67..9f710b40 100644 --- a/tests/test_auth.py +++ b/tests/test_auth.py @@ -2,7 +2,9 @@ import unittest from unittest.mock import MagicMock, patch + from lyricsgenius import OAuth2 +from lyricsgenius.errors import InvalidStateError client_id = os.environ["GENIUS_CLIENT_ID"] client_secret = os.environ["GENIUS_CLIENT_SECRET"] @@ -59,27 +61,70 @@ def test_init(self): client_secret, scope='all') self.assertEqual(auth.scope, scope) - def test_get_user_token_client_flow(self): - # client-only flow - auth = OAuth2(client_id, redirect_uri, client_only_app=True) - redirected = 'https://example.com/callback#access_token=test' - client_flow_token = 'test' - - r = auth.get_user_token(redirected) - self.assertEqual(r, client_flow_token) - @patch('requests.Session.request', side_effect=mocked_requests_post) def test_get_user_token_code_flow(self, mock_post): # full code exchange flow - auth = OAuth2(client_id, redirect_uri, - client_secret, scope='all') - redirected = 'https://example.com/callback?code=some_code' + + state = 'some_state' + code = 'some_code' code_flow_token = 'test' - r = auth.get_user_token(redirected) + auth = OAuth2.full_code_exchange( + client_id, + redirect_uri, + client_secret, + scope='all', + state=state + ) + + r = auth.get_user_token(code=code, state=state) self.assertEqual(r, code_flow_token) + def test_get_user_token_token_flow(self): + + state = 'some_state' + token_flow_token = 'test' + redirected_url = '{}#access_token=test'.format(redirect_uri) + + auth = OAuth2.client_only_app( + client_id, + redirect_uri, + scope='all', + state=state + ) + + r = auth.get_user_token(url=redirected_url) + self.assertEqual(r, token_flow_token) + + def test_get_user_token_invalid_state(self): + state = 'state_1' + auth = OAuth2.full_code_exchange( + client_id, + redirect_uri, + client_secret, + scope='all', + state=state + ) + + returned_code = 'some_code' + returned_state = 'state_2' + with self.assertRaises(InvalidStateError): + auth.get_user_token(code=returned_code, state=returned_state) + + def test_get_user_token_no_parameter(self): + state = 'some_state' + auth = OAuth2.full_code_exchange( + client_id, + redirect_uri, + client_secret, + scope='all', + state=state + ) + + with self.assertRaises(AssertionError): + auth.get_user_token() + def test_prompt_user(self): auth = OAuth2(client_id, redirect_uri, client_secret, scope='all') diff --git a/tests/test_base.py b/tests/test_base.py index 9ae7ff51..e20adef7 100644 --- a/tests/test_base.py +++ b/tests/test_base.py @@ -2,10 +2,7 @@ from requests.exceptions import HTTPError -try: - from .test_genius import genius -except ModuleNotFoundError: - from test_genius import genius +from . import genius class TestAPIBase(unittest.TestCase): diff --git a/tests/test_genius.py b/tests/test_genius.py index 717e4864..ce73396d 100644 --- a/tests/test_genius.py +++ b/tests/test_genius.py @@ -1,14 +1,6 @@ -import os import unittest -from lyricsgenius import Genius - - -# Import client access token from environment variable -access_token = os.environ.get("GENIUS_ACCESS_TOKEN", None) -assert access_token is not None, ( - "Must declare environment variable: GENIUS_ACCESS_TOKEN") -genius = Genius(access_token, sleep_time=1.0, timeout=15) +from . import genius class TestEndpoints(unittest.TestCase): @@ -16,6 +8,7 @@ class TestEndpoints(unittest.TestCase): @classmethod def setUpClass(cls): print("\n---------------------\nSetting up Endpoint tests...\n") + cls.search_term = "Ezra Furman" cls.song_title_only = "99 Problems" cls.tag = genius.tag('pop') @@ -86,6 +79,7 @@ class TestLyrics(unittest.TestCase): @classmethod def setUpClass(cls): print("\n---------------------\nSetting up lyrics tests...\n") + cls.song_url = "https://genius.com/Andy-shauf-begin-again-lyrics" cls.song_id = 2885745 cls.lyrics_ending = ( @@ -96,7 +90,7 @@ def setUpClass(cls): ) def test_lyrics_with_url(self): - lyrics = genius.lyrics(self.song_url) + lyrics = genius.lyrics(song_url=self.song_url) self.assertTrue(lyrics.endswith(self.lyrics_ending)) def test_lyrics_with_id(self): diff --git a/tests/test_public_methods.py b/tests/test_public_methods.py index 6a260b59..3b87b7e3 100644 --- a/tests/test_public_methods.py +++ b/tests/test_public_methods.py @@ -2,6 +2,9 @@ from lyricsgenius import PublicAPI +from . import genius + + client = PublicAPI() @@ -10,6 +13,7 @@ class TestAlbumMethods(unittest.TestCase): @classmethod def setUpClass(cls): print("\n---------------------\nSetting up album methods tests...\n") + cls.album_id = 104614 def test_album(self): @@ -48,6 +52,7 @@ class TestAnnotationMethods(unittest.TestCase): @classmethod def setUpClass(cls): print("\n---------------------\nSetting up annotation methods tests...\n") + cls.annotation_id = 10225840 def test_annotation(self): @@ -71,6 +76,7 @@ class TestArticleMethods(unittest.TestCase): @classmethod def setUpClass(cls): print("\n---------------------\nSetting up article methods tests...\n") + cls.article_id = 11880 def test_article(self): @@ -94,6 +100,7 @@ class TestArtistMethods(unittest.TestCase): @classmethod def setUpClass(cls): print("\n---------------------\nSetting up artist methods tests...\n") + cls.artist_id = 1665 def test_artist(self): @@ -134,6 +141,7 @@ class TestCoverArtMethods(unittest.TestCase): @classmethod def setUpClass(cls): print("\n---------------------\nSetting up cover arts methods tests...\n") + cls.album_id = 104614 def test_cover_arts(self): @@ -146,6 +154,7 @@ class TestDiscussionMethods(unittest.TestCase): @classmethod def setUpClass(cls): print("\n---------------------\nSetting up discussion methods tests...\n") + # cls.discussion_id = 123 # # def test_discussion(self): @@ -181,6 +190,7 @@ class TestQuestionMethods(unittest.TestCase): @classmethod def setUpClass(cls): print("\n---------------------\nSetting up question methods tests...\n") + cls.album_id = 104614 def test_questions(self): @@ -193,6 +203,7 @@ class TestReferentMethods(unittest.TestCase): @classmethod def setUpClass(cls): print("\n---------------------\nSetting up referent methods tests...\n") + cls.web_page_id = 10347 cls.referent_ids = [20793764, 20641014] @@ -211,6 +222,7 @@ class TestSearchMethods(unittest.TestCase): @classmethod def setUpClass(cls): print("\n---------------------\nSetting up search methods tests...\n") + cls.search_term = 'test' def test_search(self): @@ -255,6 +267,7 @@ class TestSongMethods(unittest.TestCase): @classmethod def setUpClass(cls): print("\n---------------------\nSetting up song methods tests...\n") + cls.song_id = 378195 def test_song(self): @@ -279,6 +292,7 @@ class TestUserMethods(unittest.TestCase): @classmethod def setUpClass(cls): print("\n---------------------\nSetting up user methods tests...\n") + cls.user_id = 1 def test_user(self): @@ -342,6 +356,7 @@ class TestVideoMethods(unittest.TestCase): @classmethod def setUpClass(cls): print("\n---------------------\nSetting up video methods tests...\n") + cls.video_id = 18681 def test_video(self): @@ -361,6 +376,7 @@ class TestMiscMethods(unittest.TestCase): @classmethod def setUpClass(cls): print("\n---------------------\nSetting up misc methods tests...\n") + # cls.line_item_id = 146262999 cls.annotation_id = 10225840 @@ -368,6 +384,22 @@ def setUpClass(cls): # r = client.line_item(self.line_item_id) # self.assertTrue("line_item" in r) + def test_page_data_album(self): + album_path = '/albums/Eminem/Music-to-be-murdered-by' + + page_data = genius.page_data(album=album_path) + self.assertTrue('page_data' in page_data) + + def test_page_data_song(self): + artist = client.artist(1665) + artist_slug = artist['artist']['slug'] + + song = genius.song(4558484) + song_path = song['song']['path'] + + page_data = genius.page_data(artist=artist_slug, song=song_path) + self.assertTrue('page_data' in page_data) + def test_voters(self): r = client.voters(annotation_id=self.annotation_id) self.assertTrue("voters" in r) diff --git a/tests/test_song.py b/tests/test_song.py index edbbe593..13845ede 100644 --- a/tests/test_song.py +++ b/tests/test_song.py @@ -1,10 +1,7 @@ import os import unittest -try: - from .test_genius import genius -except ModuleNotFoundError: - from test_genius import genius +from . import genius from lyricsgenius.types import Song from lyricsgenius.utils import clean_str @@ -14,6 +11,7 @@ class TestSong(unittest.TestCase): @classmethod def setUpClass(cls): print("\n---------------------\nSetting up Song tests...\n") + cls.artist_name = 'Andy Shauf' cls.song_title = 'begin again' # Lowercase is intentional cls.album = 'The Party' @@ -35,14 +33,6 @@ def test_artist(self): # The returned artist name does not match the artist of the requested song. self.assertEqual(self.song.artist, self.artist_name) - # def test_album(self): - # msg = "The returned album name does not match the album of the requested song." - # self.assertEqual(self.song.album, self.album, msg) - - # def test_year(self): - # msg = "The returned year does not match the year of the requested song" - # self.assertEqual(self.song.year, self.year, msg) - def test_lyrics_raw(self): lyrics = '[Verse 1: Andy Shauf]' self.assertTrue(self.song.lyrics.startswith(lyrics)) @@ -51,17 +41,9 @@ def test_lyrics_no_section_headers(self): lyrics = 'Begin again\nThis time you should take a bow at the' self.assertTrue(self.song_trimmed.lyrics.startswith(lyrics)) - # def test_media(self): - # msg = "The returned song does not have a media attribute." - # self.assertTrue(hasattr(self.song, 'media'), msg) - def test_result_is_lyrics(self): self.assertTrue(genius._result_is_lyrics(self.song.to_dict())) - # def test_producer_artists(self): - # # Producer artist should be 'Andy Shauf'. - # self.assertEqual(self.song.producer_artists[0]["name"], "Andy Shauf") - def test_saving_json_file(self): print('\n') extension = 'json' diff --git a/tests/test_utils.py b/tests/test_utils.py index 6f6eb1cd..e666143d 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -5,10 +5,8 @@ sanitize_filename, auth_from_environment ) -try: - from .test_genius import genius -except ModuleNotFoundError: - from test_genius import genius + +from . import genius class TestUtils(unittest.TestCase):