Merge pull request #47 from edx/hammad/ENT-4156

ENT-4156 | Update taxonomy models to pull skill tag descriptions

CHANGELOG.rst

@@ -14,6 +14,12 @@ Change Log
 Unreleased
 --------------------
 
+[1.4.1] - 2021-02-19
+--------------------
+
+*  added description field in Skill model and update the refresh_course_skill command to save skill description.
+*  Pinning EMSI skills API version to 7.35
+
 [1.4.0] - 2021-02-17
 --------------------
 

taxonomy/__init__.py

@@ -15,6 +15,6 @@
 # 2. MINOR version when you add functionality in a backwards compatible manner, and
 # 3. PATCH version when you make backwards compatible bug fixes.
 # More details can be found at https://semver.org/
-__version__ = '1.4.0'
+__version__ = '1.4.1'
 
 default_app_config = 'taxonomy.apps.TaxonomyConfig'  # pylint: disable=invalid-name

taxonomy/emsi_client.py

@@ -115,7 +115,7 @@ class EMSISkillsApiClient(JwtEMSIApiClient):
     Object builds an API client to make calls to get the skills from course text data.
     """
 
-    API_BASE_URL = JwtEMSIApiClient.API_BASE_URL + '/skills'
+    API_BASE_URL = JwtEMSIApiClient.API_BASE_URL + '/skills/versions/7.35'
 
     def __init__(self):
         """
@@ -134,12 +134,11 @@ def get_course_skills(self, course_text_data):
         Returns:
             dict: A dictionary containing details of all the skills.
         """
+        data = {
+            'text': course_text_data
+        }
         try:
-            data = {
-                'text': course_text_data
-            }
-            response = self.client.versions.latest.extract.post(data)
-
+            response = self.client.extract.post(data)
             return self.traverse_data(response)
         except (SlumberBaseException, ConnectionError, Timeout) as error:
             LOGGER.exception(
@@ -153,6 +152,15 @@ def traverse_data(response):
         """
         Transform data to a more useful format.
         """
+        for skill_details in response['data']:
+            # append skill description in skill data extracted from "wikipediaExtract" tag
+            try:
+                desc = next(tag['value'] for tag in skill_details['skill']['tags'] if tag['key'] == 'wikipediaExtract')
+            except StopIteration:
+                LOGGER.warning('[TAXONOMY] "wikipediaExtract" key not found in skill: %s', skill_details['skill']['id'])
+                desc = ''
+            skill_details['skill']['description'] = desc
+
         return response
 
 

taxonomy/migrations/0009_skill_description.py

@@ -0,0 +1,18 @@
+# Generated by Django 2.2.18 on 2021-02-19 03:46
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('taxonomy', '0008_auto_20210216_0710'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='skill',
+            name='description',
+            field=models.TextField(blank=True, default='', help_text='A short description for the skill received from API.'),
+        ),
+    ]

taxonomy/models.py

@@ -35,6 +35,11 @@ class Skill(TimeStampedModel):
             'The name of the skill.'
         )
     )
+    description = models.TextField(
+        blank=True,
+        help_text='A short description for the skill received from API.',
+        default='',
+    )
     info_url = models.URLField(
         verbose_name=_('Skill Information URL'),
         blank=True,

taxonomy/utils.py

@@ -11,11 +11,11 @@
 LOGGER = logging.getLogger(__name__)
 
 
-def update_skills_data(course_key, confidence, skill_data):
+def update_skills_data(course_key, skill_external_id, confidence, skill_data):
     """
     Persist the skills data in the database.
     """
-    skill, __ = Skill.objects.update_or_create(**skill_data)
+    skill, __ = Skill.objects.update_or_create(external_id=skill_external_id, defaults=skill_data)
 
     if not is_course_skill_blacklisted(course_key, skill.id):
         CourseSkills.objects.update_or_create(
@@ -42,15 +42,16 @@ def process_skills_data(course, course_skills, should_commit_to_db):
         try:
             confidence = float(record['confidence'])
             skill = record['skill']
+            skill_external_id = skill['id']
             skill_data = {
-                'external_id': skill['id'],
                 'name': skill['name'],
                 'info_url': skill['infoUrl'],
                 'type_id': skill['type']['id'],
                 'type_name': skill['type']['name'],
+                'description': skill['description']
             }
             if should_commit_to_db:
-                update_skills_data(course['key'], confidence, skill_data)
+                update_skills_data(course['key'], skill_external_id, confidence, skill_data)
         except KeyError:
             LOGGER.error('[TAXONOMY] Missing keys in skills data for course_key: %s', course['key'])
             failures.add((course['uuid'], course['key']))

test_utils/factories.py

@@ -29,6 +29,7 @@ class Meta:
     info_url = factory.LazyAttribute(lambda x: FAKER.uri())
     type_id = factory.LazyAttribute(lambda x: FAKER.slug())
     type_name = factory.LazyAttribute(lambda x: FAKER.text(max_nb_chars=20))
+    description = factory.LazyAttribute(lambda x: FAKER.text(max_nb_chars=200))
 
 
 # pylint: disable=no-member