Skip to content

Commit

Permalink
Merge pull request #130 from CogStack/admin-metrics-view
Browse files Browse the repository at this point in the history
CU-2yfm6ve: Add a project(s) metrics page
  • Loading branch information
tomolopolis authored May 18, 2023
2 parents 88a71bd + 5402b68 commit d3810f1
Show file tree
Hide file tree
Showing 9 changed files with 690 additions and 21 deletions.
4 changes: 2 additions & 2 deletions webapp/api/api/admin.py
Original file line number Diff line number Diff line change
Expand Up @@ -138,7 +138,7 @@ def download(modeladmin, request, queryset):


def download_projects_with_text(projects: QuerySet):
all_projects = _retrieve_project_data(projects)
all_projects = retrieve_project_data(projects)

sio = StringIO()
json.dump(all_projects, sio)
Expand All @@ -151,7 +151,7 @@ def download_projects_with_text(projects: QuerySet):
return response


def _retrieve_project_data(projects: QuerySet) -> Dict[str, List]:
def retrieve_project_data(projects: QuerySet) -> Dict[str, List]:
"""
A function to convert a list of projects and:
- their associated documents,
Expand Down
314 changes: 314 additions & 0 deletions webapp/api/api/metrics.py

Large diffs are not rendered by default.

44 changes: 31 additions & 13 deletions webapp/api/api/views.py
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
import logging
import pickle
import traceback
from tempfile import NamedTemporaryFile

Expand All @@ -12,7 +13,8 @@


from .admin import download_projects_with_text, download_projects_without_text, \
import_concepts_from_cdb, upload_projects_export
import_concepts_from_cdb, upload_projects_export, retrieve_project_data
from .metrics import ProjectMetrics
from .permissions import *
from .serializers import *
from .solr_utils import collections_available, search_collection, ensure_concept_searchable
Expand All @@ -27,7 +29,7 @@
print(os.environ)
"""

log = logging.getLogger('trainer')
logger = logging.getLogger(__name__)

# Maps between IDs and objects
CDB_MAP = {}
Expand Down Expand Up @@ -283,8 +285,8 @@ def add_annotation(request):
icd_code = request.data.get('icd_code')
opcs_code = request.data.get('opcs_code')

log.debug("Annotation being added")
log.debug(str(request.data))
logger.debug("Annotation being added")
logger.debug(str(request.data))

# Get project and the right version of cat
user = request.user
Expand All @@ -307,7 +309,7 @@ def add_annotation(request):
cat=cat,
icd_code=icd_code,
opcs_code=opcs_code)
log.debug('Annotation added.')
logger.debug('Annotation added.')
return Response({'message': 'Annotation added successfully', 'id': id})


Expand Down Expand Up @@ -335,7 +337,7 @@ def add_concept(request):

if cui in cat.cdb.cui2names:
err_msg = f'Cannot add a concept "{name}" with cui:{cui}. CUI already linked to {cat.cdb.cui2names[cui]}'
log.error(err_msg)
logger.error(err_msg)
return Response({'err': err_msg}, 400)

spacy_doc = cat(document.text)
Expand Down Expand Up @@ -455,14 +457,14 @@ def get_create_entity(request):
@api_view(http_method_names=['POST'])
def create_dataset(request):
filename = f'{request.data["dataset_name"]}.csv'
log.debug(request.data['dataset'])
logger.debug(request.data['dataset'])
ds = Dataset()
ds.name = request.data['dataset_name']
ds.description = request.data.get('description', 'n/a')
with NamedTemporaryFile(mode='r+') as f:
pd.DataFrame(request.data['dataset']).to_csv(f, index=False)
ds.original_file.save(filename, f)
log.debug(f'Saved new dataset:{ds.original_file.path}')
logger.debug(f'Saved new dataset:{ds.original_file.path}')
id = ds.id
return Response({'dataset_id': id})

Expand Down Expand Up @@ -494,7 +496,7 @@ def update_meta_annotation(request):
annotation = AnnotatedEntity.objects.filter(project= project_id, entity=entity_id, document=document_id)[0]
annotation.correct = True
annotation.validated = True
log.debug(annotation)
logger.debug(annotation)

annotation.save()

Expand All @@ -503,7 +505,7 @@ def update_meta_annotation(request):

meta_annotation_list = MetaAnnotation.objects.filter(annotated_entity = annotation)

log.debug(meta_annotation_list)
logger.debug(meta_annotation_list)

if len(meta_annotation_list) > 0:
meta_annotation = meta_annotation_list[0]
Expand All @@ -517,7 +519,7 @@ def update_meta_annotation(request):
meta_annotation.meta_task = meta_task
meta_annotation.meta_task_value = meta_task_value

log.debug(meta_annotation)
logger.debug(meta_annotation)
meta_annotation.save()

return Response({'meta_annotation': 'added meta annotation'})
Expand Down Expand Up @@ -615,19 +617,35 @@ def cache_model(request, p_id):
return Response({'result': f'Model already loaded for project:{p_id}'})
else:
clear_cached_medcat(CAT_MAP, project)
log.info(f'Cleared cached model{p_id}')
logger.info(f'Cleared cached model{p_id}')
return Response({'result': f'Cleared cached model:{p_id}'})


@api_view(http_method_names=['POST'])
def upload_deployment(request):
deployment_upload = request.data
upload_projects_export(deployment_upload)
# log.info(f'Errors encountered during previous deployment upload\n{errs}')
# logger.info(f'Errors encountered during previous deployment upload\n{errs}')
return Response("successfully uploaded", 200)


@api_view(http_method_names=['GET'])
def model_loaded(_):
return Response({p.id: get_cached_medcat(CAT_MAP, p) is not None
for p in ProjectAnnotateEntities.objects.all()})


@api_view(http_method_names=['GET'])
def metrics(request):
p_ids = request.GET.get('projectIds').split(',')
projects = ProjectAnnotateEntities.objects.filter(id__in=p_ids)
# assume projects all use the same model for eval purposes.
cat = get_medcat(CDB_MAP=CDB_MAP, VOCAB_MAP=VOCAB_MAP,
CAT_MAP=CAT_MAP, project=projects[0])
project_data = retrieve_project_data(projects)
metrics = ProjectMetrics(project_data, cat)
report_output = metrics.generate_report()
return Response({'results': report_output})



1 change: 1 addition & 0 deletions webapp/api/core/urls.py
Original file line number Diff line number Diff line change
Expand Up @@ -62,5 +62,6 @@
path('api/model-loaded/', api.views.model_loaded),
path('api/cache-model/<int:p_id>/', api.views.cache_model),
path('api/upload-deployment/', api.views.upload_deployment),
path('api/metrics/', api.views.metrics),
re_path('^.*$', api.views.index, name='index'), # Match everything else to home
]
65 changes: 65 additions & 0 deletions webapp/frontend/src/components/anns/AnnoResult.vue
Original file line number Diff line number Diff line change
@@ -0,0 +1,65 @@
<template>
<tr class="result-container">
<td class="doc-name">{{result['document name']}}</td>
<td class="cui">{{result.cui}}</td>
<td class="source-value">{{result['source value']}}</td>
<td class="accu">{{result.acc}}</td>
<td class="text">
<v-runtime-template :template="text"></v-runtime-template>
</td>
</tr>
</template>

<script>
import VRuntimeTemplate from 'v-runtime-template'
export default {
name: 'AnnoResult.vue',
components: { VRuntimeTemplate },
props: {
result: Object,
type: {
type: String,
default: 'tp'
}
},
computed: {
text () {
if (!this.result || !this.result.text || !this.result['source value']) {
return ''
}
// default to tp
let highlightClass = 'highlight-task-0'
if (this.type === 'fp' || this.type === 'fn') {
highlightClass = 'highlight-task-1'
}
const srcVal = this.result['source value']
const resTxt = this.result.text
const regexp = RegExp(`${srcVal}`, 'sg')
const matches = [...resTxt.matchAll(regexp)]
let outText = '<span>'
for (let match of matches) {
if (outText === '<span>') {
outText += `${resTxt.slice(0, match.index)}`
} else {
outText += `${resTxt.slice(matches[matches.indexOf(match) - 1].index + srcVal.length, match.index)}`
}
outText += `<span class="${highlightClass}">${srcVal}</span>`
if (matches.length === 1 || match === matches[-1]) {
outText += `${resTxt.slice(match.index + srcVal.length)}</span>`
}
}
return outText
}
}
}
</script>

<style scoped lang="scss">
.result-container {
> td {
padding: 10px;
}
}
</style>
10 changes: 7 additions & 3 deletions webapp/frontend/src/router.js
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ import Router from 'vue-router'
import Home from './views/Home.vue'
import TrainAnnotations from './views/TrainAnnotations.vue'
import Demo from './views/Demo.vue'
import Metrics from './views/Metrics'

Vue.use(Router)

Expand All @@ -13,12 +14,15 @@ export default new Router({
{
path: '/train-annotations/:projectId/:docId?',
name: 'train-annotations',
// route level code-splitting
// this generates a separate chunk (about.[hash].js) for this route
// which is lazy-loaded when the route is visited.
component: TrainAnnotations,
props: true
},
{
path: '/metrics/',
name: 'metrics',
component: Metrics,
query: true
},
{
path: '/demo',
name: 'demo',
Expand Down
46 changes: 44 additions & 2 deletions webapp/frontend/src/views/Home.vue
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,12 @@
<th>Complete</th>
<th v-if="isAdmin">Concepts Imported</th>
<th v-if="isAdmin">Model Loaded</th>
<th v-if="isAdmin">
Metrics
<button class="btn btn-outline-primary load-metrics" @click="loadMetrics" v-if="selectedProjects.length > 0">
<font-awesome-icon icon="chevron-right"></font-awesome-icon>
</button>
</th>
<th v-if="isAdmin">Save Model</th>
</tr>
</thead>
Expand Down Expand Up @@ -56,6 +62,14 @@
</button>
</div>
</td>
<td @click.stop v-if="isAdmin">
<button class="btn"
:class="{'btn-primary': selectedProjects.indexOf(project) !== -1, 'btn-outline-primary': selectedProjects.indexOf(project) === -1}"
@click="selectProject(project)">
<!-- <font-awesome-icon icon="times" class="selected-project" v-if="selectedProjects.indexOf(project) !== -1"></font-awesome-icon>-->
<font-awesome-icon icon="fa-chart-pie"></font-awesome-icon>
</button>
</td>
<td @click.stop v-if="isAdmin">
<button class="btn btn-outline-primary" :disabled="saving" @click="saveModel(project.id)"><font-awesome-icon icon="save"></font-awesome-icon></button>
</td>
Expand Down Expand Up @@ -118,7 +132,8 @@ export default {
isAdmin: false,
cdbSearchIndexStatus: {},
cdbLoaded: {},
clearModelModal: false
clearModelModal: false,
selectedProjects: []
}
},
created () {
Expand Down Expand Up @@ -238,6 +253,21 @@ export default {
}, 5000)
})
},
selectProject (project) {
if (this.selectedProjects.indexOf(project) !== -1) {
this.selectedProjects.splice(this.selectedProjects.indexOf(project), 1)
} else {
this.selectedProjects.push(project)
}
},
loadMetrics () {
this.$router.push({
name: 'metrics',
query: {
projectIds: this.selectedProjects.map(p => p.id).join(',')
}
})
},
fetchSearchIndexStatus () {
const cdbIds = _.uniq(this.projects.map(p => p.cdb_search_filter[0]))
this.$http.get(`/api/concept-db-search-index-created/?cdbs=${cdbIds.join(',')}`).then(resp => {
Expand Down Expand Up @@ -335,7 +365,7 @@ h3 {
}
}
.clear-model-cache {
.clear-model-cache, {
font-size: 15px;
color: $task-color-2;
cursor: pointer;
Expand All @@ -344,4 +374,16 @@ h3 {
top: -5px;
}
.selected-project {
font-size: 15px;
color: green;
position: relative;
right: -40px;
top: -10px;
}
.load-metrics {
padding: 0 5px;
}
</style>
Loading

0 comments on commit d3810f1

Please sign in to comment.