Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Development #470

Merged
merged 2 commits into from
Jan 4, 2024
Merged

Development #470

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
cb16218
feat (arc): better exception presentation
santanche Jan 4, 2024
60cf63e
feat (report/annotations): new JSON format
santanche Jan 4, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion src/adonisjs/public/editor/annotate/js/metrics.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -118,7 +118,7 @@ class AnnotationMetrics {
const ds = r - er // deviation score

const arc =
(max - er == 0) ? '0/0' : (r - er) / (max - er) // adjusted ratio of clustering
(max - er == 0) ? ('' + (r - er)) + ('/' + (max - er)) : (r - er) / (max - er) // adjusted ratio of clustering

console.log('\n\n=== Clustering Free Recall ===')
console.log(JSON.stringify(categoriesOrder))
Expand Down
1 change: 1 addition & 0 deletions src/adonisjs/public/report/annotations/index.html
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -105,6 +105,7 @@
subscribe="case/source/get"></dcc-rest>
<dcc-button label="Export Analysis" xstyle="in" topic="report/download"></dcc-button>
<dcc-button label="Export BILOU" xstyle="in" topic="report/bilou"></dcc-button>
<dcc-button label="Export JSON" xstyle="in" topic="report/json"></dcc-button>
</div>

</div>
Expand Down
267 changes: 163 additions & 104 deletions src/adonisjs/public/report/js/report-annotations.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ class ReportManager {
start () {
MessageBus.i.subscribe('report/download', this._downloadAnalysis.bind(this))
MessageBus.i.subscribe('report/bilou', this._downloadBILOU.bind(this))
MessageBus.i.subscribe('report/json', this._downloadJSON.bind(this))
this._roomId = new URL(document.location).searchParams.get('roomid')
}

Expand Down Expand Up @@ -78,6 +79,123 @@ class ReportManager {
return result
}

_download (table) {
const element = document.createElement('a')
element.setAttribute('href',
'data:text/plain;charset=utf-8,' + encodeURIComponent(table))
element.setAttribute('download', 'annotations.csv')
element.style.display = 'none'
document.body.appendChild(element)
element.click()
document.body.removeChild(element)
}

/*
* Export Analysis
*/

_calculateMetrics (annotations) {
let ctideas = 0, ctright = 0, ctinfright = 0
let ctwrong = 0, ctrightencap = 0, ctinfrightencap = 0, ctwrongencap = 0
const catIndex = {}
const catOrder = []

for (const an of annotations) {
ctideas++
let start = -1
for (const f of an.fragments) {
start = (start == -1) ? f.start : start
}
if (an.categories.includes('encapsulated'))
if (an.categories.includes('wrong'))
ctwrongencap++
else {
ctinfrightencap++
if (an.categories.includes('right'))
ctrightencap++
}
if (!an.categories.includes('wrong'))
ctinfright++
for (const c of an.categories) {
if (c == 'right') ctright++
else if (c == 'wrong') ctwrong++
if (ReportManager.catList.includes(c)) {
catIndex[c] = c
catOrder.push([ReportManager.catList.indexOf(c)+1, start])
}
}
}
const selfOrder = AnnotationMetrics.i._selfOrderCount(catOrder)

const clustering = AnnotationMetrics.i._clusteringFreeRecall(catOrder)

let o1csv = ''
let sep = ''
for (const g of selfOrder.groups) {
o1csv += sep + ReportManager.catList[g[0]-1] + ':' + g[2]
sep = '; '
}

let o2csv = ''
sep = ''
const ordered = {}
for (const g of selfOrder.ordered) {
ordered[ReportManager.catList[g[0]-1]] = g[2]
o2csv += sep + ReportManager.catList[g[0]-1] + ':' + g[2]
sep = '; '
}

let countCat = ''
for (const c of ReportManager.catList)
countCat += ',' + (ordered[c] ? ordered[c] : 0)

const ctcategories = Object.keys(catIndex).length

return `${ctcategories},${ctright},${ctinfright},${ctideas},${ctrightencap},${ctinfrightencap},${ctwrong},${ctwrongencap},` +
`${ctcategories * ctideas},${(ctideas == 0) ? 0 : ctright / ctideas},${(ctideas == 0) ? 0 : ctinfright / ctideas},` +
`${(ctideas == 0) ? 0 : (ctrightencap + ctwrongencap) / ctideas},${selfOrder.score},` +
`${(ctideas == 0) ? 0 : selfOrder.score / ctideas},${clustering}${countCat},"${o1csv}","${o2csv}"`
}

async _downloadAnalysis () {
const tprefix = document.querySelector('#tprefix').value

const cases = await this._requestCases()

if (cases != null) {
let table = '"user id","annotation id","organization level","global score","student year",' +
'"used categories","right","right (inferred)","total ideas","right encapsulated",' +
'"right encapsulated (inferred)","wrong","wrong encapsulated","coverage score",' +
'"accuracy score","accuracy score (inferred)","encapsulated score","self order score",' +
'"normalized self order score","clustering in free recall"'

for (const m in ReportManager.catList)
table += ',"' + ReportManager.catList[m] + '"'

table += ',"self order groups","self order ordered"\n'

for (const c of cases.message) {
// remove prefix from title
if (c.title.startsWith(tprefix))
c.title = c.title.replace(tprefix, '')

table += '"' + c.title + '","' + c.id + '",'

const ant = await this._loadAnnotations(c.id)
const metrics = this._calculateMetrics(ant.annotations)

table += `"${ant.organization}","${ant.score}","${ant.year}",` +
metrics + '\n'
}

this._download(table)
}
}

/*
* Export BILOU
*/

async _buildBILOU (caseId, annotations) {
const tt = await this._tokenize(caseId)
const tokens = tt.tokens
Expand Down Expand Up @@ -141,21 +259,17 @@ class ReportManager {
}
}

async _tokenize (caseId) {

const tokens = []

let result = {}

async _loadCaseText (caseId) {
const cs = await MessageBus.i.request('case/source/get',
{room_id: this._roomId, case_id: caseId})

let text = null
if (cs != null && cs.message != null) {
const compiled =
await Translator.instance.compileMarkdown(
cs.message.id, cs.message.source)

let text = ''
text = ''
for (const knot in compiled.knots) {
const mkHTML =
await Translator.instance.generateHTML(compiled.knots[knot])
Expand All @@ -165,7 +279,18 @@ class ReportManager {
text = text.replace(/<[^>]+>/gm,'')
.replace(/<\/[^>]+>/gm, '')
.replace(/^[\r\n][\r\n]?/, '')
}

return text
}

async _tokenize (caseId) {
const tokens = []
let result = {}

const text = await this._loadCaseText(caseId)

if (text != null) {
let c = 0
let tk = ''
let tks = -1
Expand Down Expand Up @@ -213,75 +338,11 @@ class ReportManager {
ranges.push([f.start, f.start + f.size - 1])
}

_calculateMetrics (annotations) {
let ctideas = 0, ctright = 0, ctinfright = 0
let ctwrong = 0, ctrightencap = 0, ctinfrightencap = 0, ctwrongencap = 0
const catIndex = {}
const catOrder = []

for (const an of annotations) {
ctideas++
let start = -1
for (const f of an.fragments) {
start = (start == -1) ? f.start : start
}
if (an.categories.includes('encapsulated'))
if (an.categories.includes('wrong'))
ctwrongencap++
else {
ctinfrightencap++
if (an.categories.includes('right'))
ctrightencap++
}
if (!an.categories.includes('wrong'))
ctinfright++
for (const c of an.categories) {
if (c == 'right') ctright++
else if (c == 'wrong') ctwrong++
if (ReportManager.catList.includes(c)) {
catIndex[c] = c
catOrder.push([ReportManager.catList.indexOf(c)+1, start])
}
}
}
const selfOrder = AnnotationMetrics.i._selfOrderCount(catOrder)

const clustering = AnnotationMetrics.i._clusteringFreeRecall(catOrder)

let o1csv = ''
let sep = ''
for (const g of selfOrder.groups) {
o1csv += sep + ReportManager.catList[g[0]-1] + ':' + g[2]
sep = '; '
}

let o2csv = ''
sep = ''
const ordered = {}
for (const g of selfOrder.ordered) {
ordered[ReportManager.catList[g[0]-1]] = g[2]
o2csv += sep + ReportManager.catList[g[0]-1] + ':' + g[2]
sep = '; '
}

let countCat = ''
for (const c of ReportManager.catList)
countCat += ',' + (ordered[c] ? ordered[c] : 0)

const ctcategories = Object.keys(catIndex).length

return `${ctcategories},${ctright},${ctinfright},${ctideas},${ctrightencap},${ctinfrightencap},${ctwrong},${ctwrongencap},` +
`${ctcategories * ctideas},${(ctideas == 0) ? 0 : ctright / ctideas},${(ctideas == 0) ? 0 : ctinfright / ctideas},` +
`${(ctideas == 0) ? 0 : (ctrightencap + ctwrongencap) / ctideas},${selfOrder.score},` +
`${(ctideas == 0) ? 0 : selfOrder.score / ctideas},${clustering}${countCat},"${o1csv}","${o2csv}"`
}

async _downloadBILOU () {
const tprefix = document.querySelector('#tprefix').value

const cases = await this._requestCases()

console.log(cases)
let table = ''

if (cases != null) {
Expand All @@ -295,51 +356,49 @@ class ReportManager {
}
}

async _downloadAnalysis () {
const tprefix = document.querySelector('#tprefix').value
/*
* Export JSON
*/

const cases = await this._requestCases()

if (cases != null) {
let table = '"user id","annotation id","organization level","global score","student year",' +
'"used categories","right","right (inferred)","total ideas","right encapsulated",' +
'"right encapsulated (inferred)","wrong","wrong encapsulated","coverage score",' +
'"accuracy score","accuracy score (inferred)","encapsulated score","self order score",' +
'"normalized self order score","clustering in free recall"'
async _buildJSON (caseId, annotations) {
const text = await this._loadCaseText(caseId)
const annComp = []

for (const m in ReportManager.catList)
table += ',"' + ReportManager.catList[m] + '"'
for (const an of annotations) {
for (let f = 0; f < an.fragments.length; f++) {
annComp.push([
an.fragments[f].fragment,
an.fragments[f].start,
an.fragments[f].start + an.fragments[f].size - 1,
(an.fragments.length == 1) ? 'U' : ((f == 0) ? 'B' : ((f+1 < an.fragments.length) ? 'I' : 'L')),
an.categories
])
}
}

table += ',"self order groups","self order ordered"\n'
return {
doc_id: caseId,
text: text,
annotations: annComp
}
}

for (const c of cases.message) {
// remove prefix from title
if (c.title.startsWith(tprefix))
c.title = c.title.replace(tprefix, '')
async _downloadJSON () {
const tprefix = document.querySelector('#tprefix').value

table += '"' + c.title + '","' + c.id + '",'
const cases = await this._requestCases()

let table = ''
if (cases != null) {
for (const c of cases.message) {
const ant = await this._loadAnnotations(c.id)
const metrics = this._calculateMetrics(ant.annotations)

table += `"${ant.organization}","${ant.score}","${ant.year}",` +
metrics + '\n'
const annJson = await this._buildJSON(c.id, ant.annotations)
table += JSON.stringify(annJson) + '\n'
}

this._download(table)
}
}

_download (table) {
const element = document.createElement('a')
element.setAttribute('href',
'data:text/plain;charset=utf-8,' + encodeURIComponent(table))
element.setAttribute('download', 'annotations.csv')
element.style.display = 'none'
document.body.appendChild(element)
element.click()
document.body.removeChild(element)
}
}

(function () {
Expand Down
Loading