From 3e69b0ed84d73b4ddb48a91fb14adef2a0c79e48 Mon Sep 17 00:00:00 2001 From: David Manthey Date: Mon, 26 Feb 2024 11:43:43 -0500 Subject: [PATCH] Add the ability to config from db as well as girder.conf --- .circleci/config.yml | 2 +- docs/USAGE.rst | 5 + ruff.toml | 63 ++++++ tox.ini | 10 +- wsi_deid/__init__.py | 115 ++++++++++- wsi_deid/config.py | 108 ++++++++-- wsi_deid/constants.py | 1 + wsi_deid/import_export.py | 49 ++--- wsi_deid/jobs.py | 16 +- wsi_deid/process.py | 30 +-- wsi_deid/rest.py | 83 ++++---- .../web_client/stylesheets/ConfigView.styl | 9 +- wsi_deid/web_client/stylesheets/ItemView.styl | 3 +- wsi_deid/web_client/templates/ConfigView.pug | 189 +++++++++++++++++- wsi_deid/web_client/views/ConfigView.js | 36 +++- wsi_deid/web_client/views/ItemView.js | 16 +- 16 files changed, 624 insertions(+), 111 deletions(-) create mode 100644 ruff.toml diff --git a/.circleci/config.yml b/.circleci/config.yml index 5babbb4f..33ac2307 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -93,7 +93,7 @@ jobs: steps: - checkout - tox: - env: flake8,lintclient + env: lint,lintclient test: docker: - image: girder/tox-and-node diff --git a/docs/USAGE.rst b/docs/USAGE.rst index ac586e6e..63677338 100644 --- a/docs/USAGE.rst +++ b/docs/USAGE.rst @@ -328,6 +328,11 @@ Checking on Running Jobs You can check on running background jobs by selecting your user name in the upper right and selecting "My Jobs". The Jobs page shows running and completed jobs. Selecting a running job will show its progress. +Alternate Import - Direct From Assetstore +----------------------------------------- + +Girder supports importing files directly from an assetstore. See the Girder documentation on how this is done. If images and folders are imported into the ``Unfiled`` folder, they are treated as if they were imported via the ``Import`` button from the import folder. Because the general import allows importing any files, you may have non-image files in the system. + Redaction ========= diff --git a/ruff.toml b/ruff.toml new file mode 100644 index 00000000..46400c07 --- /dev/null +++ b/ruff.toml @@ -0,0 +1,63 @@ +include = ["*.py", "*.pyi", "**/pyproject.toml", "*.ipynb"] +exclude = [ + "build", + "*/web_client/*", + "*/*egg*/*", +] +lint.ignore = [ + "B017", + "B026", + "B904", + "B905", + "D100", + "D101", + "D102", + "D103", + "D104", + "D105", + "D106", + "D107", + "D200", + "D203", + "D205", + "D212", + "D213", + "D400", + "D401", + "D404", + "D415", + "E741", + "C408", + "PT011", + "PT012", + "PT017", +] +line-length = 100 +lint.select = [ + "B", # bugbear + "C90", # mccabe + "D", # pydocstyle + "E", # pycodestyle errors + "F", # pyflakes + "Q", # flake8-quotes + # "I", # isort + "W", # pycodestyle warnings + "YTT", + "ASYNC", + "COM", + "C4", + "EM", + "EXE", + "ISC", + "G", + "PIE", + "PYI", + "PT", + "RSE", +] + +[lint.flake8-quotes] +inline-quotes = "single" + +[lint.mccabe] +max-complexity = 14 diff --git a/tox.ini b/tox.ini index 1b8315a1..1cea1d67 100644 --- a/tox.ini +++ b/tox.ini @@ -36,18 +36,20 @@ setenv = NPM_CONFIG_PROGRESS=false NPM_CONFIG_PREFER_OFFLINE=true -[testenv:flake8] +[testenv:lint] basepython = python3 skipsdist = true skip_install = true deps = - flake8<6 + flake8 flake8-bugbear flake8-docstrings flake8-isort flake8-quotes pep8-naming + ruff commands = + ruff wsi_deid flake8 {posargs} [testenv:lintclient] @@ -148,6 +150,10 @@ skip_install = true deps = autopep8 isort + unify + ruff commands = isort {posargs:.} autopep8 -ria wsi_deid tests + unify --in-place --recursive wsi_deid + ruff wsi_deid docs --fix diff --git a/wsi_deid/__init__.py b/wsi_deid/__init__.py index b3505dda..7e9dc893 100644 --- a/wsi_deid/__init__.py +++ b/wsi_deid/__init__.py @@ -1,4 +1,6 @@ +import json import os +import re import girder import PIL.Image @@ -14,6 +16,7 @@ from pkg_resources import DistributionNotFound, get_distribution from . import assetstore_import +from .config import configSchemas from .constants import PluginSettings from .import_export import SftpMode from .rest import WSIDeIDResource, addSystemEndpoints @@ -66,13 +69,119 @@ def validateRemoteSftpPort(doc): doc['value'] = None else: if not isinstance(value, int): - raise ValidationException('Remote SFTP Port must be an integer value') + msg = 'Remote SFTP Port must be an integer value' + raise ValidationException(msg) @setting_utilities.validator(PluginSettings.WSI_DEID_SFTP_MODE) def validateSettingSftpMode(doc): - if not doc['value'] in [mode.value for mode in SftpMode]: - raise ValidationException('SFTP Mode must be one of "local", "remote", or "both"', 'value') + if doc['value'] not in [mode.value for mode in SftpMode]: + msg = 'SFTP Mode must be one of "local", "remote", or "both"' + raise ValidationException(msg, 'value') + + +@setting_utilities.validator({ + PluginSettings.WSI_DEID_BASE + 'add_title_to_label', + PluginSettings.WSI_DEID_BASE + 'always_redact_label', + PluginSettings.WSI_DEID_BASE + 'redact_macro_square', + PluginSettings.WSI_DEID_BASE + 'show_import_button', + PluginSettings.WSI_DEID_BASE + 'show_export_button', + PluginSettings.WSI_DEID_BASE + 'show_next_item', + PluginSettings.WSI_DEID_BASE + 'show_next_folder', + PluginSettings.WSI_DEID_BASE + 'require_redact_category', + PluginSettings.WSI_DEID_BASE + 'require_reject_reason', + PluginSettings.WSI_DEID_BASE + 'edit_metadata', + PluginSettings.WSI_DEID_BASE + 'show_metadata_in_lists', + PluginSettings.WSI_DEID_BASE + 'reimport_if_moved', + PluginSettings.WSI_DEID_BASE + 'validate_image_id_field', +}) +def validateBoolean(doc): + if doc.get('value', None) is not None: + doc['value'] = str(doc['value']).lower() in {'true', 'on', 'yes'} + + +@setting_utilities.validator({ + PluginSettings.WSI_DEID_BASE + 'redact_macro_long_axis_percent', + PluginSettings.WSI_DEID_BASE + 'redact_macro_short_axis_percent', +}) +def validateDecimalPercent(doc): + if doc.get('value', None) is not None: + doc['value'] = float(doc['value']) + if doc['value'] < 0 or doc['value'] > 100: + msg = 'Percent must be between 0 and 100' + raise ValidationException(msg) + + +@setting_utilities.validator({ + PluginSettings.WSI_DEID_BASE + 'folder_name_field', +}) +def validateFolderNameField(doc): + if doc.get('value', None): + doc['value'] = str(doc['value']).strip() + if not doc['value']: + doc['value'] = None + + +@setting_utilities.validator({ + PluginSettings.WSI_DEID_BASE + 'image_name_field', +}) +def validateImageNameField(doc): + if doc.get('value', None) is not None: + doc['value'] = str(doc['value']).strip() + + +@setting_utilities.validator({ + PluginSettings.WSI_DEID_BASE + 'new_token_pattern', +}) +def validateNewTokenPattern(doc): + if doc.get('value', None) is not None: + doc['value'] = str(doc['value']).strip() + if doc['value'] and '@' not in doc['value'] and '#' not in doc['value']: + msg = 'The token pattern must contain at least one @ or # character for templating' + raise ValidationException(msg) + if not doc['value']: + doc['value'] = None + + +@setting_utilities.validator({ + PluginSettings.WSI_DEID_BASE + 'hide_metadata_keys', + PluginSettings.WSI_DEID_BASE + 'hide_metadata_keys_format_aperio', + PluginSettings.WSI_DEID_BASE + 'hide_metadata_keys_format_hamamatsu', + PluginSettings.WSI_DEID_BASE + 'hide_metadata_keys_format_philips', + PluginSettings.WSI_DEID_BASE + 'hide_metadata_keys_format_isyntax', + PluginSettings.WSI_DEID_BASE + 'import_text_association_columns', + PluginSettings.WSI_DEID_BASE + 'no_redact_control_keys', + PluginSettings.WSI_DEID_BASE + 'no_redact_control_keys_format_aperio', + PluginSettings.WSI_DEID_BASE + 'no_redact_control_keys_format_hamamatsu', + PluginSettings.WSI_DEID_BASE + 'no_redact_control_keys_format_philips', + PluginSettings.WSI_DEID_BASE + 'no_redact_control_keys_format_isyntax', + PluginSettings.WSI_DEID_BASE + 'phi_pii_types', + PluginSettings.WSI_DEID_BASE + 'reject_reasons', + PluginSettings.WSI_DEID_BASE + 'upload_metadata_add_to_images', + PluginSettings.WSI_DEID_BASE + 'upload_metadata_for_export_report', +}) +def validateJsonSchema(doc): + import jsonschema + + if doc.get('value', None): + schemakey = doc['key'] + if schemakey.startswith(PluginSettings.WSI_DEID_BASE): + schemakey = schemakey[len(PluginSettings.WSI_DEID_BASE):] + if schemakey not in configSchemas and '_key' in schemakey: + schemakey = schemakey.split('_keys')[0] + '_keys' + if isinstance(doc['value'], str): + doc['value'] = json.loads(doc['value']) + jsonschema.validate(instance=doc['value'], schema=configSchemas[schemakey]) + if '_keys' in schemakey: + for k, v in doc['value'].items(): + try: + re.compile(k) + re.compile(v) + except Exception: + msg = f'All keys and values if {doc["key"]} must be regular expressions' + raise ValidationException(msg) + else: + doc.get('value', None) class GirderPlugin(plugin.GirderPlugin): diff --git a/wsi_deid/config.py b/wsi_deid/config.py index bfe96ed6..22f21799 100644 --- a/wsi_deid/config.py +++ b/wsi_deid/config.py @@ -1,5 +1,7 @@ import girder.utility.config +from .constants import PluginSettings + CONFIG_SECTION = 'wsi_deid' NUMERIC_VALUES = ( r'^\s*[+-]?(\d+([.]\d*)?([eE][+-]?\d+)?|[.]\d+([eE][+-]?\d+)?)(\s*,\s*[+-]?' @@ -60,6 +62,7 @@ 'upload_metadata_for_export_report': [ 'ImageID', 'Proc_Seq', 'Proc_Type', 'Slide_ID', 'Spec_Site', 'TokenID', ], + 'upload_metadata_add_to_images': None, 'import_text_association_columns': [], 'folder_name_field': 'TokenID', 'image_name_field': 'ImageID', @@ -67,7 +70,7 @@ 'reject_reasons': [{ 'category': 'Cannot_Redact', 'text': 'Cannot redact PHI', - 'key': 'Cannot_Redact' + 'key': 'Cannot_Redact', }, { 'category': 'Slide_Quality', 'text': 'Slide Quality', @@ -79,16 +82,16 @@ {'key': 'Debris', 'text': 'Debris or dust'}, {'key': 'Air_Bubbles', 'text': 'Air bubbles'}, {'key': 'Pathologist_Markings', 'text': "Pathologist's Markings"}, - {'key': 'Other_Slide_Quality', 'text': 'Other'} - ] + {'key': 'Other_Slide_Quality', 'text': 'Other'}, + ], }, { 'category': 'Image_Quality', 'text': 'Image Quality', 'types': [ {'key': 'Out_Of_Focus', 'text': 'Out of focus'}, {'key': 'Low_Resolution', 'text': 'Low resolution'}, - {'key': 'Other_Image_Quality', 'text': 'Other'} - ] + {'key': 'Other_Image_Quality', 'text': 'Other'}, + ], }], 'phi_pii_types': [ { @@ -98,30 +101,111 @@ {'key': 'Patient_Name', 'text': 'Patient Name'}, {'key': 'Patient_DOB', 'text': 'Date of Birth '}, {'key': 'SSN', 'text': 'Social Security Number'}, - {'key': 'Other_Personal', 'text': 'Other Personal'} - ] + {'key': 'Other_Personal', 'text': 'Other Personal'}, + ], }, { 'category': 'Demographics', 'key': 'Demographics', - 'text': 'Demographics' + 'text': 'Demographics', }, { 'category': 'Facility_Physician', 'key': 'Facility_Physician', - 'text': 'Facility/Physician Information' + 'text': 'Facility/Physician Information', }, { 'category': 'Other_PHIPII', 'key': 'Other_PHIPII', - 'text': 'Other PHI/PII' - } - ] + 'text': 'Other PHI/PII', + }, + ], + 'reimport_if_moved': False, + 'new_token_pattern': '####@@####', +} + + +configSchemas = { + 'hide_metadata_keys': { + '$schema': 'http://json-schema.org/schema#', + 'patternProperties': { + '^.*$': + {'type': 'string'}, + }, + 'additionalProperties': False, + }, + 'import_text_association_columns': { + '$schema': 'http://json-schema.org/schema#', + 'type': 'array', + 'items': {'type': 'string'}}, + 'no_redact_control_keys': { + '$schema': 'http://json-schema.org/schema#', + 'patternProperties': { + '^.*$': + {'type': 'string'}, + }, + 'additionalProperties': False, + }, + 'phi_pii_types': { + '$schema': 'http://json-schema.org/schema#', + 'type': 'array', + 'items': {'type': 'object', 'properties': { + 'category': {'type': 'string'}, + 'text': {'type': 'string'}, + 'types': {'type': 'array', + 'items': {'type': 'object', 'properties': { + 'key': {'type': 'string'}, + 'text': {'type': 'string'}}, + 'required': ['key', 'text']}}, + 'key': {'type': 'string'}}, + 'anyOf': [ + {'required': ['category', 'text', 'key']}, + {'required': ['category', 'text', 'types']}, + ]}}, + 'reject_reasons': { + '$schema': 'http://json-schema.org/schema#', + 'type': 'array', + 'items': {'type': 'object', 'properties': { + 'category': {'type': 'string'}, + 'text': {'type': 'string'}, + 'types': {'type': 'array', + 'items': {'type': 'object', 'properties': { + 'key': {'type': 'string'}, + 'text': {'type': 'string'}}, + 'required': ['key', 'text']}}, + 'key': {'type': 'string'}}, + 'anyOf': [ + {'required': ['category', 'text', 'key']}, + {'required': ['category', 'text', 'types']}, + ]}}, + 'upload_metadata_add_to_images': { + '$schema': 'http://json-schema.org/schema#', + 'anyOf': [ + {'type': 'null'}, + {'type': 'array', 'items': {'type': 'string'}}]}, + 'upload_metadata_for_export_report': { + '$schema': 'http://json-schema.org/schema#', + 'anyOf': [ + {'type': 'null'}, + {'type': 'array', 'items': {'type': 'string'}}]}, } def getConfig(key=None, fallback=None): configDict = girder.utility.config.getConfig().get(CONFIG_SECTION) or {} + configDict = configDict.copy() + try: + from girder.models.setting import Setting + + for subkey in defaultConfig: + try: + val = Setting().get(PluginSettings.WSI_DEID_BASE + subkey) + if val is not None: + configDict[subkey] = val + except Exception: + pass + except Exception: + pass if key is None: config = defaultConfig.copy() config.update(configDict) diff --git a/wsi_deid/constants.py b/wsi_deid/constants.py index 9aeffa6d..9f264a83 100644 --- a/wsi_deid/constants.py +++ b/wsi_deid/constants.py @@ -24,6 +24,7 @@ class PluginSettings(object): WSI_DEID_SCHEMA_FOLDER = 'wsi_deid.schema_folder' WSI_DEID_DB_API_URL = 'wsi_deid.db_api_url' WSI_DEID_DB_API_KEY = 'wsi_deid.db_api_key' + WSI_DEID_BASE = 'wsi_deid.base_' class SftpMode(Enum): diff --git a/wsi_deid/import_export.py b/wsi_deid/import_export.py index 810145e8..0558caa7 100644 --- a/wsi_deid/import_export.py +++ b/wsi_deid/import_export.py @@ -432,7 +432,7 @@ def startOcrJobForUnfiled(itemIds, imageInfoDict, user, reportInfo): type='wsi_deid.associate_unfiled', user=user, asynchronous=True, - args=(itemIds, imageInfoDict, reportInfo) + args=(itemIds, imageInfoDict, reportInfo), ) Job().scheduleJob(unfiledJob) return unfiledJob['_id'] @@ -480,7 +480,8 @@ def ingestData(ctx, user=None, walkData=None): # noqa importPath = Setting().get(PluginSettings.WSI_DEID_IMPORT_PATH) importFolderId = Setting().get(PluginSettings.HUI_INGEST_FOLDER) if not importPath or not importFolderId: - raise Exception('Import path and/or folder not specified.') + msg = 'Import path and/or folder not specified.' + raise Exception(msg) importFolder = Folder().load(importFolderId, force=True, exc=True) ctx.update(message='Scanning import folder') if not walkData: @@ -617,7 +618,7 @@ def importReport(ctx, report, excelReport, user, importPath, reason=None): 'excel': 'ExcelFilePath', } dataList = [] - reportFields = config.getConfig('upload_metadata_for_export_report') + reportFields = config.getConfig('upload_metadata_for_export_report') or [] statusKey = 'SoftwareStatus' reasonKey = 'Status/FailureReason' anyErrors = False @@ -715,7 +716,8 @@ def exportItems(ctx, user=None, all=False): exportPath = Setting().get(PluginSettings.WSI_DEID_EXPORT_PATH) exportFolderId = Setting().get(PluginSettings.HUI_FINISHED_FOLDER) if not exportPath or not exportFolderId: - raise Exception('Export path and/or finished folder not specified.') + msg = 'Export path and/or finished folder not specified.' + raise Exception(msg) exportFolder = Folder().load(exportFolderId, force=True, exc=True) report = [] summary = {} @@ -729,7 +731,7 @@ def exportItems(ctx, user=None, all=False): type='wsi_deid.sftp_job', user=user, asynchronous=True, - args=(exportFolder, user, all) + args=(exportFolder, user, all), ) Job().scheduleJob(job=sftp_job) if export_enabled: @@ -805,7 +807,7 @@ def sftp_items(job): job, export_all, user, - sftp_report + sftp_report, ) if export_result == ExportResult.PREVIOUSLY_EXPORTED: previous_exported_count += 1 @@ -837,7 +839,7 @@ def sftp_items(job): Job.updateJob( job, log=f'Job failed with the following exception: {str(exc)}.', - status=JobStatus.ERROR + status=JobStatus.ERROR, ) finally: sftp_client.close() @@ -854,7 +856,8 @@ def get_sftp_client(): transport.connect(username=user, password=password) sftp_client = paramiko.SFTPClient.from_transport(transport) if sftp_client is None: - raise Exception('There was an error connecting to the remote server.') + msg = 'There was an error connecting to the remote server.' + raise Exception(msg) return sftp_client @@ -959,13 +962,12 @@ def sftp_one_item(filepath, file, destination, sftp_client, job, export_all, use reports.append({ 'item': item, 'status': 'finished', - 'time': new_export_record['time'] + 'time': new_export_record['time'], }) return ExportResult.EXPORTED_SUCCESSFULLY else: - raise Exception( - f'There was an error transferring file {file_name} to remote destination.' - ) + msg = f'There was an error transferring file {file_name} to remote destination.' + raise Exception(msg) def exportItemsNext(mode, ctx, byteCount, totalByteCount, filepath, file, @@ -1078,7 +1080,7 @@ def buildExportDataSet(report): 'different': 'FailedToExport', } curtime = datetime.datetime.utcnow() - exportFields = config.getConfig('upload_metadata_for_export_report') + exportFields = config.getConfig('upload_metadata_for_export_report') or [] statusReasonFields = [] rejectReasonRequired = config.getConfig('require_reject_reason') if rejectReasonRequired: @@ -1121,7 +1123,7 @@ def buildExportDataSet(report): data['Total_VendorMetadataFields_ModifiedOrCreated'] = len( info['redactList']['metadata']) data['Automatic_DEID_PHIPII_MetadataFieldsModifiedRedacted'] = ', '.join(sorted( - k.rsplit(';', 1)[-1] for k, v in info['redactList']['metadata'].items()) + k.rsplit(';', 1)[-1] for k, v in info['redactList']['metadata'].items()), ) or 'N/A' data['Addtl_UserIdentifiedPHIPII_BINARY'] = 'Yes' if ( info['details']['redactionCount']['images'] or @@ -1131,13 +1133,13 @@ def buildExportDataSet(report): data['Addtl_UserIdentifiedPHIPII_MetadataFields'] = ', '.join(sorted( k.rsplit(';', 1)[-1] for k, v in info['redactList']['metadata'].items() if v.get('reason'))) or 'N/A' - data['Addtl_UserIdentifiedPHIPII_Category_MetadataFields'] = ', '.join(sorted(set( + data['Addtl_UserIdentifiedPHIPII_Category_MetadataFields'] = ', '.join(sorted({ v['category'] for k, v in info['redactList']['metadata'].items() - if v.get('reason') and v.get('category')))) or 'N/A' + if v.get('reason') and v.get('category')})) or 'N/A' data['Addtl_UserIdentifiedPHIPII_DetailedType_MetadataFields'] = ', '.join(sorted( - set( + { v['reason'] for k, v in info['redactList']['metadata'].items() - if v.get('reason') and v.get('category') == 'Personal_Info'))) or 'N/A' + if v.get('reason') and v.get('category') == 'Personal_Info'})) or 'N/A' data['Total_VendorImageComponents'] = info[ 'details']['fieldCount']['images'] data['Total_UserIdentifiedPHIPII_ImageComponents'] = info[ @@ -1145,12 +1147,12 @@ def buildExportDataSet(report): data['UserIdentifiedPHIPII_ImageComponents'] = ', '.join(sorted( k for k, v in info['redactList']['images'].items() if v.get('reason'))) or 'N/A' - data['UserIdentifiedPHIPII_Category_ImageComponents'] = ', '.join(sorted(set( + data['UserIdentifiedPHIPII_Category_ImageComponents'] = ', '.join(sorted({ v['category'] for k, v in info['redactList']['images'].items() - if v.get('reason') and v.get('category')))) or 'N/A' - data['UserIdentifiedPHIPII_DetailedType_ImageComponents'] = ', '.join(sorted(set( + if v.get('reason') and v.get('category')})) or 'N/A' + data['UserIdentifiedPHIPII_DetailedType_ImageComponents'] = ', '.join(sorted({ v['reason'] for k, v in info['redactList']['images'].items() - if v.get('reason') and v.get('category') == 'Personal_Info'))) or 'N/A' + if v.get('reason') and v.get('category') == 'Personal_Info'})) or 'N/A' except KeyError: pass dataList.append(data) @@ -1243,7 +1245,8 @@ def saveToReports(path, mimetype=None, user=None, folderName=None): reportsFolderId = Setting().get(PluginSettings.HUI_REPORTS_FOLDER) reportsFolder = Folder().load(reportsFolderId, force=True, exc=False) if not reportsFolder: - raise Exception('Reports folder not specified.') + msg = 'Reports folder not specified.' + raise Exception(msg) if folderName: reportsFolder = Folder().createFolder( reportsFolder, folderName, creator=user, reuseExisting=True) diff --git a/wsi_deid/jobs.py b/wsi_deid/jobs.py index b855a9ae..795ce63f 100644 --- a/wsi_deid/jobs.py +++ b/wsi_deid/jobs.py @@ -18,7 +18,7 @@ def start_ocr_item_job(job): Job().updateJob( job, log='Expected a Girder item as an argument\n', - status=JobStatus.ERROR + status=JobStatus.ERROR, ) return item = job_args[0] @@ -30,9 +30,9 @@ def start_ocr_item_job(job): message = f'Attempting to find label text for file {item["name"]} resulted in {str(e)}.' status = JobStatus.ERROR if status == JobStatus.SUCCESS and len(label_barcode) > 0: - message = f'Found label barcode for file {item["name"]}: {label_barcode}.\n', + message = f'Found label barcode for file {item["name"]}: {label_barcode}.\n' if status == JobStatus.SUCCESS and len(label_text) > 0: - message = f'Found label text for file {item["name"]}: {label_text}.\n', + message = f'Found label text for file {item["name"]}: {label_text}.\n' else: message = f'Could not find label text for file {item["name"]}\n' Job().updateJob(job, log=message, status=status) @@ -68,14 +68,14 @@ def start_ocr_batch_job(job): Job().updateJob( job, log='Starting batch job to find label text on items.\n', - status=JobStatus.RUNNING + status=JobStatus.RUNNING, ) job_args = job.get('args', None) if job_args is None: Job().updateJob( job, log='Expected a list of girder items as an argument.\n', - status=JobStatus.ERROR + status=JobStatus.ERROR, ) return itemIds = job_args[0] @@ -208,14 +208,14 @@ def associate_unfiled_images(job): # noqa Job().updateJob( job, log='Starting job to associate unfiled images with upload data.\n', - status=JobStatus.RUNNING + status=JobStatus.RUNNING, ) job_args = job.get('args', None) if job_args is None or len(job_args) != 3: Job().updateJob( job, log='Expected a list of girder items and upload information as arguments.\n', - status=JobStatus.ERROR + status=JobStatus.ERROR, ) return itemIds, uploadInfo, reportInfo = job_args @@ -255,7 +255,7 @@ def associate_unfiled_images(job): # noqa if len(label_text) > 0: for key, value in uploadInfo.items(): # key is the TokenID from the import spreadsheet, and value is associated info - matchTextFields = config.getConfig('import_text_association_columns') + matchTextFields = config.getConfig('import_text_association_columns') or [] uploadFields = value.get('fields', {}) text_to_match = [ uploadFields[field] for field in matchTextFields if field in uploadFields] diff --git a/wsi_deid/process.py b/wsi_deid/process.py index 90b0db7c..125ca120 100644 --- a/wsi_deid/process.py +++ b/wsi_deid/process.py @@ -164,14 +164,14 @@ def get_standard_redactions_format_aperio(item, tileSource, tiffinfo, title): 'internal;openslide;aperio.Filename': title_redaction_list_entry, 'internal;openslide;aperio.Title': title_redaction_list_entry, 'internal;openslide;tiff.Software': generate_system_redaction_list_entry( - get_deid_field(item, metadata.get('openslide', {}).get('tiff.Software')) + get_deid_field(item, metadata.get('openslide', {}).get('tiff.Software')), ), }, } if metadata['openslide'].get('aperio.Date'): redactList['metadata']['internal;openslide;aperio.Date'] = ( generate_system_redaction_list_entry( - '01/01/' + metadata['openslide']['aperio.Date'][6:] + '01/01/' + metadata['openslide']['aperio.Date'][6:], ) ) return redactList @@ -184,8 +184,8 @@ def get_standard_redactions_format_hamamatsu(item, tileSource, tiffinfo, title): 'metadata': { 'internal;openslide;hamamatsu.Reference': generate_system_redaction_list_entry(title), 'internal;openslide;tiff.Software': generate_system_redaction_list_entry( - get_deid_field(item, metadata.get('openslide', {}).get('tiff.Software')) - ) + get_deid_field(item, metadata.get('openslide', {}).get('tiff.Software')), + ), }, } for key in {'Created', 'Updated'}: @@ -204,7 +204,7 @@ def get_standard_redactions_format_philips(item, tileSource, tiffinfo, title): 'internal;xml;PIM_DP_UFS_BARCODE': generate_system_redaction_list_entry( title + '|' + get_deid_field(item)), 'internal;tiff;software': generate_system_redaction_list_entry( - get_deid_field(item, metadata.get('tiff', {}).get('software')) + get_deid_field(item, metadata.get('tiff', {}).get('software')), ), }, } @@ -243,7 +243,7 @@ def get_standard_redactions_format_isyntax(item, tileSource, tiffinfo, title): title + '|' + get_deid_field(item)), 'internal;isyntax;software_versions': generate_system_redaction_list_entry(( tileSource.getInternalMetadata()['isyntax'].get('software_versions', '') + - ' "DSA Redaction %s' % __version__ + '"').strip()) + ' "DSA Redaction %s' % __version__ + '"').strip()), }, } for key in {'acquisition_datetime', 'date_of_last_calibration'}: @@ -418,7 +418,8 @@ def redact_item(item, tempdir): fadvise_willneed(item) func = globals().get('redact_format_' + format) if func is None: - raise Exception('Cannot redact this format.') + msg = 'Cannot redact this format.' + raise Exception(msg) file, mimetype = func(item, tempdir, redactList, newTitle, labelImage, macroImage) info = { 'format': format, @@ -493,7 +494,7 @@ def redact_tiff_tags(ifds, redactList, title): tag = tifftools.Tag[tiffkey].value redactedTags.setdefault(tiffdir, {}) redactedTags[tiffdir][tag] = value['value'] - for titleKey in {'DocumentName', 'NDPI_REFERENCE', }: + for titleKey in {'DocumentName', 'NDPI_REFERENCE'}: redactedTags[tifftools.Tag[titleKey].value] = title for idx, ifd in enumerate(ifds): # convert to a list since we may mutage the tag dictionary @@ -643,7 +644,8 @@ def redact_format_aperio(item, tempdir, redactList, title, labelImage, macroImag associatedImages = tileSource.getAssociatedImagesList() if mainImageDir != [d + (1 if d and 'thumbnail' in associatedImages else 0) for d in range(len(mainImageDir))]: - raise Exception('Aperio TIFF directories are not in the expected order.') + msg = 'Aperio TIFF directories are not in the expected order.' + raise Exception(msg) firstAssociatedIdx = max(mainImageDir) + 1 # Set new image description ifds[0]['tags'][tifftools.Tag.ImageDescription.value] = { @@ -709,7 +711,7 @@ def redact_format_aperio_add_image(key, image, ifds, firstAssociatedIdx, tempdir key, image.width, image.height) imageinfo['ifds'][0]['tags'][tifftools.Tag.ImageDescription.value] = { 'datatype': tifftools.Datatype.ASCII, - 'data': imageDescription + 'data': imageDescription, } imageinfo['ifds'][0]['tags'][tifftools.Tag.NewSubfileType] = { 'data': [9 if key == 'macro' else 1], 'datatype': tifftools.Datatype.LONG} @@ -1109,7 +1111,7 @@ def redact_format_hamamatsu_replace_macro(macroImage, ifds, tempdir): 'UFS_IMAGE_DIMENSION_TYPE': ('0x301D', '0x2005', 'IString'), 'UFS_IMAGE_DIMENSION_UNIT': ('0x301D', '0x2006', 'IString'), 'UFS_IMAGE_GENERAL_HEADERS': ('0x301D', '0x2000', 'IDataObjectArray'), - 'UFS_IMAGE_NUMBER_OF_BLOCKS': ('0x301D', '0x2001', 'IUInt32') + 'UFS_IMAGE_NUMBER_OF_BLOCKS': ('0x301D', '0x2001', 'IUInt32'), } @@ -1235,7 +1237,7 @@ def redact_format_philips(item, tempdir, redactList, title, labelImage, macroIma labelinfo = tifftools.read_tiff(labelPath) labelinfo['ifds'][0]['tags'][tifftools.Tag.ImageDescription.value] = { 'datatype': tifftools.Datatype.ASCII, - 'data': 'Label' + 'data': 'Label', } labelinfo['ifds'][0]['tags'][tifftools.Tag.NewSubfileType] = { 'data': [1], 'datatype': tifftools.Datatype.LONG} @@ -1538,12 +1540,12 @@ def add_title_to_image(image, title, previouslyAdded=False, minWidth=384, try: imageDrawFont = PIL.ImageFont.truetype( font='/usr/share/fonts/truetype/dejavu/DejaVuSansMono.ttf', - size=int(fontSize * targetW) + size=int(fontSize * targetW), ) except IOError: try: imageDrawFont = PIL.ImageFont.truetype( - size=int(fontSize * targetW) + size=int(fontSize * targetW), ) except IOError: imageDrawFont = PIL.ImageFont.load_default() diff --git a/wsi_deid/rest.py b/wsi_deid/rest.py index d4bb5695..cb16a06e 100644 --- a/wsi_deid/rest.py +++ b/wsi_deid/rest.py @@ -78,12 +78,15 @@ def move_item(item, user, settingkey, options=None): """ folderId = Setting().get(settingkey) if not folderId: - raise RestException('The appropriate folder is not configured.') + msg = 'The appropriate folder is not configured.' + raise RestException(msg) folder = Folder().load(folderId, force=True) if not folder: - raise RestException('The appropriate folder does not exist.') + msg = 'The appropriate folder does not exist.' + raise RestException(msg) if str(folder['_id']) == str(item['folderId']): - raise RestException('The item is already in the appropriate folder.') + msg = 'The item is already in the appropriate folder.' + raise RestException(msg) folder, origFolders = create_folder_hierarchy(item, user, folder) if settingkey == PluginSettings.HUI_QUARANTINE_FOLDER: quarantineInfo = { @@ -92,18 +95,19 @@ def move_item(item, user, settingkey, options=None): 'originalBaseParentId': item['baseParentId'], 'originalUpdated': item['updated'], 'quarantineUserId': user['_id'], - 'quarantineTime': datetime.datetime.utcnow() + 'quarantineTime': datetime.datetime.utcnow(), } rejectInfo = None if settingkey == PluginSettings.HUI_REJECTED_FOLDER and options is not None: rejectReason = options.get('rejectReason', None) if rejectReason: rejectInfo = { - 'rejectReason': rejectReason + 'rejectReason': rejectReason, } requireRejectReason = config.getConfig('require_reject_reason') if requireRejectReason and rejectInfo is None: - raise RestException('A rejection reason is required.') + msg = 'A rejection reason is required.' + raise RestException(msg) # move the item item = Item().move(item, folder) if settingkey == PluginSettings.HUI_QUARANTINE_FOLDER: @@ -143,11 +147,13 @@ def process_item(item, user=None): origFolderId = Setting().get(PluginSettings.HUI_ORIGINAL_FOLDER) procFolderId = Setting().get(PluginSettings.HUI_PROCESSED_FOLDER) if not origFolderId or not procFolderId: - raise RestException('The appropriate folder is not configured.') + msg = 'The appropriate folder is not configured.' + raise RestException(msg) origFolder = Folder().load(origFolderId, force=True) procFolder = Folder().load(procFolderId, force=True) if not origFolder or not procFolder: - raise RestException('The appropriate folder does not exist.') + msg = 'The appropriate folder does not exist.' + raise RestException(msg) creator = User().load(item['creatorId'], force=True) # Generate the redacted file first, so if it fails we don't do anything # else @@ -222,7 +228,7 @@ def ocr_item(item, user): type='wsi_deid.ocr_job', user=user, asynchronous=True, - args=(item,) + args=(item,), ) Job().scheduleJob(job=ocr_job) return { @@ -323,7 +329,7 @@ def __init__(self, apiRoot): @autoDescribeRoute( Description('Check if a folder is a project folder.') .modelParam('id', model=Folder, level=AccessType.READ) - .errorResponse() + .errorResponse(), ) @access.public(scope=TokenScope.DATA_READ) def isProjectFolder(self, folder): @@ -372,7 +378,7 @@ def _actionForItem(self, item, user, action, options=None): .jsonParam('options', 'Additional information pertaining to the action.', required=False, paramType='body') .errorResponse() - .errorResponse('Write access was denied on the item.', 403) + .errorResponse('Write access was denied on the item.', 403), ) @access.user def itemAction(self, item, action, options): @@ -394,7 +400,7 @@ def itemAction(self, item, action, options): .modelParam('id', model=Item, level=AccessType.READ) .jsonParam('redactList', 'A JSON object containing the redactList to set', paramType='body', requireObject=True) - .errorResponse() + .errorResponse(), ) @access.user def setRedactList(self, item, redactList): @@ -402,7 +408,7 @@ def setRedactList(self, item, redactList): @autoDescribeRoute( Description('Ingest data from the import folder asynchronously.') - .errorResponse() + .errorResponse(), ) @access.user def ingest(self): @@ -412,7 +418,7 @@ def ingest(self): @autoDescribeRoute( Description('Export recently finished items to the export folder asynchronously.') - .errorResponse() + .errorResponse(), ) @access.user def export(self): @@ -422,7 +428,7 @@ def export(self): @autoDescribeRoute( Description('Export all finished items to the export folder asynchronously.') - .errorResponse() + .errorResponse(), ) @access.user def exportAll(self): @@ -435,14 +441,14 @@ def exportAll(self): @autoDescribeRoute( Description('Run OCR to find label text on items in the import folder without OCR metadata') - .errorResponse() + .errorResponse(), ) @access.user def ocrReadyToProcess(self): user = self.getCurrentUser() itemIds = [] ingestFolder = Folder().load(Setting().get( - PluginSettings.HUI_INGEST_FOLDER), user=user, level=AccessType.WRITE + PluginSettings.HUI_INGEST_FOLDER), user=user, level=AccessType.WRITE, ) resp = {'action': 'ocrall'} for _, file in Folder().fileList(ingestFolder, user, data=False): @@ -468,7 +474,7 @@ def ocrReadyToProcess(self): @autoDescribeRoute( Description('Get the ID of the next unprocessed item.') - .errorResponse() + .errorResponse(), ) @access.user def nextUnprocessedItem(self): @@ -494,7 +500,7 @@ def nextUnprocessedItem(self): Description( 'Get the IDs of the next two folders with unprocessed items and ' 'the id of the finished folder.') - .errorResponse() + .errorResponse(), ) @access.user def nextUnprocessedFolders(self): @@ -522,7 +528,7 @@ def nextUnprocessedFolders(self): @autoDescribeRoute( Description('Get settings that affect the UI.') - .errorResponse() + .errorResponse(), ) @access.public(scope=TokenScope.DATA_READ) def getSettings(self): @@ -537,7 +543,7 @@ def getSettings(self): .param('id', 'The ID of the resource.', paramType='path') .param('type', 'The type of the resource (folder, user, collection).') .errorResponse('ID was invalid.') - .errorResponse('Read access was denied for the resource.', 403) + .errorResponse('Read access was denied for the resource.', 403), ) def getSubtreeCount(self, id, type): user = self.getCurrentUser() @@ -609,7 +615,7 @@ def _allKeys(self, allkeys, entry, parent=None): dataType='boolean', default=False, required=False) .pagingParams(defaultSort='lowerName') .errorResponse() - .errorResponse('Read access was denied on the parent folder.', 403) + .errorResponse('Read access was denied on the parent folder.', 403), ) @access.public(scope=TokenScope.DATA_READ) def folderItemList(self, folder, images, limit, offset, sort, recurse): @@ -667,7 +673,7 @@ def folderItemList(self, folder, images, limit, offset, sort, recurse): 'reject, quarantine, unquarantine, finish, ocr.', paramType='path', enum=['process', 'reject', 'quarantine', 'unquarantine', 'finish', 'ocr']) .errorResponse() - .errorResponse('Write access was denied on the item.', 403) + .errorResponse('Write access was denied on the item.', 403), ) @access.user def itemListAction(self, ids, action): @@ -687,7 +693,7 @@ def _itemListAction(self, action, items, user): with ProgressContext( True, user=user, title='%s items' % pp.capitalize(), message='%s %s' % (pp.capitalize(), items[0]['name']), - total=len(items), current=0 + total=len(items), current=0, ) as ctx: try: for idx, item in enumerate(items): @@ -720,7 +726,7 @@ def _itemListAction(self, action, items, user): required=False, dataType='int') .param('recurse', 'Return items recursively under this folder.', dataType='boolean', default=False, required=False) - .errorResponse() + .errorResponse(), ) @access.user def folderAction(self, folder, action, limit=None, recurse=False): @@ -739,7 +745,7 @@ def folderAction(self, folder, action, limit=None, recurse=False): @autoDescribeRoute( Description('Get the list of known and allowed image names for refiling.') .modelParam('id', model=Item, level=AccessType.READ) - .errorResponse() + .errorResponse(), ) @access.user def getRefileList(self, item): @@ -758,7 +764,7 @@ def getRefileList(self, item): @autoDescribeRoute( Description('Get the list of known and allowed image names for refiling.') .modelParam('id', model=Folder, level=AccessType.READ) - .errorResponse() + .errorResponse(), ) @access.user def getRefileListFolder(self, folder): @@ -777,7 +783,7 @@ def getRefileListFolder(self, folder): @autoDescribeRoute( Description('Get the current import schema') - .errorResponse() + .errorResponse(), ) @access.admin def getSchema(self): @@ -791,7 +797,7 @@ def getSchema(self): .param('imageId', 'The new imageId') .param('tokenId', 'The new tokenId', required=False) .errorResponse() - .errorResponse('Write access was denied on the item.', 403) + .errorResponse('Write access was denied on the item.', 403), ) @access.user def refileItem(self, item, imageId, tokenId): @@ -800,7 +806,8 @@ def refileItem(self, item, imageId, tokenId): user = self.getCurrentUser() if imageId and imageId != item['name'].split('.', 1)[0] and Item().findOne({ 'name': {'$regex': '^' + re.escape(imageId) + r'\..*'}}): - raise RestException('An image with that name already exists.') + msg = 'An image with that name already exists.' + raise RestException(msg) if not imageId: imageId = TokenOnlyPrefix + tokenId uploadInfo = item.get('wsi_uploadInfo') @@ -820,7 +827,7 @@ def refileItem(self, item, imageId, tokenId): .modelParam('id', model=Item, level=AccessType.READ) .param('tokenId', 'The new tokenId', required=True) .jsonParam('refileData', 'Data used instead of internal data', paramType='body') - .errorResponse('Write access was denied on the item.', 403) + .errorResponse('Write access was denied on the item.', 403), ) @access.user def refileItemFull(self, item, tokenId, refileData): @@ -838,7 +845,7 @@ def refileItemFull(self, item, tokenId, refileData): .param('limit', 'Maximum number of items in folder to process', required=False, dataType='int') .param('recurse', 'Return items recursively under this folder.', - dataType='boolean', default=False, required=False) + dataType='boolean', default=False, required=False), ) @access.user def refileFolderFull(self, folder, tokenId, limit=None, recurse=False): @@ -864,7 +871,7 @@ def refileFolderFull(self, folder, tokenId, limit=None, recurse=False): Description('Refile multiple images at once.') .jsonParam('imageRefileData', 'Data used to refile images', paramType='body') .errorResponse() - .errorResponse('Write access was denied for an item.', 403) + .errorResponse('Write access was denied for an item.', 403), ) @access.user def refileItems(self, imageRefileData): @@ -879,7 +886,8 @@ def refileItems(self, imageRefileData): imageId = imageRefileData[itemId]['imageId'] if imageId and imageId != item['name'].split('.', 1)[0] and Item().findOne({ 'name': {'$regex': '^' + re.escape(imageId) + r'\..*'}}): - raise RestException('An image with that name already exists.') + msg = 'An image with that name already exists.' + raise RestException(msg) if not imageId: imageId = TokenOnlyPrefix + tokenId uploadInfo = item.get('wsi_uploadInfo') @@ -895,7 +903,7 @@ def refileItems(self, imageRefileData): @autoDescribeRoute( Description('Pass a set of values to the Matching API.') - .jsonParam('match', 'JSON match data', paramType='body') + .jsonParam('match', 'JSON match data', paramType='body'), ) @access.public def callMatchingAPI(self, match): @@ -923,7 +931,7 @@ def callMatchingAPI(self, match): @autoDescribeRoute( # noqa Description('Simulate the SEER*DMS Matching API for testing.') - .jsonParam('match', 'JSON match data', paramType='body') + .jsonParam('match', 'JSON match data', paramType='body'), ) @access.public def fakeMatchingAPI(self, match): # noqa @@ -984,7 +992,7 @@ def addSystemEndpoints(apiRoot): .notes('Must be a system administrator to call this.') .param('includeSettings', 'False to only show config; true to include full ' 'settings.', required=False, dataType='boolean', default=False) - .errorResponse('You are not a system administrator.', 403) + .errorResponse('You are not a system administrator.', 403), ) @boundHandler def getCurrentConfig(self, includeSettings=False): @@ -1001,7 +1009,6 @@ def getCurrentConfig(self, includeSettings=False): result[k][subk] = json.loads(json.dumps(subv)) except Exception: print(k, subk, subv) - pass elif not callable(v): result[k] = v if includeSettings: diff --git a/wsi_deid/web_client/stylesheets/ConfigView.styl b/wsi_deid/web_client/stylesheets/ConfigView.styl index b6c5056e..cbc277bd 100644 --- a/wsi_deid/web_client/stylesheets/ConfigView.styl +++ b/wsi_deid/web_client/stylesheets/ConfigView.styl @@ -1,4 +1,11 @@ -#g-hui-form +#g-wsi_deid-form .g-hui-description font-size 12px margin 0 0 2px + + input[type="checkbox"] + margin-right 7px + height 13px + + #g-advanced-settings-tab + background-color rgb(240, 238, 231) diff --git a/wsi_deid/web_client/stylesheets/ItemView.styl b/wsi_deid/web_client/stylesheets/ItemView.styl index 6ccd7f59..3a264093 100644 --- a/wsi_deid/web_client/stylesheets/ItemView.styl +++ b/wsi_deid/web_client/stylesheets/ItemView.styl @@ -84,8 +84,7 @@ top 0 right 0 bottom 0 - background-size contain - background-repeat no-repeat + background-size cover opacity 0.5 .g-widget-auximage-image-redact-square:after diff --git a/wsi_deid/web_client/templates/ConfigView.pug b/wsi_deid/web_client/templates/ConfigView.pug index 6b96b017..313ce105 100644 --- a/wsi_deid/web_client/templates/ConfigView.pug +++ b/wsi_deid/web_client/templates/ConfigView.pug @@ -165,10 +165,10 @@ form#g-wsi_deid-form(role="form") option(value='remote', selected=(settings['wsi_deid.sftp_mode'] === 'remote')) Remote (SFTP) transfer only option(value='both', selected=(settings['wsi_deid.sftp_mode'] === 'both')) Local export and remote transfer .form-group + input#g-wsi-deid-ocr-on-import.input-sm(type="checkbox", checked=(settings['wsi_deid.ocr_on_import'] ? "checked" : undefined)) label(for="g-wsi-deid-ocr-on-import") Find Label Text on Import p.g-hui-description | Run a background job to find label text on new images during an import - input#g-wsi-deid-ocr-on-import.input-sm(type="checkbox", checked=(settings['wsi_deid.ocr_on_import'] ? "checked" : undefined)) .form-group label(for="g-wsi-deid-db-api-url") SEER*DMS URL p.g-hui-description @@ -185,6 +185,193 @@ form#g-wsi_deid-form(role="form") type="text", value=settings['wsi_deid.db_api_key'] || '', title="The SEER*DMS API key", spellcheck="false") + + .g-settings-form-container + h4 Configuration Settings + .form-group + p.g-hui-description + | All of the settings in this section can be specified through the Girder configuration file. If changed via these settings, these values override the values in the configuration file. + + .form-group + input#g-wsi-deid-base_add_title_to_label.input-sm(type="checkbox", checked=(settings['wsi_deid.base_add_title_to_label'] ? "checked" : undefined)) + label(for="g-wsi-deid-base_add_title_to_label") Add File Name To Label + p.g-hui-description + | The label image will be redacted if appropriate; the file name is added to the top of the output label image. + .form-group + input#g-wsi-deid-base_always_redact_label.input-sm(type="checkbox", checked=(settings['wsi_deid.base_always_redact_label'] ? "checked" : undefined)) + label(for="g-wsi-deid-base_always_redact_label") Always Redact Label Image + p.g-hui-description + | The label image will be always be redacted. + .form-group + input#g-wsi-deid-base_redact_macro_square.input-sm(type="checkbox", checked=(settings['wsi_deid.base_redact_macro_square'] ? "checked" : undefined)) + label(for="g-wsi-deid-base_redact_macro_square") Always Redact Region of Macro Image + p.g-hui-description + | If selected, a default region of the macro image will be redacted. If a size is not specified, this region is a square at the upper left of the image spacing the shorter of the width or height. + label(for="g-wsi-deid-base_redact_macro_long_axis_percent") Redact Macro Image, Percentage of Long Axis + input#g-wsi-deid-base_redact_macro_long_axis_percent.form-control.input-sm( + type="text", value=settings['wsi_deid.base_redact_macro_long_axis_percent'], + title="Specify 0 to redact a square, otherwise a positive value less than or equal to 100") + label(for="g-wsi-deid-base_redact_macro_short_axis_percent") Redact Macro Image, Percentage of Short Axis + input#g-wsi-deid-base_redact_macro_short_axis_percent.form-control.input-sm( + type="text", value=settings['wsi_deid.base_redact_macro_short_axis_percent'], + title="Specify 0 to redact a square, otherwise a positive value less than or equal to 100") + + .form-group + input#g-wsi-deid-base_edit_metadata.input-sm(type="checkbox", checked=(settings['wsi_deid.base_edit_metadata'] ? "checked" : undefined)) + label(for="g-wsi-deid-base_edit_metadata") Allow Editing Metadata + p.g-hui-description + | If selected, metadata can be changed; otherwise redaction only removes metadata + .form-group + input#g-wsi-deid-base_show_metadata_in_lists.input-sm(type="checkbox", checked=(settings['wsi_deid.base_show_metadata_in_lists'] ? "checked" : undefined)) + label(for="g-wsi-deid-base_show_metadata_in_lists") Show Metadata In Lists + p.g-hui-description + | If selected, metadata is shown in lists, making them much more detailed + .form-group + label(for="g-wsi-deid-base_upload_metadata_for_export_report") Uploaded Metadata for Export Report + p.g-hui-description + | This is a JSON list of column names from the manifest file. Data from these columns will be added to the export report on the redacted images. + input#g-wsi-deid-base_upload_metadata_for_export_report.form-control.input-sm( + type="text", value=settings['wsi_deid.base_upload_metadata_for_export_report'] || '', + title='A JSON list of column names, like ["histology", "primary_site", "tumor_record_number"]', + spellcheck="false") + .form-group + label(for="g-wsi-deid-base_upload_metadata_add_to_images") Uploaded Metadata to Add to Images + p.g-hui-description + | This is a JSON list of column names from the manifest file. Data from these columns will be added to the custom metadata on the redacted images. + input#g-wsi-deid-base_upload_metadata_add_to_images.form-control.input-sm( + type="text", value=settings['wsi_deid.base_upload_metadata_add_to_images'] || '', + title='A JSON list of column names, like ["histology", "primary_site", "tumor_record_number"]', + spellcheck="false") + + .form-group + input#g-wsi-deid-base_require_redact_category.input-sm(type="checkbox", checked=(settings['wsi_deid.base_require_redact_category'] ? "checked" : undefined)) + label(for="g-wsi-deid-base_require_redact_category") Require a Redaction Category + p.g-hui-description + | If selected, when redacting metadata or an image, the type of PHI/PII must be selected + + .form-group + input#g-wsi-deid-base_require_reject_reason.input-sm(type="checkbox", checked=(settings['wsi_deid.base_require_reject_reason'] ? "checked" : undefined)) + label(for="g-wsi-deid-base_require_reject_reason") Require a Rejection Reason + p.g-hui-description + | If an image is rejected, the reason for rejection must be selected + + .form-group + input#g-wsi-deid-base_show_import_button.input-sm(type="checkbox", checked=(settings['wsi_deid.base_show_import_button'] ? "checked" : undefined)) + label(for="g-wsi-deid-base_show_import_button") Show Import Button + .form-group + label(for="g-wsi-deid-base_folder_name_field") Token ID Column Name + p.g-hui-description + | The token id column of a manifest file is used to group subjects. + input#g-wsi-deid-base_folder_name_field.form-control.input-sm( + type="text", value=settings['wsi_deid.base_folder_name_field'] || '', + title="The column in the manifest file with the subject name", + spellcheck="false") + .form-group + label(for="g-wsi-deid-base_image_name_field") Image ID Column Name + p.g-hui-description + | The image id column of a manifest file is used to distinguish images for a single subject. This may be blank + input#g-wsi-deid-base_image_name_field.form-control.input-sm( + type="text", value=settings['wsi_deid.base_image_name_field'] || '', + title="The column in the manifest file with the image name", + spellcheck="false") + .form-group + input#g-wsi-deid-base_validate_image_id_field.input-sm(type="checkbox", checked=(settings['wsi_deid.base_validate_image_id_field'] ? "checked" : undefined)) + label(for="g-wsi-deid-base_validate_image_id_field") Validate Image Name Field + p.g-hui-description + | If selected and the import manifest file contains an image id column (regardless of column name), the value in that field must be is the folder name field value combined with the Proc_Seq and Slide_ID field. In unselected, only the schema imposes requirements on the image id column values. + .form-group + label(for="g-wsi-deid-base_new_token_pattern") New Token Pattern + p.g-hui-description + | A template for generating new token IDs for otherwise unfiled images. # is replaced by a random digit. @ is replaced by a random uppercase letter. + input#g-wsi-deid-base_new_token_pattern.form-control.input-sm( + type="text", value=settings['wsi_deid.base_new_token_pattern'] || '', + title="A pattern for new tokens, such as ####@@####", + spellcheck="false") + .form-group + label(for="g-wsi-deid-base_import_text_association_columns") Import Text Association Columns + p.g-hui-description + | This is a JSON list of column names from the manifest file. Data from these columns will be compared against label OCR results to find match metadata with images. + input#g-wsi-deid-base_import_text_association_columns.form-control.input-sm( + type="text", value=settings['wsi_deid.base_import_text_association_columns'] || '', + title='A JSON list of column names, like ["Name", "DOB", "PatientID"]', + spellcheck="false") + .form-group + input#g-wsi-deid-base_reimport_if_moved.input-sm(type="checkbox", checked=(settings['wsi_deid.base_reimport_if_moved'] ? "checked" : undefined)) + label(for="g-wsi-deid-base_reimport_if_moved") Reimport Files if Moved + p.g-hui-description + | When importing from the import directory, if a file is already in the system (for instance, already processed), then if this is unselected, the file will not be reimported. If selected, files already in the system WILL be reimported. + .form-group + input#g-wsi-deid-base_show_export_button.input-sm(type="checkbox", checked=(settings['wsi_deid.base_show_export_button'] ? "checked" : undefined)) + label(for="g-wsi-deid-base_show_export_button") Show Export Button + .form-group + input#g-wsi-deid-base_show_next_item.input-sm(type="checkbox", checked=(settings['wsi_deid.base_show_next_item'] ? "checked" : undefined)) + label(for="g-wsi-deid-base_show_next_item") Show Next Item on Left Navigation Pane + p.g-hui-description + | If you change this, you need to reload the webpage for it to take effect. + + .form-group + input#g-wsi-deid-base_show_next_folder.input-sm(type="checkbox", checked=(settings['wsi_deid.base_show_next_folder'] ? "checked" : undefined)) + label(for="g-wsi-deid-base_show_next_folder") Show Next Folder on Left Navigation Pane + p.g-hui-description + | If you change this, you need to reload the webpage for it to take effect. + + #g-configuration-accordion.panel-group + .panel.panel-default + .panel-heading(data-toggle="collapse", + data-parent="#g-configuration-accordion", + data-target="#g-advanced-settings-tab") + .panel-title + a + b Advanced Settings + #g-advanced-settings-tab.panel-collapse.collapse + .panel-body + p These settings should only be changed if you are certain of what you are doing. + + .form-group + button#g-wsi-deid-reset-settings.btn.btn-sm.btn-default Reset to Configuration File Values + + .form-group + label(for="g-wsi-deid-base_phi_pii_types") PHI/PII Redaction Categories + p.g-hui-description + | This is a JSON list of redaction reasons. Each entry needs to have a category (stored internally), a text field (shown in the UI), and either a key (stored internally), or a types entry with a list of entries with text and key values. + input#g-wsi-deid-base_phi_pii_types.form-control.input-sm( + type="text", value=settings['wsi_deid.base_phi_pii_types'] || '', + title='A JSON list of dictionaries.', + spellcheck="false") + .form-group + label(for="g-wsi-deid-base_reject_reasons") Rejection Reasons + p.g-hui-description + | This is a JSON list of rejection reasons. Each entry needs to have a category (stored internally), a text field (shown in the UI), and either a key (stored internally), or a types entry with a list of entries with text and key values. + input#g-wsi-deid-base_reject_reasons.form-control.input-sm( + type="text", value=settings['wsi_deid.base_reject_reasons'] || '', + title='A JSON list of dictionaries.', + spellcheck="false") + + - + var regexList = [ + ['hide_metadata_keys', 'Hide Metadata - General'], + ['hide_metadata_keys_format_aperio', 'Hide Metadata - Aperio (.svs)'], + ['hide_metadata_keys_format_hamamatsu', 'Hide Metadata - Hamamatsu (.ndpi)'], + ['hide_metadata_keys_format_philips', 'Hide Metadata - Philips TIFF (.ptif)'], + ['hide_metadata_keys_format_isyntax', 'Hide Metadata - Philips iSyntax (.isyntax, .isyntax2)'], + ['no_redact_control_keys', 'No Redaction Controls - General'], + ['no_redact_control_keys_format_aperio', 'No Redaction Controls - Aperio (.svs)'], + ['no_redact_control_keys_format_hamamatsu', 'No Redaction Controls - Hamamatsu (.ndpi)'], + ['no_redact_control_keys_format_philips', 'No Redaction Controls - Philips TIFF (.ptif)'], + ['no_redact_control_keys_format_isyntax', 'No Redaction Controls - Philips iSyntax (.isyntax, .isyntax2)'] + ]; + for entry in regexList + .form-group + label(for="g-wsi-deid-base_" + entry[0]) + = entry[1] + p.g-hui-description + | A dictionary where the keys are regular expressions to match metadata keys and the values are regular expressions to match metadata values. + input.form-control.input-sm( + id='g-wsi-deid-base_' + entry[0], + type="text", value=settings['wsi_deid.base_' + entry[0]] || '', + title='A JSON dictionary of regex keys and values.', + spellcheck="false") + p#g-hui-error-message.g-validation-failed-message .g-hui-buttons button#g-hui-save.btn.btn-sm.btn-primary Save diff --git a/wsi_deid/web_client/views/ConfigView.js b/wsi_deid/web_client/views/ConfigView.js index 4a30747a..e896419e 100644 --- a/wsi_deid/web_client/views/ConfigView.js +++ b/wsi_deid/web_client/views/ConfigView.js @@ -1,7 +1,8 @@ import $ from 'jquery'; import _ from 'underscore'; -import View from '@girder/core/views/View'; +import Backbone from 'backbone'; +import View from '@girder/core/views/View'; import PluginConfigBreadcrumbWidget from '@girder/core/views/widgets/PluginConfigBreadcrumbWidget'; import BrowserWidget from '@girder/core/views/widgets/BrowserWidget'; import { restRequest } from '@girder/core/rest'; @@ -30,11 +31,34 @@ var ConfigView = View.extend({ } return result; }); + Object.keys(this.baseSettings).forEach((key) => { + const element = this.$('#g-wsi-deid-base_' + key.replace('wsi_deid.base_', '')); + if (!element.length) { + return; + } + var result = { + key, + value: (element.is('input[type="checkbox"]') ? !!element.is(':checked') : (element.val().trim())) + }; + if (this.baseSettings[key] !== result.value) { + settings.push(result); + } + }); this._saveSettings(settings); }, 'click #g-hui-cancel': function (event) { router.navigate('plugins', { trigger: true }); }, + 'click #g-wsi-deid-reset-settings': function (event) { + const settings = []; + Object.keys(this.baseSettings).forEach((key) => settings.push({ + key, + value: null + })); + this._saveSettings(settings).done(() => { + Backbone.history.loadUrl(Backbone.history.fragment); + }); + }, 'click .g-open-browser': '_openBrowser' }, initialize: function () { @@ -86,8 +110,18 @@ var ConfigView = View.extend({ } }).done((resp) => { this.defaults = resp; + }), + restRequest({ + url: `wsi_deid/settings`, + error: null + }).done((settings) => { + this.baseSettings = {}; + Object.keys(settings).forEach((key) => { + this.baseSettings['wsi_deid.base_' + key] = settings[key]; + }); }) ).done(() => { + Object.assign(this.settings, this.baseSettings); this.render(); for (const [key, value] of Object.entries(this.settingsKeys)) { diff --git a/wsi_deid/web_client/views/ItemView.js b/wsi_deid/web_client/views/ItemView.js index 7e847755..6e97ace2 100644 --- a/wsi_deid/web_client/views/ItemView.js +++ b/wsi_deid/web_client/views/ItemView.js @@ -120,19 +120,25 @@ wrap(ItemView, 'render', function (render) { parentElem.append(inputControl); }; - const resizeRedactSquare = (elem) => { + const resizeRedactSquare = (elem, settings) => { const image = elem.find('.g-widget-auximage-image img'); if (!image.length) { return; } const minwh = Math.min(image.width(), image.height()); if (minwh > 0) { + let redw = minwh; + let redh = minwh; + if (settings.redact_macro_long_axis_percent && settings.redact_macro_short_axis_percent && settings.redact_macro_long_axis_percent > 0 && settings.redact_macro_short_axis_percent > 0) { + redw = image.width() * (image.width() >= image.height() ? settings.redact_macro_long_axis_percent : settings.redact_macro_short_axis_percent) / 100; + redh = image.height() * (image.width() >= image.height() ? settings.redact_macro_short_axis_percent : settings.redact_macro_long_axis_percent) / 100; + } const redactsquare = elem.find('.g-widget-auximage-image-redact-square'); - redactsquare.width(minwh); - redactsquare.height(minwh); + redactsquare.width(redw); + redactsquare.height(redh); return; } - window.setTimeout(() => resizeRedactSquare(elem), 1000); + window.setTimeout(() => resizeRedactSquare(elem, settings), 1000); }; const resizeRedactBackground = (elem) => { @@ -272,7 +278,7 @@ wrap(ItemView, 'render', function (render) { if (keyname === 'macro') { const redactsquare = $('
 
'); elem.find('.g-widget-auximage-image').append(redactsquare); - resizeRedactSquare(elem); + resizeRedactSquare(elem, settings); if (!settings.redact_macro_square) { const check = $('Partial'); if ((redactList.images[keyname] || {}).square) {