Skip to content

Commit

Permalink
Editor / Add keywords even if thesaurus not available in record langu…
Browse files Browse the repository at this point in the history 
…ages (#8268)

* Editor / Add keywords even if thesaurus not available in record languages

Add english as default language for thesaurus searches in order to populate metadata with english label if the thesaurus is not available in the current record language. Can happen with some DCAT-AP related vocabularies that have to be used in Member States records not often in english.

This will also avoid the message box when keywords are not found due to language mismatch. The message will still be displayed if a keyword in the metadata is not found in the thesaurus, proposing the user to only keep keywords from the thesaurus.

Funded by Service Public de Wallonie

* Update KeywordsApi.java

* Code quality improvements in KeywordsApi

---------

Co-authored-by: Jose García <[email protected]>
  • Loading branch information
@fxprunayre @josegar74
fxprunayre and josegar74 authored Dec 11, 2024
1 parent 96869d8 commit 795b153
Show file tree
Hide file tree
Showing 8 changed files with 73 additions and 95 deletions.
8 changes: 5 additions & 3 deletions core/src/main/java/org/fao/geonet/kernel/search/KeywordsSearcher.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -101,9 +101,11 @@ public static Element toRawElement(Element rootEl, KeywordBean kb) {

elKeyword.addContent(elSelected);
elKeyword.addContent(new Element("id").addContent(Integer.toString(kb.getId())));
elKeyword.addContent(new Element("value").addContent(kb.getDefaultValue()).setAttribute("language", defaultLang));
elKeyword.addContent(new Element("definition").addContent(kb.getDefaultDefinition()).setAttribute("language", defaultLang));
elKeyword.addContent(new Element("defaultLang").addContent(defaultLang));
if (defaultLang != null) {
elKeyword.addContent(new Element("value").addContent(kb.getDefaultValue()).setAttribute("language", defaultLang));
elKeyword.addContent(new Element("definition").addContent(kb.getDefaultDefinition()).setAttribute("language", defaultLang));
elKeyword.addContent(new Element("defaultLang").addContent(defaultLang));
}
Element thesaurusElement = new Element("thesaurus");
thesaurusElement.addContent(new Element("key").setText(kb.getThesaurusKey()));
thesaurusElement.addContent(new Element("title").setText(kb.getThesaurusTitle()));
Expand Down
2 changes: 1 addition & 1 deletion .../iso19115-3.2018/src/main/plugin/iso19115-3.2018/layout/layout-custom-fields-keywords.xsl
View file
Original file line number Diff line number Diff line change
Expand Up @@ -179,7 +179,7 @@
in default language
-->
<xsl:variable name="keywords" select="string-join(
if ($guiLangId and mri:keyword//*[@locale = concat('#', $guiLangId)])
if ($guiLangId and mri:keyword//*[@locale = concat('#', $guiLangId)][*/text() != ''])
then mri:keyword//*[@locale = concat('#', $guiLangId)][. != '']/replace(text(), ',', ',,')
else mri:keyword/*[1][. != '']/replace(text(), ',', ',,'), ',')"/>

Expand Down
118 changes: 45 additions & 73 deletions services/src/main/java/org/fao/geonet/api/registries/vocabularies/KeywordsApi.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -46,7 +46,6 @@
import org.fao.geonet.api.exception.ResourceNotFoundException;
import org.fao.geonet.api.exception.WebApplicationException;
import org.fao.geonet.api.registries.model.ThesaurusInfo;
import org.fao.geonet.api.tools.i18n.LanguageUtils;
import org.fao.geonet.constants.Geonet;
import org.fao.geonet.domain.ISODate;
import org.fao.geonet.exceptions.BadParameterEx;
Expand Down Expand Up @@ -110,12 +109,6 @@
description = ApiParams.API_CLASS_REGISTRIES_OPS)
public class KeywordsApi {

/**
* The language utils.
*/
@Autowired
LanguageUtils languageUtils;

@Autowired
SettingManager settingManager;

Expand Down Expand Up @@ -161,8 +154,7 @@ public class KeywordsApi {
* @throws Exception the exception
*/
@io.swagger.v3.oas.annotations.Operation(
summary = "Search keywords",
description = "")
summary = "Search keywords")
@RequestMapping(
path = "/search",
method = RequestMethod.GET,
Expand All @@ -175,52 +167,46 @@ public class KeywordsApi {
@ResponseBody
public Object searchKeywords(
@Parameter(
description = "Query",
required = false
description = "Query"
)
@RequestParam(
required = false
)
String q,
@Parameter(
description = "Query in that language",
required = false
description = "Query in that language"
)
@RequestParam(
value = "lang",
defaultValue = "eng"
)
String lang,
@Parameter(
description = "Number of rows",
required = false
description = "Number of rows"
)
@RequestParam(
required = false,
defaultValue = "1000"
)
int rows,
@Parameter(
description = "Start from",
required = false
description = "Start from"
)
@RequestParam(
defaultValue = "0",
required = false
)
int start,
@Parameter(
description = "Return keyword information in one or more languages",
required = false
description = "Return keyword information in one or more languages"
)
@RequestParam(
value = XmlParams.pLang,
required = false
)
List<String> targetLangs,
@Parameter(
description = "Thesaurus identifier",
required = false
description = "Thesaurus identifier"
)
@RequestParam(
required = false
Expand All @@ -235,24 +221,21 @@ public Object searchKeywords(
// )
// String thesauriDomainName,
@Parameter(
description = "Type of search",
required = false
description = "Type of search"
)
@RequestParam(
defaultValue = "CONTAINS"
)
KeywordSearchType type,
@Parameter(
description = "URI query",
required = false
description = "URI query"
)
@RequestParam(
required = false
)
String uri,
@Parameter(
description = "Sort by",
required = false
description = "Sort by"
)
@RequestParam(
required = false,
Expand Down Expand Up @@ -354,23 +337,19 @@ public Object getKeywordById(
@RequestParam(name = "thesaurus")
String sThesaurusName,
@Parameter(
description = "Languages.",
required = false)
description = "Languages.")
@RequestParam(name = "lang", required = false)
String[] langs,
@Parameter(
description = "Only print the keyword, no thesaurus information.",
required = false)
description = "Only print the keyword, no thesaurus information.")
@RequestParam(required = false, defaultValue = "false")
boolean keywordOnly,
@Parameter(
description = "XSL template to use (ISO19139 keyword by default, see convert.xsl).",
required = false)
description = "XSL template to use (ISO19139 keyword by default, see convert.xsl).")
@RequestParam(required = false)
String transformation,
@Parameter(
description = "langMap, that converts the values in the 'lang' parameter to how they will be actually represented in the record. {'fre':'fra'} or {'fre':'fr'}. Missing/empty means to convert to iso 2 letter.",
required = false)
description = "langMap, that converts the values in the 'lang' parameter to how they will be actually represented in the record. {'fre':'fra'} or {'fre':'fr'}. Missing/empty means to convert to iso 2 letter.")
@RequestParam (name = "langMap", required = false)
String langMapJson,
@Parameter(hidden = true)
Expand Down Expand Up @@ -428,23 +407,19 @@ public Object getKeywordByIds(
@RequestParam(name = "thesaurus")
String sThesaurusName,
@Parameter(
description = "Languages.",
required = false)
description = "Languages.")
@RequestParam(name = "lang", required = false)
String[] langs,
@Parameter(
description = "Only print the keyword, no thesaurus information.",
required = false)
description = "Only print the keyword, no thesaurus information.")
@RequestParam(required = false, defaultValue = "false")
boolean keywordOnly,
@Parameter(
description = "XSL template to use (ISO19139 keyword by default, see convert.xsl).",
required = false)
description = "XSL template to use (ISO19139 keyword by default, see convert.xsl).")
@RequestParam(required = false)
String transformation,
@Parameter(
description = "langMap, that converts the values in the 'lang' parameter to how they will be actually represented in the record. {'fre':'fra'} or {'fre':'fr'}. Missing/empty means to convert to iso 2 letter.",
required = false)
description = "langMap, that converts the values in the 'lang' parameter to how they will be actually represented in the record. {'fre':'fra'} or {'fre':'fr'}. Missing/empty means to convert to iso 2 letter.")
@RequestParam (name = "langMap", required = false)
String langMapJson,
@Parameter(hidden = true)
Expand Down Expand Up @@ -489,7 +464,7 @@ private Object getKeyword(
if (thesaurus == null) {
String finalSThesaurusName = sThesaurusName;
Optional<Thesaurus> thesaurusEntry = thesaurusManager.getThesauriMap().values().stream().filter(t -> t.getKey().endsWith(finalSThesaurusName)).findFirst();
if (!thesaurusEntry.isPresent()) {
if (thesaurusEntry.isEmpty()) {
throw new IllegalArgumentException(String.format(
"Thesaurus '%s' not found.", sThesaurusName));
} else {
Expand All @@ -501,7 +476,12 @@ private Object getKeyword(
if (langs == null) {
langs = context.getLanguage().split(",");
}
String[] iso3langCodes = Arrays.copyOf(langs, langs.length);
List<String> langList = new ArrayList<>(List.of(langs));
if (!langList.contains("eng")) {
langList.add("eng");
}

String[] iso3langCodes = langList.toArray(String[]::new);
for (int i = 0; i < langs.length; i++) {
if (StringUtils.isNotEmpty(langs[i])) {
langs[i] = mapper.iso639_2_to_iso639_1(langs[i], langs[i].substring(0,2)); //default: fra -> fr
Expand All @@ -511,7 +491,7 @@ private Object getKeyword(
Element descKeys;
Map<String, Map<String, String>> jsonResponse = new HashMap<>();

uri = URLDecoder.decode(uri, "UTF-8");
uri = URLDecoder.decode(uri, StandardCharsets.UTF_8);

if (uri == null) {
descKeys = new Element("descKeys");
Expand Down Expand Up @@ -566,8 +546,8 @@ private Object getKeyword(
JSONObject obj = JSONObject.fromObject(langMapJson);
langConversion = new Element("languageConversions");
for(Object entry : obj.entrySet()) {
String key = ((Map.Entry) entry).getKey().toString();
String value = ((Map.Entry) entry).getValue().toString();
String key = ((Map.Entry<?, ?>) entry).getKey().toString();
String value = ((Map.Entry<?, ?>) entry).getValue().toString();
Element conv = new Element("conversion");
conv.setAttribute("from", key);
conv.setAttribute("to", value.replace("#",""));
Expand All @@ -593,12 +573,7 @@ private Object getKeyword(

Element requestParams = new Element("request");
for (Map.Entry<String, String> e : allRequestParams.entrySet()) {
if (e.getKey().equals("lang")) {
requestParams.addContent(new Element(e.getKey())
.setText(String.join(",", iso3langCodes)));
} else {
requestParams.addContent(new Element(e.getKey()).setText(e.getValue()));
}
requestParams.addContent(new Element(e.getKey()).setText(e.getValue()));
}
if (langConversion != null) {
requestParams.addContent(langConversion);
Expand All @@ -618,7 +593,6 @@ private Object getKeyword(
*
* @param thesaurus the thesaurus
* @param response the response
* @return the thesaurus
* @throws Exception the exception
*/
@io.swagger.v3.oas.annotations.Operation(
Expand Down Expand Up @@ -672,7 +646,6 @@ public void getThesaurus(
* Delete thesaurus.
*
* @param thesaurus the thesaurus
* @return the element
* @throws Exception the exception
*/
@io.swagger.v3.oas.annotations.Operation(
Expand Down Expand Up @@ -703,7 +676,7 @@ public void deleteThesaurus(
throw new ResourceNotFoundException(String.format(
"Thesaurus with identifier '%s' not found in the catalogue. Should be one of: %s",
thesaurus,
thesaurusMan.getThesauriMap().keySet().toString()
thesaurusMan.getThesauriMap().keySet()
));
}
Path item = thesaurusObject.getFile();
Expand Down Expand Up @@ -767,9 +740,9 @@ public String uploadThesaurus(
boolean fileUpload = file != null && !file.isEmpty();

// Upload RDF file
Path rdfFile = null;
String fname = null;
File tempDir = null;
Path rdfFile;
String fname;
File tempDir;

if (fileUpload) {

Expand All @@ -795,7 +768,7 @@ public String uploadThesaurus(
}

long fsize;
if (rdfFile != null && Files.exists(rdfFile)) {
if (Files.exists(rdfFile)) {
fsize = Files.size(rdfFile);
} else {
throw new MissingServletRequestParameterException("Thesaurus file doesn't exist", "file");
Expand Down Expand Up @@ -1012,7 +985,7 @@ public Element convertCsvToSkos(Path csvFile,
CSVParser csvParser = new CSVParser(reader, CSVFormat.DEFAULT
.withFirstRecordAsHeader()
.withIgnoreHeaderCase()
.withTrim());
.withTrim())
) {

Map<String, KeywordBean> allConcepts = new LinkedHashMap<>();
Expand Down Expand Up @@ -1075,7 +1048,7 @@ public Element convertCsvToSkos(Path csvFile,
}

Element scheme = buildConceptScheme(csvFile, thesaurusTitle, thesaurusNamespaceUrl);
if(broaderLinks.size() > 0 && !topConcepts.isEmpty()) {
if(!broaderLinks.isEmpty() && !topConcepts.isEmpty()) {
topConcepts.forEach(t -> {
Element topConcept = new Element("hasTopConcept", SKOS_NAMESPACE);
topConcept.setAttribute("resource", t, RDF_NAMESPACE);
Expand Down Expand Up @@ -1210,7 +1183,7 @@ public String uploadThesaurusFromUrl(
boolean registryUpload = !StringUtils.isEmpty(registryUrl);

// Upload RDF file
Path rdfFile = null;
Path rdfFile;
String fname = null;

// Specific upload steps
Expand Down Expand Up @@ -1307,9 +1280,9 @@ public String uploadThesaurusFromUrl(
/**
* Update the information related to a local thesaurus .
*
* @param thesaurusInfo
* @return
* @throws Exception
* @param thesaurus name of the thesaurus to update.
* @param thesaurusInfo thesaurus information to update.
* @throws Exception the exception
*/
@io.swagger.v3.oas.annotations.Operation(
summary = "Updates the information of a local thesaurus",
Expand Down Expand Up @@ -1368,11 +1341,11 @@ public void updateThesaurus(
* Download for each language the codelist from the registry. Combine
* them into one XML document which is then XSLT processed for SKOS conversion.
*
* @param registryUrl the registry url
* @param registryType
* @param itemName the item name
* @param lang the selected languages
* @param context the context
* @param registryUrl the registry url
* @param registryType type of registry
* @param itemName the item name
* @param lang the selected languages
* @param context the context
* @return the path
* @throws Exception the exception
*/
Expand Down Expand Up @@ -1448,7 +1421,6 @@ private Path getXMLContentFromUrl(String url, ServiceContext context) throws URI
* @param fname the fname
* @param type the type
* @param dir the dir
* @return Element thesaurus uploaded
* @throws Exception the exception
*/
private void uploadThesaurus(Path rdfFile, String style,
Expand Down
6 changes: 3 additions & 3 deletions services/src/main/java/org/fao/geonet/services/thesaurus/GetKeywordByIdAsConcept.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -73,7 +73,7 @@ public Element exec(Element params, ServiceContext context)
searcher = new KeywordsSearcher(context, thesaurusMan);
KeywordBean kb = null;

kb = searcher.searchById(uri, sThesaurusName, langForThesaurus);
kb = searcher.searchById(uri, sThesaurusName, langForThesaurus, "eng");
if (kb == null) {
root = new Element("descKeys");
} else {
Expand All @@ -93,11 +93,11 @@ public Element exec(Element params, ServiceContext context)
reqType = KeywordRelation.RELATED;
}

searcher.searchForRelated(params, reqType, KeywordSort.defaultLabelSorter(SortDirection.DESC), lang);
searcher.searchForRelated(params, reqType, KeywordSort.defaultLabelSorter(SortDirection.DESC), lang, "eng");
// build response for each request type
Element keywordType = new Element(request);
for (KeywordBean kbr : searcher.getResults()) {
keywordType.addContent(kbr.toElement(context.getLanguage()));
keywordType.addContent(kbr.toElement(context.getLanguage(), "eng"));
}
root.addContent(keywordType);
}
Expand Down
2 changes: 1 addition & 1 deletion services/src/main/java/org/fao/geonet/services/thesaurus/GetTopConcept.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -70,7 +70,7 @@ public Element exec(Element params, ServiceContext context)
searcher = new KeywordsSearcher(context, thesaurusMan);

try {
searcher.searchTopConcepts(sThesaurusName, langForThesaurus);
searcher.searchTopConcepts(sThesaurusName, langForThesaurus, "eng");

KeywordBean topConcept = new KeywordBean(the.getIsoLanguageMapper());
topConcept.setThesaurusInfo(the);
Expand Down
Loading

0 comments on commit 795b153

Please sign in to comment.