diff --git a/ckanext/udc_import_other_portals/jobs.py b/ckanext/udc_import_other_portals/jobs.py index 51ff48c..57950a4 100644 --- a/ckanext/udc_import_other_portals/jobs.py +++ b/ckanext/udc_import_other_portals/jobs.py @@ -28,7 +28,7 @@ def job_run_import(import_config_id: str, run_by: str, job_id: str): logic.ValidationError("import_config_id should be provided.") ) - import_config = CUDCImportConfig.get(import_config_id) + import_config: 'CUDCImportConfig' = CUDCImportConfig.get(import_config_id) if not import_config: raise logger.exception( diff --git a/ckanext/udc_import_other_portals/logic/base.py b/ckanext/udc_import_other_portals/logic/base.py index 7655d26..dbf7b7a 100644 --- a/ckanext/udc_import_other_portals/logic/base.py +++ b/ckanext/udc_import_other_portals/logic/base.py @@ -1,4 +1,5 @@ from ckanext.udc_import_other_portals.logger import ImportLogger, generate_trace +from ckanext.udc_import_other_portals.model import CUDCImportConfig from ckanext.udc_import_other_portals.worker.socketio_client import SocketClient import ckan.plugins.toolkit as toolkit from ckan.types import Context @@ -7,6 +8,7 @@ from ckan.common import current_user from ckan.lib.search.common import SearchIndexError +import threading from typing import List, Dict, cast from .deduplication import find_duplicated_packages, process_duplication @@ -14,12 +16,13 @@ base_logger = logging.getLogger(__name__) +lock = threading.Lock() class ImportError(ValueError): pass -def get_package(context: Context, package_id: str=None, package_name: str=None): +def get_package(context: Context, package_id: str = None, package_name: str = None): if not package_id and not package_name: raise ValueError("Either package_id or package_name should be provided.") if package_id and package_name: @@ -30,15 +33,15 @@ def get_package(context: Context, package_id: str=None, package_name: str=None): data_dict = {"name": package_name} logic.check_access("package_show", context, data_dict=data_dict) package_dict = logic.get_action("package_show")(context, data_dict) - + # Prevent the package with the same name but different id (the provided id is treated as a name) if package_id and package_dict["id"] != package_id: raise ValueError(f"Package id={package_id} is not found.") - + return package_dict -def check_existing_package_id_or_name(context, id: str=None, name: str=None): +def check_existing_package_id_or_name(context, id: str = None, name: str = None): if not id and not name: raise ValueError("Either id or name should be provided.") if id and name: @@ -92,6 +95,32 @@ def delete_package(context: Context, package_id: str): logic.get_action("package_delete")(context, {"id": package_id}) +def get_organization(context: Context, organization_id: str = None): + data_dict = {"id": organization_id} + logic.check_access("organization_show", context, data_dict=data_dict) + return logic.get_action("organization_show")(context, data_dict) + + +def get_organization_ids(context: Context): + logic.check_access("organization_list", context) + return logic.get_action("organization_list")(context) + + +def ensure_organization(context: Context, organization: dict): + with lock: + if not context: + # testing environment, do not create organization + return + logic.check_access("organization_list", context) + organization_ids = logic.get_action("organization_list")(context) + for organization_id in organization_ids: + if organization_id == organization["id"]: + return + + logic.check_access("organization_create", context, data_dict=organization) + logic.get_action("organization_create")(context, organization) + + class BaseImport: """ Abstract class that manages logging and provides interface to backend APIs @@ -102,7 +131,7 @@ class BaseImport: running = False socket_client: SocketClient = None - def __init__(self, context, import_config, job_id): + def __init__(self, context, import_config: "CUDCImportConfig", job_id): self.context = context self.import_config = import_config self.job_id = job_id @@ -123,7 +152,7 @@ def build_context(self): ) return context - def map_to_cudc_package(self, src: dict): + def map_to_cudc_package(self, src: dict, target: dict): """ Map source package to cudc package. @@ -142,8 +171,47 @@ def process_package(self, src): Returns: str: The ID of the mapped package. """ + # Some defaults + target = { + "owner_org": self.import_config.owner_org, + "type": "catalogue", + "license_id": "notspecified", + } + platform = self.import_config.platform + if platform == "ckan": + org_import_mode = self.import_config.other_config.get("org_import_mode") + org_mapping = self.import_config.other_config.get("org_mapping") or {} + + if org_import_mode == "importToOwnOrg": + if org_mapping.get(src["owner_org"]): + target["owner_org"] = org_mapping[src["owner_org"]] + else: + # Use the same organization id + target["owner_org"] = src["owner_org"] + + + query = ( + model.Session.query(model.Group) + .filter(model.Group.id == src["owner_org"]) + .filter(model.Group.is_organization == True) + ) + org = query.first() + + # Create the organization if not exists + if org is None: + ensure_organization( + self.build_context(), + { + "id": src["organization"]["id"], + "name": src["organization"]["name"], + "title": src["organization"]["title"], + "description": src["organization"]["description"], + }, + ) + model.Session.commit() + try: - mapped = self.map_to_cudc_package(src) + mapped = self.map_to_cudc_package(src, target) except Exception as e: self.logger.error(f"ERROR: Failed to map package from source.") self.logger.exception(e) @@ -260,19 +328,19 @@ def run_imports(self): def ensure_license(context, license_id, license_title, license_url, check=True): """Ensure that the license exists in the database.""" - if not context: - # tesing environment, do not create license - return - licenses = logic.get_action("licenses_get")(context) - for license in licenses: - if license["id"] == license_id: + with lock: + if not context: + # tesing environment, do not create license return - try: - logic.get_action("license_create")( - context, {"id": license_id, "title": license_title, "url": license_url} - ) - except: - # Weird concurrency issue - pass - return - + licenses = logic.get_action("licenses_get")(context) + for license in licenses: + if license["id"] == license_id: + return + try: + logic.get_action("license_create")( + context, {"id": license_id, "title": license_title, "url": license_url} + ) + except: + # Weird concurrency issue + pass + return diff --git a/ckanext/udc_import_other_portals/logic/ckan_based/api.py b/ckanext/udc_import_other_portals/logic/ckan_based/api.py index a8942e0..0816bb4 100644 --- a/ckanext/udc_import_other_portals/logic/ckan_based/api.py +++ b/ckanext/udc_import_other_portals/logic/ckan_based/api.py @@ -24,7 +24,7 @@ def get_package(package_id, base_api, api_key=None): return res["result"] -def get_all_packages(base_api, size=None, api_key=None): +def get_all_packages(base_api, size=None, api_key=None, cb=None): """ Retrieve all packages from the CKAN API using the package_search endpoint. @@ -45,6 +45,8 @@ def get_all_packages(base_api, size=None, api_key=None): # Construct the API request URL url = f"{base_api}/3/action/package_search?rows={rows}&start={offset}" print(f"getting package rows={rows} offset={offset}") + if cb: + cb(f"Got {offset} packages") try: # Make the API request @@ -138,3 +140,36 @@ def check_site_alive(base_api): return res["result"] except: return False + +def get_organization(base_api, organization_id=None): + res = requests.get(f"{base_api}/3/action/organization_show?id={organization_id}").json() + return res["result"] + +def get_organization_ids(base_api): + res = requests.get(f"{base_api}/3/action/organization_list").json() + return res["result"] + +def get_organizations(base_api): + """ + Example response: + [ + { + "approval_status": "approved", + "created": "2018-07-27T18:51:10.451359", + "description": "", + "display_name": "Argentia Private Investments Inc. | Argentia Private Investments Inc.", + "id": "76287b5c-ceb0-44fb-a62f-3cd4ee5de656", + "image_display_url": "", + "image_url": "", + "is_organization": true, + "name": "api", + "num_followers": 0, + "package_count": 0, + "state": "active", + "title": "Argentia Private Investments Inc. | Argentia Private Investments Inc.", + "type": "organization" + }, + ] + """ + res = requests.get(f"{base_api}/3/action/organization_list?all_fields=true&limit=1000").json() + return res["result"] diff --git a/ckanext/udc_import_other_portals/logic/ckan_based/base.py b/ckanext/udc_import_other_portals/logic/ckan_based/base.py index 1bc9a40..9ec06bf 100644 --- a/ckanext/udc_import_other_portals/logic/ckan_based/base.py +++ b/ckanext/udc_import_other_portals/logic/ckan_based/base.py @@ -1,5 +1,6 @@ import traceback import logging +from ckanext.udc_import_other_portals.model import CUDCImportConfig from ckanext.udc_import_other_portals.worker.socketio_client import SocketClient import time from concurrent.futures import ThreadPoolExecutor, as_completed @@ -18,9 +19,9 @@ class CKANBasedImport(BaseImport): Abstract class for imports """ - def __init__(self, context, import_config, job_id, base_api): + def __init__(self, context, import_config: 'CUDCImportConfig', job_id: str): super().__init__(context, import_config, job_id) - self.base_api = base_api + self.base_api = import_config.other_config.get("base_api") def iterate_imports(self): """ @@ -35,10 +36,12 @@ def run_imports(self): """ self.running = True self.socket_client = SocketClient(self.job_id) - self.all_packages = get_all_packages(self.base_api) + self.logger = ImportLogger(base_logger, 0, self.socket_client) + + self.all_packages = get_all_packages(self.base_api, cb=lambda x: self.logger.info(x)) self.packages_ids = [p['id'] for p in self.all_packages] # Set the import size for reporting in the frontend - self.import_size = len(self.packages_ids) + self.logger.total = self.import_size = len(self.packages_ids) # Make sure the sockeio server is connected while not self.socket_client.registered: @@ -46,7 +49,6 @@ def run_imports(self): base_logger.info("Waiting socketio to be connected.") base_logger.info("socketio connected.") print("self.import_size", self.import_size) - self.logger = ImportLogger(base_logger, self.import_size, self.socket_client) # Check if packages are deleted from the remote since last import if self.import_config.other_data is None: diff --git a/ckanext/udc_import_other_portals/logic/ckan_based/city_of_toronto.py b/ckanext/udc_import_other_portals/logic/ckan_based/city_of_toronto.py index c2b572c..869f7ee 100644 --- a/ckanext/udc_import_other_portals/logic/ckan_based/city_of_toronto.py +++ b/ckanext/udc_import_other_portals/logic/ckan_based/city_of_toronto.py @@ -48,16 +48,8 @@ "XML": "application/xml", } - +# "https://ckan0.cf.opendata.inter.prod-toronto.ca/api" class CityOfTorontoImport(CKANBasedImport): - def __init__(self, context, import_config, job_id): - super().__init__( - context, - import_config, - job_id, - # City of Toronto URL - "https://ckan0.cf.opendata.inter.prod-toronto.ca/api", - ) def iterate_imports(self): """ @@ -82,7 +74,7 @@ def iterate_imports(self): yield package - def map_to_cudc_package(self, src: dict): + def map_to_cudc_package(self, src: dict, target: dict): """ Map source package to cudc package. @@ -93,13 +85,6 @@ def map_to_cudc_package(self, src: dict): from datetime import datetime import re - # Default fields in CUDC - target = { - "owner_org": self.import_config.owner_org, - "type": "catalogue", - "license_id": "notspecified", - } - global package_mapping # One-to-one Mapping diff --git a/ckanext/udc_import_other_portals/logic/ckan_based/gov_of_canada.py b/ckanext/udc_import_other_portals/logic/ckan_based/gov_of_canada.py index 4c611e7..c198354 100644 --- a/ckanext/udc_import_other_portals/logic/ckan_based/gov_of_canada.py +++ b/ckanext/udc_import_other_portals/logic/ckan_based/gov_of_canada.py @@ -67,16 +67,8 @@ "climatology_meterology_atmosphere": "Climatology Meteorology Atmosphere", } - +# https://open.canada.ca/data/api/ class GovOfCanadaImport(CKANBasedImport): - def __init__(self, context, import_config, job_id): - super().__init__( - context, - import_config, - job_id, - # City of Toronto URL - "https://open.canada.ca/data/api", - ) def iterate_imports(self): """ @@ -85,18 +77,23 @@ def iterate_imports(self): for package in self.all_packages: yield package - def map_to_cudc_package(self, src: dict): + def map_to_cudc_package(self, src: dict, target: dict): """ Map source package to cudc package. Args: src (dict): The source package that needs to be mapped. """ - from ckanext.udc_import_other_portals.logic.base import ensure_license + from ckanext.udc_import_other_portals.logic.base import ( + ensure_license, + ensure_organization, + ) + from ckanext.udc_import_other_portals.logic.ckan_based.api import ( + get_organization, + ) import re # Default fields in CUDC - target = {"owner_org": self.import_config.owner_org, "type": "catalogue"} global package_mapping, subject_mapping, topic_mapping @@ -111,14 +108,12 @@ def map_to_cudc_package(self, src: dict): license_url = src.get("license_url") if license_id and license_title and license_url: - ensure_license( - self.build_context(), license_id, license_title, license_url - ) + ensure_license(self.build_context(), license_id, license_title, license_url) target["license_id"] = license_id # name target["name"] = "gov-canada-" + src["name"] - + # source target["url"] = f"https://open.canada.ca/data/en/dataset/{src['name']}" @@ -129,20 +124,20 @@ def map_to_cudc_package(self, src: dict): keywords_en = kw.get("en") or kw.get("en-t-fr") if isinstance(keywords_en, list) and len(keywords_en) > 0: tags = keywords_en - + # Add subject to tags if src.get("subject"): for subject in src["subject"]: if subject in subject_mapping: tags.append(subject_mapping[subject]) - - # Remove special characters from tags, + + # Remove special characters from tags, # can only contain alphanumeric characters, spaces (" "), hyphens ("-"), underscores ("_") or dots (".")' tags = [re.sub(r"[^a-zA-Z0-9 ._-]", "", tag) for tag in tags] # Remove tags that are longer than 100 characters tags = [tag for tag in tags if len(tag) <= 100] target["tags"] = [{"name": tag} for tag in tags] - + # topic -> theme theme = [] if src.get("topic_category"): @@ -163,17 +158,21 @@ def map_to_cudc_package(self, src: dict): # metadata_contact -> contact point (access_steward) metadata_contact = src.get("metadata_contact") or {} - metadata_contact_en = metadata_contact.get("en") or metadata_contact.get("en-t-fr") + metadata_contact_en = metadata_contact.get("en") or metadata_contact.get( + "en-t-fr" + ) if metadata_contact_en: # split and remove empty strings - metadata_contact_en = [x.strip() for x in metadata_contact_en.split(",") if x.strip()] - target["access_steward"] = ', '.join(metadata_contact_en) - + metadata_contact_en = [ + x.strip() for x in metadata_contact_en.split(",") if x.strip() + ] + target["access_steward"] = ", ".join(metadata_contact_en) + # DOI -> unique_identifier if src.get("digital_object_identifier"): target["unique_identifier"] = src.get("digital_object_identifier") target["global_unique_identifier"] = "Yes" - + # date_published -> published_date # "2019-07-23 17:53:27.345526" -> "2019-07-23" if src.get("date_published"): @@ -229,6 +228,7 @@ def test(self): example = {} all_subjects = set() all_topics = set() + all_orgs = dict() for src in self.all_packages: for key in src: if key not in example: @@ -245,10 +245,15 @@ def test(self): all_subjects.update(src[key]) if key == "topic_category": all_topics.update(src[key]) + print(src["id"], src["name"]) + if key == "owner_org": + if src[key] not in all_orgs: + all_orgs[src[key]] = src["organization"] # print("example", json.dumps(example, indent=2)) print(all_subjects) print(all_topics) + print(all_orgs) for src in self.all_packages: # diff --git a/ckanext/udc_react/ckan-udc-react/src/dashboard/drawerConfig.tsx b/ckanext/udc_react/ckan-udc-react/src/dashboard/drawerConfig.tsx index 6f16f11..9a420bc 100644 --- a/ckanext/udc_react/ckan-udc-react/src/dashboard/drawerConfig.tsx +++ b/ckanext/udc_react/ckan-udc-react/src/dashboard/drawerConfig.tsx @@ -1,5 +1,5 @@ import { CloudSync, Publish, Edit, OpenInBrowser, Settings } from '@mui/icons-material'; -import ImportDashboard from '../import/import'; +import ImportDashboard from '../import/ImportDashboard'; import { ImportStatus } from '../import/importStatus'; import QAPage from '../qa/QAPage'; import ConfigManagementPage from '../qa/ConfigManagementPage'; diff --git a/ckanext/udc_react/ckan-udc-react/src/import/ImportDashboard.tsx b/ckanext/udc_react/ckan-udc-react/src/import/ImportDashboard.tsx new file mode 100644 index 0000000..32a317f --- /dev/null +++ b/ckanext/udc_react/ckan-udc-react/src/import/ImportDashboard.tsx @@ -0,0 +1,40 @@ +import { Container } from '@mui/material'; +import DynamicTabs, { IDynamicTab } from './tabs'; +import { useEffect, useState } from 'react'; +import { useApi } from '../api/useApi'; +import { IImportConfig } from './types'; +import ImportPanel from './ImportPanel'; + +export default function ImportDashboard() { + const { api, executeApiCall } = useApi(); + const [tabs, setTabs] = useState([]); + + const load = async (option?: string) => { + // Get organizations + const organizations = await executeApiCall(api.getOrganizations); + + const importConfigs: IImportConfig = await executeApiCall(api.getImportConfigs); + const newTabs = []; + for (const [uuid, config] of Object.entries(importConfigs)) { + const { code, name } = config; + newTabs.push({ + key: uuid, label: name, panel: + }) + } + newTabs.push({ key: "new-import", label: "New Import", panel: }); + setTabs(newTabs); + } + const requestRefresh = () => { + load(); + } + + useEffect(() => { + load(); + }, []); + + return ( + + + + ); +} diff --git a/ckanext/udc_react/ckan-udc-react/src/import/import.tsx b/ckanext/udc_react/ckan-udc-react/src/import/ImportPanel.tsx similarity index 53% rename from ckanext/udc_react/ckan-udc-react/src/import/import.tsx rename to ckanext/udc_react/ckan-udc-react/src/import/ImportPanel.tsx index 36e5220..12cb598 100644 --- a/ckanext/udc_react/ckan-udc-react/src/import/import.tsx +++ b/ckanext/udc_react/ckan-udc-react/src/import/ImportPanel.tsx @@ -1,21 +1,12 @@ -import { Container, Paper, Box, InputLabel, FormControl, Button, Divider, Switch, TextField, Autocomplete, FormControlLabel, FormGroup } from '@mui/material'; +import { Paper, Box, InputLabel, FormControl, Button, Divider, Switch, TextField, Autocomplete, FormControlLabel, RadioGroup, Radio, CircularProgress } from '@mui/material'; import Grid from '@mui/material/Unstable_Grid2'; // Grid version 2 -import DynamicTabs, { IDynamicTab } from './tabs'; import CodeMirror from "@uiw/react-codemirror"; import { python } from '@codemirror/lang-python'; -import { BootstrapTextField } from './inputs'; -import { useEffect, useState } from 'react'; +import { useMemo, useState } from 'react'; import { SaveOutlined, PlayArrowOutlined, DeleteForeverOutlined } from '@mui/icons-material'; import { useApi } from '../api/useApi'; import { CKANOrganization } from '../api/api'; - -export interface IImportConfig { - [uuid: string]: { - uuid?: string - name: string; // The name of the import task - code: string; // Some code or identifier related to the import task (e.g., configuration or script code) - }; -} +import OrganizationMapper from './mapper/OrganizationMapper'; export interface ImportPanelProps { defaultConfig?: { @@ -25,7 +16,11 @@ export interface ImportPanelProps { notes?: string; owner_org?: string; stop_on_error?: boolean; - other_config?: object; + other_config?: { + org_import_mode?: string; + base_api?: string; + org_mapping?: { [k: string]: string }; + }; cron_schedule?: string; platform?: string; created_at?: string; @@ -36,11 +31,11 @@ export interface ImportPanelProps { } const supportedPlatform = [ - {id: "ckan", label: "CKAN"}, - {id: "socrata", label: "Socrata"}, + { id: "ckan", label: "CKAN" }, + { id: "socrata", label: "Socrata" }, ] -function ImportPanel(props: ImportPanelProps) { +export default function ImportPanel(props: ImportPanelProps) { const { api, executeApiCall } = useApi(); const [importConfig, setImportConfig] = useState({ @@ -52,8 +47,11 @@ function ImportPanel(props: ImportPanelProps) { stop_on_error: props.defaultConfig?.stop_on_error ?? false, cron_schedule: props.defaultConfig?.cron_schedule ?? "", platform: props.defaultConfig?.platform ?? "ckan", + other_config: props.defaultConfig?.other_config ?? {}, }); + const [loading, setLoading] = useState({ save: false, saveAndRun: false, delete: false }); + const handleChange = (field: string) => (e: any) => { setImportConfig(initials => ({ ...initials, @@ -90,15 +88,19 @@ function ImportPanel(props: ImportPanelProps) { } const handleSave = async () => { + setLoading(prev => ({ ...prev, save: true })); try { await executeApiCall(() => api.updateImportConfig(importConfig)); props.onUpdate(); } catch (e) { console.error(e) + } finally { + setLoading(prev => ({ ...prev, save: false })); } } const handleSaveAndRun = async () => { + setLoading(prev => ({ ...prev, saveAndRun: true })); try { const { result } = await executeApiCall(() => api.updateImportConfig(importConfig)); if (result?.id) { @@ -109,19 +111,40 @@ function ImportPanel(props: ImportPanelProps) { } catch (e) { console.error(e) + } finally { + setLoading(prev => ({ ...prev, saveAndRun: false })); } } const handleDelete = async () => { + setLoading(prev => ({ ...prev, delete: true })); try { if (importConfig.uuid) await executeApiCall(() => api.deleteImportConfig(importConfig.uuid!)) props.onUpdate(); } catch (e) { console.error(e) + } finally { + setLoading(prev => ({ ...prev, delete: false })); } } + + const handleChangeOtherConfig = (name: string) => (value: any) => { + setImportConfig(initials => ({ + ...initials, + other_config: { + ...initials.other_config, + [name]: value?.target instanceof Object ? value.target.value : value + } + })); + } + + const orgMappingComponent = useMemo(() => { + console.log(importConfig.other_config) + return + }, [importConfig.other_config.base_api]); + return ( @@ -133,15 +156,6 @@ function ImportPanel(props: ImportPanelProps) { helperText={importConfig.uuid && 'UUID: ' + importConfig.uuid} /> - - option.display_name} - value={props.organizations.find(org => org.id === importConfig.owner_org) || null} - onChange={handleChangeOrganization} - renderInput={(params) => } - /> - } /> + {importConfig.platform === 'ckan' && ( + <> + + + + + + + + } + label="Import everything to a specified organization" + /> + } + label="Import into its own organization, create if it does not exist, or map to an existing organization" + /> + + + + + {importConfig.other_config.org_import_mode === "importToSpecifiedOrg" && ( + + option.display_name} + value={props.organizations.find(org => org.id === importConfig.owner_org) || null} + onChange={handleChangeOrganization} + renderInput={(params) => } + /> + + )} + + {importConfig.other_config.org_import_mode === "importToOwnOrg" && ( + + {orgMappingComponent} + + )} + + )} + - + Python code snippets @@ -195,7 +261,7 @@ function ImportPanel(props: ImportPanelProps) { - + Stop on error @@ -208,18 +274,38 @@ function ImportPanel(props: ImportPanelProps) { - - {importConfig.uuid && - } @@ -227,36 +313,3 @@ function ImportPanel(props: ImportPanelProps) { ); } -export default function ImportDashboard() { - const { api, executeApiCall } = useApi(); - const [tabs, setTabs] = useState([]); - - const load = async (option?: string) => { - // Get organizations - const organizations = await executeApiCall(api.getOrganizations); - - const importConfigs: IImportConfig = await executeApiCall(api.getImportConfigs); - const newTabs = []; - for (const [uuid, config] of Object.entries(importConfigs)) { - const { code, name } = config; - newTabs.push({ - key: uuid, label: name, panel: - }) - } - newTabs.push({ key: "new-import", label: "New Import", panel: }); - setTabs(newTabs); - } - const requestRefresh = () => { - load(); - } - - useEffect(() => { - load(); - }, []); - - return ( - - - - ); -} diff --git a/ckanext/udc_react/ckan-udc-react/src/import/importStatus.tsx b/ckanext/udc_react/ckan-udc-react/src/import/importStatus.tsx index 7494ec5..990405c 100644 --- a/ckanext/udc_react/ckan-udc-react/src/import/importStatus.tsx +++ b/ckanext/udc_react/ckan-udc-react/src/import/importStatus.tsx @@ -4,7 +4,7 @@ import DynamicTabs, { IDynamicTab } from './tabs'; import CodeMirror from "@uiw/react-codemirror"; import { useEffect, useState } from 'react'; -import { IImportConfig } from './import'; +import { IImportConfig } from './types'; import DeleteIcon from '@mui/icons-material/Delete'; import { useApi } from '../api/useApi'; import { FinishedPackagesTable } from './realtime/FinishedPackagesTable'; diff --git a/ckanext/udc_react/ckan-udc-react/src/import/mapper/OrganizationMapper.tsx b/ckanext/udc_react/ckan-udc-react/src/import/mapper/OrganizationMapper.tsx new file mode 100644 index 0000000..2e40b18 --- /dev/null +++ b/ckanext/udc_react/ckan-udc-react/src/import/mapper/OrganizationMapper.tsx @@ -0,0 +1,240 @@ +import React, { useState, useEffect } from 'react'; +import { + Box, + Grid, + Typography, + Button, + List, + ListItem, + ListItemText, + Paper, + Modal, + IconButton, + Fade, + CircularProgress, + Autocomplete, + TextField, + Dialog, + DialogTitle, + DialogContent, + DialogActions, +} from '@mui/material'; +import EditIcon from '@mui/icons-material/Edit'; +import SearchIcon from '@mui/icons-material/Search'; +import DeleteIcon from '@mui/icons-material/Delete'; + +interface Organization { + id: string; + name: string; + description: string; +} + +interface OrganizationMapperProps { + externalBaseApi: string; + onChange: (mapping: { [k: string]: string }) => void; + defaultValue?: { [k: string]: string }; +} + +const OrganizationMapper: React.FC = ({ externalBaseApi, onChange, defaultValue }) => { + const [organizationsA, setOrganizationsA] = useState([]); + const [organizationsB, setOrganizationsB] = useState([]); + const [filteredOrganizationsB, setFilteredOrganizationsB] = useState([]); + const [searchTerm, setSearchTerm] = useState(''); + const [selectedExternalOrg, setSelectedExternalOrg] = useState(null); + const [modalOpen, setModalOpen] = useState(false); + const [mapping, setMapping] = useState<{ [k: string]: string }>(defaultValue || {}); + const [loading, setLoading] = useState(false); + + const fetchOrganizations = async (baseApi: string) => { + try { + setLoading(true); + const response = await fetch(`${baseApi}/3/action/organization_list?all_fields=true&limit=10000`); + const data = await response.json(); + return data.result + .filter((org: any) => org.package_count > 0) + .reduce((acc: { [key: string]: Organization }, org: any) => { + acc[org.id] = { + id: org.id, + name: org.title, + description: org.description, + }; + return acc; + }, {}); + } catch (error) { + console.error('Error fetching organizations:', error); + return {}; + } finally { + setLoading(false); + } + }; + + useEffect(() => { + const fetchData = async () => { + setLoading(true); + const fetchedOrganizationsA = await fetchOrganizations('/api'); + setOrganizationsA(Object.values(fetchedOrganizationsA)); + const fetchedOrganizationsB = await fetchOrganizations(externalBaseApi); + setOrganizationsB(Object.values(fetchedOrganizationsB)); + setFilteredOrganizationsB(Object.values(fetchedOrganizationsB)); + setLoading(false); + }; + fetchData(); + }, [externalBaseApi]); + + useEffect(() => { + setFilteredOrganizationsB( + organizationsB.filter((org) => + org.name.toLowerCase().includes(searchTerm.toLowerCase()) + ) + ); + }, [searchTerm, organizationsB]); + + const handleExternalOrgSelect = (org: Organization) => { + setSelectedExternalOrg(org); + }; + + const handleMap = (internalOrg: Organization | null) => { + if (internalOrg && selectedExternalOrg) { + setMapping((prevMapping) => { + const newMapping = { ...prevMapping, [selectedExternalOrg.id]: internalOrg.id }; + onChange(newMapping); + return newMapping; + }); + } else if (!internalOrg && selectedExternalOrg) { + setMapping((prevMapping) => { + const newMapping = { ...prevMapping }; + delete newMapping[selectedExternalOrg.id]; + onChange(newMapping); + return newMapping; + }); + } + setSelectedExternalOrg(null); + }; + + const handleDeleteMapping = (externalId: string) => { + setMapping((prevMapping) => { + const newMapping = { ...prevMapping }; + delete newMapping[externalId]; + onChange(newMapping); + return newMapping; + }); + }; + + const handleOpenModal = () => { + setModalOpen(true); + }; + + const handleCloseModal = () => { + setModalOpen(false); + }; + + return ( + + + + + {"Map Organizations"} + + + {loading ? ( + + + + ) : ( + + {/* External Organization List Panel */} + + + External Organizations + setSearchTerm(e.target.value)} + sx={{ mb: 2, mt: 2 }} + InputProps={{ + startAdornment: ( + + ), + }} + + /> + + {filteredOrganizationsB.map((org) => ( + handleExternalOrgSelect(org)} + > + + internalOrg.id === mapping[org.id])?.name || 'Unknown'}` : ''} + /> + {selectedExternalOrg?.id === org.id && ( + + internalOrg.id === mapping[org.id])} + options={organizationsA} + getOptionLabel={(option) => option?.name || ''} + onChange={(_, value) => handleMap(value)} + renderInput={(params) => ( + + )} + /> + + + )} + + + + + ))} + + + + + {/* Mapping Result Panel */} + + + Mapped Organizations + + {Object.entries(mapping).map(([externalId, internalId]) => { + const externalOrg = organizationsB.find((org) => org.id === externalId); + const internalOrg = organizationsA.find((org) => org.id === internalId); + + return ( + + + handleDeleteMapping(externalId)}> + + + + ); + })} + + + + + )} + + + + + + + ); +}; + +export default OrganizationMapper; diff --git a/ckanext/udc_react/ckan-udc-react/src/import/realtime/RealtimeImportPanel.tsx b/ckanext/udc_react/ckan-udc-react/src/import/realtime/RealtimeImportPanel.tsx index cc436d0..61eaaa1 100644 --- a/ckanext/udc_react/ckan-udc-react/src/import/realtime/RealtimeImportPanel.tsx +++ b/ckanext/udc_react/ckan-udc-react/src/import/realtime/RealtimeImportPanel.tsx @@ -143,6 +143,9 @@ export function RealtimeImportPanel(props: ImportPanelProps) { Progress + + {importLogs.length > 0 ? importLogs[0].message : ''} + diff --git a/ckanext/udc_react/ckan-udc-react/src/import/types.ts b/ckanext/udc_react/ckan-udc-react/src/import/types.ts new file mode 100644 index 0000000..a57d4b5 --- /dev/null +++ b/ckanext/udc_react/ckan-udc-react/src/import/types.ts @@ -0,0 +1,10 @@ +import { CKANOrganization } from '../api/api'; + + +export interface IImportConfig { + [uuid: string]: { + uuid?: string + name: string; // The name of the import task + code: string; // Some code or identifier related to the import task (e.g., configuration or script code) + }; +}