Change: Only embed specs/namespaces for types that are included in NWB file on export

+io/+internal/+h5/deleteAttribute.m

@@ -0,0 +1,20 @@
+function deleteAttribute(fileReference, objectLocation, attributeName)
+% deleteAttribute - Delete the specified attribute from an NWB file
+
+    arguments
+        fileReference {io.internal.h5.mustBeH5FileReference}
+        objectLocation (1,1) string
+        attributeName (1,1) string
+    end
+
+    objectLocation = io.internal.h5.validateLocation(objectLocation);    
+
+    % Open the HDF5 file in read-write mode
+    [fileId, fileCleanupObj] = io.internal.h5.resolveFileReference(fileReference, "w"); %#ok<ASGLU>
+
+    % Open the object (dataset or group)
+    [objectId, objectCleanupObj] = io.internal.h5.openObject(fileId, objectLocation); %#ok<ASGLU>
+
+    % Delete the attribute
+    H5A.delete(objectId, attributeName);
+end

+io/+internal/+h5/deleteGroup.m

@@ -0,0 +1,16 @@
+function deleteGroup(fileReference, groupLocation)
+% deleteGroup - Delete the specified group from an NWB file
+
+    arguments
+        fileReference {io.internal.h5.mustBeH5FileReference}
+        groupLocation (1,1) string
+    end
+
+    groupLocation = io.internal.h5.validateLocation(groupLocation);    
+
+    % Open the HDF5 file in read-write mode
+    [fileId, fileCleanupObj] = io.internal.h5.resolveFileReference(fileReference, "w"); %#ok<ASGLU>
+
+    % Delete the group
+    H5L.delete(fileId, groupLocation, 'H5P_DEFAULT');
+end

+io/+internal/+h5/listGroupNames.m

@@ -0,0 +1,28 @@
+function groupNames = listGroupNames(fileReference, h5Location)
+
+    arguments
+        fileReference {io.internal.h5.mustBeH5FileReference}
+        h5Location (1,1) string
+    end
+
+    [fileId, fileCleanupObj] = io.internal.h5.resolveFileReference(fileReference); %#ok<ASGLU>
+
+    % Open the specified location (group)
+    [groupId, groupCleanupObj] = io.internal.h5.openGroup(fileId, h5Location); %#ok<ASGLU>
+
+    % Use H5L.iterate to iterate over the links
+    [~, ~, groupNames] = H5L.iterate(...
+        groupId, "H5_INDEX_NAME", "H5_ITER_INC", 0, @collectGroupNames, {});
+
+    % Define iteration function
+    function [status, groupNames] = collectGroupNames(groupId, name, groupNames)
+        % Only retrieve name of groups
+        objId = H5O.open(groupId, name, 'H5P_DEFAULT');
+        objInfo = H5O.get_info(objId);
+        if objInfo.type == H5ML.get_constant_value('H5O_TYPE_GROUP')
+            groupNames{end+1} = name;
+        end
+        H5O.close(objId);
+        status = 0; % Continue iteration
+    end
+end

+io/+internal/+h5/mustBeH5File.m

@@ -0,0 +1,17 @@
+function mustBeH5File(value)
+    arguments
+        value {mustBeFile}
+    end
+
+    VALID_FILE_ENDING = ["h5", "nwb"];
+    validExtensions = "." + VALID_FILE_ENDING;
+
+    hasH5Extension = endsWith(value, validExtensions, 'IgnoreCase', true);    
+
+    if ~hasH5Extension
+        exception = MException(...
+            'MatNWB:validators:mustBeH5File', ...
+            'Expected file "%s" to have .h5 or .nwb file extension', value);
+        throwAsCaller(exception)
+    end
+end

+io/+internal/+h5/mustBeH5FileReference.m

@@ -0,0 +1,15 @@
+function mustBeH5FileReference(value)
+    arguments
+        value {mustBeA(value, ["char", "string", "H5ML.id"])}
+    end
+
+    if isa(value, "char") || isa(value, "string")
+        try
+            io.internal.h5.mustBeH5File(value)
+        catch ME
+            throwAsCaller(ME)
+        end
+    else
+        % value is a H5ML.id, ok!
+    end
+end

+io/+internal/+h5/openFile.m

@@ -0,0 +1,46 @@
+function [fileId, fileCleanupObj] = openFile(fileName, permission)
+% openFile Opens an HDF5 file with the specified permissions and ensures cleanup.
+%
+%   [fileId, fileCleanupObj] = io.internal.h5.openFile(fileName) opens the HDF5 
+%   file specified by fileName in read-only mode ('r') by default.
+%
+%   [fileId, fileCleanupObj] = io.internal.h5.openFile(fileName, permission) 
+%   opens the HDF5 file specified by fileName with the access mode defined by 
+%   permission. 
+%
+%   Input Arguments:
+%       fileName   - A string or character vector specifying the path to the 
+%                    HDF5 file. This must be a .h5 or .nwb file.
+%
+%       permission - (Optional) A scalar string specifying the file access mode.
+%                    Valid values are "r" for read-only (default) and "w" for 
+%                    read-write.
+%
+%   Output Arguments:
+%       fileId         - The file identifier returned by H5F.open, used to 
+%                        reference the open file.
+%
+%       fileCleanupObj - A cleanup object (onCleanup) that ensures the file is 
+%                        closed automatically when fileCleanupObj goes out of 
+%                        scope.
+%
+%   Example:
+%       [fid, cleanupObj] = io.internal.h5.openFile("data.h5", "w");
+%       % Use fid for file operations.
+%       % When cleanupObj is cleared or goes out of scope, the file is 
+%       % automatically closed.
+
+    arguments
+        fileName {io.internal.h5.mustBeH5File}
+        permission (1,1) string {mustBeMember(permission, ["r", "w"])} = "r"
+    end
+
+    switch permission
+        case "r"
+            accessFlag = 'H5F_ACC_RDONLY';
+        case "w"
+            accessFlag = 'H5F_ACC_RDWR';
+    end
+    fileId = H5F.open(fileName, accessFlag, 'H5P_DEFAULT');
+    fileCleanupObj = onCleanup(@(fid) H5F.close(fileId));
+end

+io/+internal/+h5/openGroup.m

@@ -0,0 +1,13 @@
+function [groupId, groupCleanupObj] = openGroup(fileId, h5Location)
+% openGroup Opens an HDF5 group at given location and ensures cleanup.
+
+    arguments
+        fileId {mustBeA(fileId, "H5ML.id")}
+        h5Location (1,1) string
+    end
+
+    % Open the specified location (group)
+    groupLocation = io.internal.h5.validateLocation(h5Location);    
+    groupId = H5G.open(fileId, groupLocation);
+    groupCleanupObj = onCleanup(@(gid) H5G.close(groupId));
+end

+io/+internal/+h5/openObject.m

@@ -0,0 +1,13 @@
+function [objectId, objectCleanupObj] = openObject(fileId, objectLocation)
+% openObject Opens an HDF5 object at given location and ensures cleanup.
+
+    arguments
+        fileId {mustBeA(fileId, "H5ML.id")}
+        objectLocation (1,1) string
+    end
+
+    % Open the object (dataset or group)
+    objectLocation = io.internal.h5.validateLocation(objectLocation);
+    objectId = H5O.open(fileId, objectLocation, 'H5P_DEFAULT');
+    objectCleanupObj = onCleanup(@(oid) H5O.close(objectId));
+end

+io/+internal/+h5/resolveFileReference.m

@@ -0,0 +1,30 @@
+function [h5FileId, fileCleanupObj] = resolveFileReference(fileReference, permission)
+% resolveFileReference - Resolve a file reference to a H5 File ID.
+%
+% Utility method to resolve a file reference, which can be either a
+% filepath or a file id for a h5 file.
+% 
+% The returned value will always be a file ID. This allows functions that
+% does operations on h5 files to receive either a file path or a file id
+%
+% Note: If the file reference is a file ID for an open file, the permission 
+% might be different than the provided/requested permission. 
+
+    arguments
+        fileReference {io.internal.h5.mustBeH5FileReference}
+        permission (1,1) string {mustBeMember(permission, ["r", "w"])} = "r"
+    end
+
+    if isa(fileReference, "char") || isa(fileReference, "string")
+        % Need to open the file
+        if isfile(fileReference)
+            [h5FileId, fileCleanupObj] = io.internal.h5.openFile(fileReference, permission);
+        else
+            error('File "%s" does not exist', fileReference)
+        end
+    else
+        h5FileId = fileReference;
+        % If the file is already open, we are not responsible for closing it
+        fileCleanupObj = [];
+    end
+end

+io/+internal/+h5/validateLocation.m

@@ -0,0 +1,9 @@
+function locationName = validateLocation(locationName)
+    arguments
+        locationName (1,1) string
+    end
+
+    if ~startsWith(locationName, "/")
+        locationName = "/" + locationName;
+    end
+end

+io/+spec/listEmbeddedSpecNamespaces.m

@@ -0,0 +1,11 @@
+function namespaceNames = listEmbeddedSpecNamespaces(fileReference)
+
+    arguments
+        fileReference {io.internal.h5.mustBeH5FileReference}
+    end
+
+    [fileId, fileCleanupObj] = io.internal.h5.resolveFileReference(fileReference); %#ok<ASGLU>
+
+    specLocation = io.spec.internal.readEmbeddedSpecLocation(fileId);
+    namespaceNames = io.internal.h5.listGroupNames(fileId, specLocation);
+end

+io/+spec/validateEmbeddedSpecifications.m

@@ -0,0 +1,48 @@
+function validateEmbeddedSpecifications(h5_file_id, expectedNamespaceNames)
+% validateEmbeddedSpecifications - Validate the embedded specifications
+%
+% This function does two things:
+%   1) Displays a warning if specifications of expected namespaces
+%      are not embedded in the file. 
+%        E.g if cached namespaces were cleared prior to export.
+%   
+%   2) Deletes specifications for unused namespaces that are embedded. 
+%      - E.g. If neurodata type from an embedded namespace was removed and the
+%        file was re-exported
+
+% NB: Input h5_file_id must point to a file opened with write access
+
+    specLocation = io.spec.internal.readEmbeddedSpecLocation(h5_file_id);
+    embeddedNamespaceNames = io.internal.h5.listGroupNames(h5_file_id, specLocation);
+
+    checkMissingNamespaces(expectedNamespaceNames, embeddedNamespaceNames)
+
+    unusedNamespaces = checkUnusedNamespaces(...
+        expectedNamespaceNames, embeddedNamespaceNames);
+
+    if ~isempty(unusedNamespaces)
+        deleteUnusedNamespaces(h5_file_id, unusedNamespaces, specLocation)
+    end
+end
+
+function checkMissingNamespaces(expectedNamespaceNames, embeddedNamespaceNames)
+% checkMissingNamespaces - Check if any namespace specs are missing from the file
+    missingNamespaces = setdiff(expectedNamespaceNames, embeddedNamespaceNames);
+    if ~isempty(missingNamespaces)
+        missingNamespacesStr = strjoin("  " + string(missingNamespaces), newline);
+        warning('NWB:validators:MissingEmbeddedNamespace', 'Namespace is missing:\n%s', missingNamespacesStr)
+    end
+end
+
+function unusedNamespaces = checkUnusedNamespaces(expectedNamespaceNames, embeddedNamespaceNames)
+% checkUnusedNamespaces - Check if any namespace specs in the file are unused
+    unusedNamespaces = setdiff(embeddedNamespaceNames, expectedNamespaceNames);
+end
+
+function deleteUnusedNamespaces(fileId, unusedNamespaces, specRootLocation)
+    for i = 1:numel(unusedNamespaces)
+        thisName = unusedNamespaces{i};
+        namespaceSpecLocation = strjoin( {specRootLocation, thisName}, '/');
+        io.internal.h5.deleteGroup(fileId, namespaceSpecLocation)
+    end
+end

+io/+spec/writeEmbeddedSpecifications.m

@@ -1,4 +1,11 @@
 function writeEmbeddedSpecifications(fid, jsonSpecs)
+% writeEmbeddedSpecifications - Write schema specifications to an NWB file
+
+    arguments
+        fid         % File id for a h5 file
+        jsonSpecs   % String representation of schema specifications in json format
+    end
+
     specLocation = io.spec.internal.readEmbeddedSpecLocation(fid);
 
     if isempty(specLocation)
@@ -37,8 +44,8 @@ function writeEmbeddedSpecifications(fid, jsonSpecs)
 function versionNames = getVersionNames(namespaceGroupId)
     [~, ~, versionNames] = H5L.iterate(namespaceGroupId,...
         'H5_INDEX_NAME', 'H5_ITER_NATIVE',...
-        0, @removeGroups, {});
-    function [status, versionNames] = removeGroups(~, name, versionNames)
+        0, @appendName, {});
+    function [status, versionNames] = appendName(~, name, versionNames)
         versionNames{end+1} = name;
         status = 0;
     end

+matnwb/+extension/+internal/buildRepoDownloadUrl.m

@@ -0,0 +1,25 @@
+function downloadUrl = buildRepoDownloadUrl(repositoryUrl, branchName)
+% buildRepoDownloadUrl - Build a download URL for a given repository and branch
+    arguments
+        repositoryUrl (1,1) string
+        branchName (1,1) string
+    end
+
+    if endsWith(repositoryUrl, '/')
+        repositoryUrl = extractBefore(repositoryUrl, strlength(repositoryUrl));
+    end
+
+    if contains(repositoryUrl, 'github.com')
+        downloadUrl = sprintf( '%s/archive/refs/heads/%s.zip', repositoryUrl, branchName );
+
+    elseif contains(repositoryUrl, 'gitlab.com')
+        repoPathSegments = strsplit(repositoryUrl, '/');
+        repoName = repoPathSegments{end};
+        downloadUrl = sprintf( '%s/-/archive/%s/%s-%s.zip', ...
+            repositoryUrl, branchName, repoName, branchName);
+
+    else
+        error('NWB:BuildRepoDownloadUrl:UnsupportedRepository', ...
+            'Expected repository URL to point to a GitHub or a GitLab repository')
+    end
+end

+matnwb/+extension/+internal/downloadExtensionRepository.m

@@ -0,0 +1,46 @@
+function [wasDownloaded, repoTargetFolder] = downloadExtensionRepository(...
+    repositoryUrl, repoTargetFolder, extensionName)
+% downloadExtensionRepository - Download the repository (source) for an extension
+%
+%   The metadata for a neurodata extension only provides the url to the
+%   repository containing the extension, not the full download url. This
+%   function tries to download a zipped version of the repository from
+%   either the "main" or the "master" branch.
+%
+%   Works for repositories located on GitHub or GitLab
+%
+%   As of Dec. 2024, this approach works for all registered extensions
+
+    arguments
+        repositoryUrl (1,1) string
+        repoTargetFolder (1,1) string
+        extensionName (1,1) string
+    end
+
+    import matnwb.extension.internal.downloadZippedRepo
+    import matnwb.extension.internal.buildRepoDownloadUrl
+
+    defaultBranchNames = ["main", "master"];
+
+    wasDownloaded = false;
+    for i = 1:2
+        try
+            branchName = defaultBranchNames(i);
+            downloadUrl = buildRepoDownloadUrl(repositoryUrl, branchName);
+            repoTargetFolder = downloadZippedRepo(downloadUrl, repoTargetFolder);
+            wasDownloaded = true;
+            break
+        catch ME
+            if strcmp(ME.identifier, 'MATLAB:webservices:HTTP404StatusCodeError')
+                continue
+            elseif strcmp(ME.identifier, 'NWB:BuildRepoDownloadUrl:UnsupportedRepository')
+                error('NWB:InstallExtension:UnsupportedRepository', ...
+                    ['Extension "%s" is located in an unsupported repository ', ...
+                     '/ source location. \nPlease create an issue on MatNWB''s ', ...
+                     'github page'], extensionName)
+            else
+                rethrow(ME)
+            end
+        end
+    end
+end