Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

add figurl support #39

Open
wants to merge 27 commits into
base: master
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
27 commits
Select commit Hold shift + click to select a range
abeb50c
figurl support
magland Feb 21, 2024
75d0ac0
Merge branch 'master' into figurl-support
edeno Feb 27, 2024
f1a4ed9
Add remfile
edeno Feb 27, 2024
13b8fdd
Fix missing units
edeno Feb 27, 2024
bae0e09
Add streaming conversion function
edeno Feb 27, 2024
69ad773
Merge branch 'master' into figurl-support
edeno Feb 27, 2024
2383cc6
Merge branch 'master' into figurl-support
edeno Feb 27, 2024
732f8b6
Rename function for consistency
edeno Feb 27, 2024
3292499
Add docstrings
edeno Feb 27, 2024
8988020
Add function that converts the directory of jsons to a single json
edeno Feb 27, 2024
4210c0b
Need trial_id as default factor
edeno Feb 27, 2024
e6be2df
Fix value id of factor
edeno Feb 27, 2024
c9b9889
Enable remove file
edeno Feb 27, 2024
400e9b1
Minor formatting
edeno Feb 27, 2024
82f852f
Add figurl creation script
edeno Feb 27, 2024
07597d2
Add remfile and kachery-cloud for figurl support
edeno Feb 27, 2024
5332648
Rename stop_time to end_time
edeno Feb 27, 2024
88bd0ca
Make create figurl function
edeno Feb 28, 2024
7754811
Fix error type
edeno Feb 28, 2024
04d653e
Fix error type
edeno Feb 28, 2024
7dfece2
Output to current directory
edeno Feb 28, 2024
f00e570
Use subject and session to identify
edeno Feb 28, 2024
f01f648
Example notebook
edeno Feb 28, 2024
0227349
Add trial start as time period
edeno Feb 28, 2024
3e18d87
Pass in time periods
edeno Feb 28, 2024
f8ab01e
Allow command line argument to be passed in
edeno Feb 28, 2024
faf2ec9
Discover trial events
edeno Feb 28, 2024
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions .gitignore
Original file line number Diff line number Diff line change
@@ -1,3 +1,5 @@
dist
package-lock.json
*.mat
*.json
!public/DATA/*HPa_01_04_*.json
Expand Down
140 changes: 140 additions & 0 deletions app/js/figurl-interface/deserializeReturnValue.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,140 @@
// eslint-disable-next-line @typescript-eslint/no-explicit-any
const deserializeReturnValue = (x) => {
if (!x) return x;
else if (typeof x === 'object') {
if (Array.isArray(x)) {
return x.map((a) => deserializeReturnValue(a));
} else if (x._type === 'ndarray') {
const shape = x.shape;
const dtype = x.dtype;
const data_b64 = x.data_b64;
const dataBuffer = _base64ToArrayBuffer(data_b64);
switch (dtype) {
case 'float32':
return applyShape(new Float32Array(dataBuffer), shape);
case 'int32':
return applyShape(new Int32Array(dataBuffer), shape);
case 'int16':
return applyShape(new Int16Array(dataBuffer), shape);
case 'int8':
return applyShape(new Int8Array(dataBuffer), shape);
case 'uint32':
return applyShape(new Uint32Array(dataBuffer), shape);
case 'uint16':
return applyShape(new Uint16Array(dataBuffer), shape);
case 'uint8':
return applyShape(new Uint8Array(dataBuffer), shape);
default:
throw Error(`Datatype not yet implemented for ndarray: ${dtype}`);
}
} else {
// eslint-disable-next-line @typescript-eslint/no-explicit-any
const ret = {};
for (const k in x) {
ret[k] = deserializeReturnValue(x[k]);
}
return ret;
}
} else return x;
};

const applyShape = (
x,
shape
) => {
if (shape.length === 1) {
if (shape[0] !== x.length) throw Error('Unexpected length of array');
return Array.from(x);
} else if (shape.length === 2) {
const n1 = shape[0];
const n2 = shape[1];
if (n1 * n2 !== x.length) throw Error('Unexpected length of array');
const ret = [];
for (let i1 = 0; i1 < n1; i1++) {
ret.push(Array.from(x.slice(i1 * n2, (i1 + 1) * n2)));
}
return ret;
} else if (shape.length === 3) {
const n1 = shape[0];
const n2 = shape[1];
const n3 = shape[2];
if (n1 * n2 * n3 !== x.length) throw Error('Unexpected length of array');
const ret = [];
for (let i1 = 0; i1 < n1; i1++) {
const A = [];
for (let i2 = 0; i2 < n2; i2++) {
A.push(Array.from(x.slice(i1 * n2 * n3 + i2 * n3, i1 * n2 * n3 + (i2 + 1) * n3)));
}
ret.push(A);
}
return ret;
} else if (shape.length === 4) {
const n1 = shape[0];
const n2 = shape[1];
const n3 = shape[2];
const n4 = shape[3];
if (n1 * n2 * n3 * n4 !== x.length) throw Error('Unexpected length of array');
const ret = [];
for (let i1 = 0; i1 < n1; i1++) {
const A = [];
for (let i2 = 0; i2 < n2; i2++) {
const B = [];
for (let i3 = 0; i3 < n3; i3++) {
B.push(
Array.from(
x.slice(i1 * n2 * n3 * n4 + i2 * n3 * n4 + i3 * n4, i1 * n2 * n3 * n4 + i2 * n3 * n4 + (i3 + 1) * n4)
)
);
}
A.push(B);
}
ret.push(A);
}
return ret;
} else if (shape.length === 5) {
const n1 = shape[0];
const n2 = shape[1];
const n3 = shape[2];
const n4 = shape[3];
const n5 = shape[4];
if (n1 * n2 * n3 * n4 * n5 !== x.length) throw Error('Unexpected length of array');
const request = [];
for (let i1 = 0; i1 < n1; i1++) {
const A = [];
for (let i2 = 0; i2 < n2; i2++) {
const B = [];
for (let i3 = 0; i3 < n3; i3++) {
const C = [];
for (let i4 = 0; i4 < n4; i4++) {
C.push(
Array.from(
x.slice(
i1 * n2 * n3 * n4 * n5 + i2 * n3 * n4 * n5 + i3 * n4 * n5 + i4 * n5,
i1 * n2 * n3 * n4 * n5 + i2 * n3 * n4 * n5 + i3 * n4 * n5 + (i4 + 1) * n5
)
)
);
}
B.push(C);
}
A.push(B);
}
ret.push(A);
}
return ret;
} else {
throw Error('Not yet implemented');
}
};

const _base64ToArrayBuffer = (base64) => {
const binary_string = window.atob(base64);
const len = binary_string.length;
const bytes = new Uint8Array(len);
for (let i = 0; i < len; i++) {
bytes[i] = binary_string.charCodeAt(i);
}
return bytes.buffer;
};

export default deserializeReturnValue;
18 changes: 18 additions & 0 deletions app/js/figurl-interface/getFigureData.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,18 @@
import sendRequestToParent from './sendRequestToParent';
import deserializeReturnValue from './deserializeReturnValue';

const getFigureData = () => {
return new Promise((resolve, reject) => {
const request = {
type: 'getFigureData',
figurlProtocolVersion: 'p1'
};
sendRequestToParent(request).then(response => {
resolve(deserializeReturnValue(response.figureData));
}).catch((err) => {
reject(err);
})
})
};

export default getFigureData;
146 changes: 146 additions & 0 deletions app/js/figurl-interface/getFileData.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,146 @@
import sendRequestToParent from './sendRequestToParent';
import deserializeReturnValue from './deserializeReturnValue';

const getFileData = (
uri,
responseType,
onProgress,
o = {}
) => {
let responseType2
if (responseType === 'json-deserialized') {
responseType2 = 'json'
}
else {
responseType2 = responseType
}
const request = {
type: 'getFileData',
uri,
responseType: responseType2,
figurlProtocolVersion: 'p1'
};
if (o.startByte !== undefined) {
request.startByte = o.startByte;
request.endByte = o.endByte;
}
progressListeners[uri] = ({ loaded, total }) => {
onProgress({ loaded, total });
};
return new Promise((resolve, reject) => {
sendRequestToParent(request).then(response => {
if (!isGetFileDataResponse(response)) throw Error('Invalid response to getFigureData');
if (response.errorMessage) {
throw Error(`Error getting file data for ${uri}: ${response.errorMessage}`);
}
if (responseType === 'json-deserialized') {
resolve(deserializeReturnValue(response.fileData));
}
else {
resolve(response.fileData);
}
})
.catch((err) => {
reject(err);
})
})
};

export const getFileDataUrl = (uri) => {
const request = {
type: 'getFileDataUrl',
uri,
figurlProtocolVersion: 'p1'
};
return new Promise((resolve, reject) => {
sendRequestToParent(request).then(response => {
if (!isGetFileDataUrlResponse(response)) throw Error('Invalid response to getFigureUrlData');
if (!response.fileDataUrl || response.errorMessage) {
throw Error(`Error getting file data for ${uri}: ${response.errorMessage}`);
}
resolve(response.fileDataUrl);
})
.catch((err) => {
reject(err);
})
})
}

// export const storeFileData = async (fileData, o = {}) => {
// const request = {
// type: 'storeFile',
// fileData,
// figurlProtocolVersion: 'p1'
// };
// const response = await sendRequestToParent(request);
// if (!isStoreFileResponse(response)) throw Error('Invalid response to storeFile');
// if (response.error) {
// throw Error(`Error storing file data: ${response.error}`);
// }
// if (response.uri === undefined) {
// throw Error('Unexpected response.uri is undefined');
// }
// return response.uri;
// };

// export const storeGithubFileData = async (o) => {
// const request = {
// type: 'storeGithubFile',
// fileData: o.fileData,
// uri: o.uri,
// figurlProtocolVersion: 'p1'
// };
// const response = await sendRequestToParent(request);
// if (!isStoreGithubFileResponse(response)) throw Error('Invalid response to storeFile');
// if (response.error) {
// throw Error(`Error storing file data: ${response.error}`);
// }
// };

const progressListeners = {};

export const handleFileDownloadProgress = ({
uri,
loaded,
total,
}) => {
const x = progressListeners[uri];
if (x) {
x({ loaded, total });
}
};

// export const useFileData = (uri, responseType, o = {}) => {
// // eslint-disable-next-line @typescript-eslint/no-explicit-any
// const [fileData, setFileData] = useState<any | undefined>(undefined);
// const [errorMessage, setErrorMessage] = useState<string | undefined>(undefined);
// const { progress, reportProgress } = useMemo(() => {
// let _callback = ({ loaded, total }) => {};
// const reportProgress = (a) => {
// _callback(a);
// };
// const progress = {
// onProgress: (callback) => {
// _callback = callback;
// },
// };
// return { progress, reportProgress };
// }, []);
// useEffect(() => {
// setErrorMessage(undefined);
// setFileData(undefined);
// getFileData(uri, responseType, reportProgress, {
// startByte: o.startByte,
// endByte: o.endByte
// })
// .then((data) => {
// setFileData(data);
// })
// .catch((err) => {
// setErrorMessage(err.message);
// });
// }, [uri, reportProgress, o.startByte, o.endByte, responseType]);
// return { fileData, progress, errorMessage };
// };

export default getFileData;
25 changes: 25 additions & 0 deletions app/js/figurl-interface/sendMessageToParent.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,25 @@
// const urlSearchParams = new URLSearchParams(window.location.search)
// const queryParams = Object.fromEntries(urlSearchParams.entries())

function parseQuery(queryString) {
const ind = queryString.indexOf('?');
if (ind < 0) return {};
const query = {};
const pairs = queryString.slice(ind + 1).split('&');
for (let i = 0; i < pairs.length; i++) {
const pair = pairs[i].split('=');
query[decodeURIComponent(pair[0])] = decodeURIComponent(pair[1] || '');
}
return query;
}

// Important to do it this way because it is difficult to handle special characters (especially #) by using URLSearchParams or window.location.search
const queryParams = parseQuery(window.location.href);

const sendMessageToParent = (x, { parentOrigin }) => {
const parentOrOpenerWindow = queryParams.useOpener === '1' ? window.opener : window.parent;
// if no parent, this will post to itself
parentOrOpenerWindow.postMessage(x, parentOrigin);
};

export default sendMessageToParent;
Loading