From 67cb50e3fb154b90ef22abb923613087faca78e3 Mon Sep 17 00:00:00 2001 From: elainefan331 Date: Thu, 7 May 2026 11:16:06 -0400 Subject: [PATCH 01/29] feat: add search tables migration and update db config for local postgres --- backend/config/config.js | 13 +- .../20260507145253-create-search-tables.js | 130 ++++++++++++++++++ 2 files changed, 141 insertions(+), 2 deletions(-) create mode 100644 backend/migrations/20260507145253-create-search-tables.js diff --git a/backend/config/config.js b/backend/config/config.js index 7c15ed9..d117965 100644 --- a/backend/config/config.js +++ b/backend/config/config.js @@ -1,9 +1,18 @@ require("dotenv").config(); module.exports = { + // development: { + // dialect: "sqlite", + // storage: "./database.sqlite", + // logging: console.log, + // }, development: { - dialect: "sqlite", - storage: "./database.sqlite", + dialect: "postgres", + host: "localhost", + port: 5432, + database: "neurojson_dev", + username: process.env.DB_USER_LOCAL, + password: process.env.DB_PASSWORD_LOCAL, logging: console.log, }, test: { diff --git a/backend/migrations/20260507145253-create-search-tables.js b/backend/migrations/20260507145253-create-search-tables.js new file mode 100644 index 0000000..d2c9deb --- /dev/null +++ b/backend/migrations/20260507145253-create-search-tables.js @@ -0,0 +1,130 @@ +"use strict"; + +/** @type {import('sequelize-cli').Migration} */ +module.exports = { + async up(queryInterface, Sequelize) { + // ioviews table + await queryInterface.createTable("ioviews", { + id: { + type: Sequelize.INTEGER, + autoIncrement: true, + primaryKey: true, + allowNull: false, + }, + dbname: { + type: Sequelize.STRING(30), + allowNull: true, + }, + dsname: { + type: Sequelize.STRING(30), + allowNull: true, + }, + subj: { + type: Sequelize.STRING(12), + allowNull: true, + }, + view: { + type: Sequelize.STRING(12), + allowNull: true, + }, + json: { + type: Sequelize.JSONB, + allowNull: true, + }, + search_vector: { + type: Sequelize.DataTypes.TSVECTOR, + allowNull: true, + }, + updated_at: { + type: Sequelize.DATE, + allowNull: false, + defaultValue: Sequelize.literal("CURRENT_TIMESTAMP"), + }, + }); + + // ioviews indexes + await queryInterface.addIndex("ioviews", ["view"], { + name: "idx_ioviews_view", + }); + await queryInterface.addIndex("ioviews", ["dbname"], { + name: "idx_ioviews_dbname", + }); + await queryInterface.addIndex("ioviews", ["updated_at"], { + name: "idx_ioviews_updated_at", + }); + + // GIN indexes need raw query (not supported by addIndex) + await queryInterface.sequelize.query(` + CREATE INDEX IF NOT EXISTS idx_ioviews_search + ON ioviews USING GIN(search_vector); + CREATE INDEX IF NOT EXISTS idx_ioviews_json + ON ioviews USING GIN(json); + `); + + // iolinks table + await queryInterface.createTable("iolinks", { + id: { + type: Sequelize.INTEGER, + autoIncrement: true, + primaryKey: true, + allowNull: false, + }, + dbname: { + type: Sequelize.STRING(30), + allowNull: true, + }, + dsname: { + type: Sequelize.STRING(30), + allowNull: true, + }, + subj: { + type: Sequelize.TEXT, + allowNull: true, + }, + view: { + type: Sequelize.TEXT, + allowNull: true, + }, + json: { + type: Sequelize.JSONB, + allowNull: true, + }, + }); + + // iolinks indexes + await queryInterface.addIndex("iolinks", ["view"], { + name: "idx_iolinks_view", + }); + await queryInterface.addIndex("iolinks", ["dbname"], { + name: "idx_iolinks_dbname", + }); + await queryInterface.sequelize.query(` + CREATE INDEX IF NOT EXISTS idx_iolinks_json + ON iolinks USING GIN(json); + `); + + // sync_state table + await queryInterface.createTable("sync_state", { + dbname: { + type: Sequelize.STRING(30), + primaryKey: true, + allowNull: false, + }, + last_seq: { + type: Sequelize.TEXT, + allowNull: true, + }, + synced_at: { + type: Sequelize.DATE, + allowNull: false, + defaultValue: Sequelize.literal("CURRENT_TIMESTAMP"), + }, + }); + }, + + async down(queryInterface, Sequelize) { + await queryInterface.dropTable("ioviews"); + await queryInterface.dropTable("iolinks"); + await queryInterface.dropTable("sync_state"); + }, +}; From 7432b3539892f99b174061815b80894c13605bcc Mon Sep 17 00:00:00 2001 From: elainefan331 Date: Fri, 8 May 2026 10:54:52 -0400 Subject: [PATCH 02/29] feat: add incremental sync script to populate ioviews and iolinks from CouchDB --- backend/package.json | 3 +- backend/sync/incrementalSync.js | 263 ++++++++++++++++++++++++++++++++ 2 files changed, 265 insertions(+), 1 deletion(-) create mode 100644 backend/sync/incrementalSync.js diff --git a/backend/package.json b/backend/package.json index 1dde6de..fa156b3 100644 --- a/backend/package.json +++ b/backend/package.json @@ -11,7 +11,8 @@ "migrate:undo:all": "npx sequelize-cli db:migrate:undo:all", "seed": "npx sequelize-cli db:seed:all", "seed:undo": "npx sequelize-cli db:seed:undo:all", - "db:reset": "npx sequelize-cli db:migrate:undo:all && npx sequelize-cli db:migrate" + "db:reset": "npx sequelize-cli db:migrate:undo:all && npx sequelize-cli db:migrate", + "sync": "node sync/incrementalSync.js" }, "keywords": [ "express", diff --git a/backend/sync/incrementalSync.js b/backend/sync/incrementalSync.js new file mode 100644 index 0000000..b885fc7 --- /dev/null +++ b/backend/sync/incrementalSync.js @@ -0,0 +1,263 @@ +"use strict"; + +require("dotenv").config(); +const axios = require("axios"); +const { sequelize } = require("../src/config/database"); + +const COUCHDB_URL = process.env.COUCHDB_URL || "https://neurojson.io:7777"; + +// fetch database list dynamically from registry +async function getDatabases() { + try { + const response = await axios.get(`${COUCHDB_URL}/sys/registry`); + const databases = response.data + .map((db) => db.id) + .filter((id) => id && id !== "sys"); + console.log(`Found ${databases.length} databases in registry`); + return databases; + } catch (err) { + console.error("Failed to fetch registry:", err.message); + // fallback to hardcoded list if registry fails + return [ + "openneuro", + "abide", + "abide2", + "datalad-registry", + "adhd200", + "bfnirs", + "mcx", + "mmc", + "ucl-4d-neonatal-head-model", + "unc-012-infant-atlas", + "unc-infant-cortical-surface-atlas", + "cotilab", + "emnist", + "nemo-bids", + "openfnirs", + ]; + } +} + +// get last synced sequence number for a database +async function getLastSeq(dbname) { + try { + const result = await sequelize.query( + "SELECT last_seq FROM sync_state WHERE dbname = :dbname", + { + replacements: { dbname }, + type: sequelize.QueryTypes.SELECT, + } + ); + return result[0]?.last_seq || "0"; + } catch (err) { + console.error(`Error getting last_seq for ${dbname}:`, err.message); + return "0"; + } +} + +// save latest sequence number after sync +async function saveLastSeq(dbname, seq) { + await sequelize.query( + `INSERT INTO sync_state (dbname, last_seq, synced_at) + VALUES (:dbname, :seq, NOW()) + ON CONFLICT (dbname) DO UPDATE + SET last_seq = :seq, synced_at = NOW()`, + { replacements: { dbname, seq } } + ); +} + +// upsert a row into ioviews +async function upsertIoview(dbname, dsname, subj, view, json) { + await sequelize.query( + `INSERT INTO ioviews (dbname, dsname, subj, view, json, search_vector, updated_at) + VALUES (:dbname, :dsname, :subj, :view, :json, to_tsvector('english', :text), NOW()) + ON CONFLICT (dbname, dsname, subj, view) DO UPDATE + SET json = :json, + search_vector = to_tsvector('english', :text), + updated_at = NOW()`, + { + replacements: { + dbname, + dsname, + subj: String(subj), + view, + json: JSON.stringify(json), + text: JSON.stringify(json), + }, + } + ); +} + +// insert a row into iolinks +async function insertIolink(dbname, dsname, subj, view, json) { + await sequelize.query( + `INSERT INTO iolinks (dbname, dsname, subj, view, json) + VALUES (:dbname, :dsname, :subj, :view, :json)`, + { + replacements: { + dbname, + dsname, + subj: String(subj), + view, + json: JSON.stringify(json), + }, + } + ); +} + +// delete all records for a dataset +async function deleteDataset(dbname, dsname) { + await sequelize.query( + "DELETE FROM ioviews WHERE dbname = :dbname AND dsname = :dsname", + { replacements: { dbname, dsname } } + ); + await sequelize.query( + "DELETE FROM iolinks WHERE dbname = :dbname AND dsname = :dsname", + { replacements: { dbname, dsname } } + ); + console.log(` Deleted ${dbname}/${dsname}`); +} + +// first time sync - fetch from CouchDB views directly +async function firstSync(dbname) { + console.log(` ${dbname}: first sync, fetching all views...`); + + // fetch dbinfo view + const dbinfoRes = await axios.get( + `${COUCHDB_URL}/${dbname}/_design/qq/_view/dbinfo` + ); + const dbinfoRows = dbinfoRes.data.rows || []; + for (const row of dbinfoRows) { + const subj = String(row.value?.subj?.length || 0); + await upsertIoview(dbname, row.id, subj, "dbinfo", row.value); + } + console.log(` ${dbname}: dbinfo synced (${dbinfoRows.length} rows)`); + + // fetch subjects view + const subjectsRes = await axios.get( + `${COUCHDB_URL}/${dbname}/_design/qq/_view/subjects` + ); + const subjectRows = subjectsRes.data.rows || []; + for (const row of subjectRows) { + const subj = String(row.key?.[6] || ""); + await upsertIoview(dbname, row.id, subj, "subjects", { + key: row.key, + value: row.value, + }); + } + console.log(` ${dbname}: subjects synced (${subjectRows.length} rows)`); + + // fetch links view + const linksRes = await axios.get( + `${COUCHDB_URL}/${dbname}/_design/qq/_view/links` + ); + const linkRows = linksRes.data.rows || []; + for (const row of linkRows) { + const fileType = row.key?.[0]; + const subjId = String(row.key?.[1] || ""); + await insertIolink(dbname, row.id, subjId, fileType, { + key: row.key, + value: row.value, + }); + } + console.log(` ${dbname}: links synced (${linkRows.length} rows)`); +} + +// incremental sync - only fetch changes since last sync +async function incrementalSync(dbname, lastSeq) { + const { data } = await axios.get( + `${COUCHDB_URL}/${dbname}/_changes?since=${lastSeq}&include_docs=true` + ); + + if (data.results.length === 0) { + console.log(` ${dbname}: no changes since last sync`); + return data.last_seq; + } + + console.log(` ${dbname}: ${data.results.length} changes found`); + + for (const change of data.results) { + if (change.deleted) { + await deleteDataset(dbname, change.id); + continue; + } + + const doc = change.doc; + if (!doc?.value) continue; + + // upsert dbinfo + if (doc.value.subj && Array.isArray(doc.value.subj)) { + const subj = String(doc.value.subj.length); + await upsertIoview(dbname, change.id, subj, "dbinfo", doc.value); + } + + // upsert subjects + if (doc.value.subjects) { + for (const [subjId, subjData] of Object.entries(doc.value.subjects)) { + await upsertIoview(dbname, change.id, subjId, "subjects", { + key: subjData.key, + value: subjData.value, + }); + } + } + + // upsert links + if (doc.value.links) { + for (const link of doc.value.links) { + const fileType = link.key?.[0]; + const subjId = String(link.key?.[1] || ""); + await insertIolink(dbname, change.id, subjId, fileType, link); + } + } + } + + return data.last_seq; +} + +// sync a single database +async function syncDatabase(dbname) { + console.log(`\nSyncing ${dbname}...`); + const lastSeq = await getLastSeq(dbname); + + try { + if (lastSeq === "0") { + await firstSync(dbname); + } else { + await incrementalSync(dbname, lastSeq); + } + + // get and save the latest seq number + const { data: info } = await axios.get(`${COUCHDB_URL}/${dbname}`); + await saveLastSeq(dbname, String(info.update_seq)); + console.log(` ${dbname}: sync complete ✓`); + } catch (err) { + console.error(` ${dbname}: sync failed - ${err.message}`); + } +} + +// main function +async function runSync() { + console.log("=== Starting NeuroJSON sync ==="); + console.log(new Date().toISOString()); + console.log(`CouchDB: ${COUCHDB_URL}`); + console.log(`Databases: ${DATABASES.length}`); + + // change to getDatabases() when ready for full sync + // const databases = await getDatabases(); + const databases = ["bfnirs"]; // testing with small database first + + console.log(`Databases: ${databases.length}`); + + for (const db of databases) { + await syncDatabase(db); + } + + await sequelize.close(); + console.log("\n=== Sync complete ==="); + console.log(new Date().toISOString()); +} + +runSync().catch((err) => { + console.error("Sync failed:", err); + process.exit(1); +}); From f686418b5b76d1adabde2e55419dd6dafd2ce11b Mon Sep 17 00:00:00 2001 From: elainefan331 Date: Fri, 8 May 2026 15:47:06 -0400 Subject: [PATCH 03/29] feat: refactor incremental sync with transactions, concurrency, and edge case guards --- backend/sync/incrementalSync.js | 469 +++++++++++++++++++++++++++----- 1 file changed, 402 insertions(+), 67 deletions(-) diff --git a/backend/sync/incrementalSync.js b/backend/sync/incrementalSync.js index b885fc7..5a73c42 100644 --- a/backend/sync/incrementalSync.js +++ b/backend/sync/incrementalSync.js @@ -5,6 +5,7 @@ const axios = require("axios"); const { sequelize } = require("../src/config/database"); const COUCHDB_URL = process.env.COUCHDB_URL || "https://neurojson.io:7777"; +const CONCURRENCY = 5; // fetch database list dynamically from registry async function getDatabases() { @@ -17,7 +18,6 @@ async function getDatabases() { return databases; } catch (err) { console.error("Failed to fetch registry:", err.message); - // fallback to hardcoded list if registry fails return [ "openneuro", "abide", @@ -38,15 +38,275 @@ async function getDatabases() { } } -// get last synced sequence number for a database +// === Local ports of CouchDB _design/qq map functions === +// 1:1 ports of dbinfo / subjects / links views. If upstream views change, +// these drift silently. + +function transformDbinfo(doc) { + const txt = + doc["README"] || doc["README.md"] || doc["README.rst"] || ""; + const rawtext = JSON.stringify(doc); + const datainfo = doc["dataset_description.json"] || { Name: doc._id }; + const subjlist = []; + const modalitylist = []; + + for (const item of Object.keys(doc)) { + if (item.indexOf("ub-") !== -1) { + subjlist.push(item); + for (const modal of Object.keys(doc[item] || {})) { + if (modal.indexOf("ses") === 0) { + for (const m of Object.keys(doc[item][modal] || {})) { + if (m.indexOf(".") === -1 && modalitylist.indexOf(m) === -1) { + modalitylist.push(m); + } + } + } else if ( + modal.indexOf(".") === -1 && + modalitylist.indexOf(modal) === -1 + ) { + modalitylist.push(modal); + } + } + } + } + + if (subjlist.length === 0) subjlist.push("nonbids"); + + if (modalitylist.length === 0) { + if (rawtext.indexOf('"MeshNode"') !== -1) modalitylist.push("JMesh"); + if (rawtext.indexOf('"NIFTIData"') !== -1) modalitylist.push("JNIFTI"); + if (rawtext.indexOf('"SNIRFData"') !== -1) modalitylist.push("JSNIRF"); + if (rawtext.indexOf('"_ArrayType_"') !== -1) modalitylist.push("JData"); + } + + return { + name: datainfo.Name, + length: rawtext.length, + readme: String(txt).substr(0, 256), + info: datainfo, + subj: subjlist, + modality: modalitylist, + }; +} + +function transformSubjects(doc) { + const results = []; + const skipIds = ["sidecards", "derivatives", "sourcedata", "code"]; + if (skipIds.indexOf(doc._id) !== -1) return results; + + for (const subj of Object.keys(doc)) { + if (!/^[sS]ub-/.test(subj)) continue; + + const sessionlist = []; + const modalitylist = []; + const tasklist = []; + const runlist = []; + const filetype = []; + let age = -0.01; + let gender = "N"; + + const p = doc["participants.tsv"]; + if (p && Array.isArray(p.participant_id)) { + let idx = -1; + for (let i = 0; i < p.participant_id.length; i++) { + if (subj.indexOf(String(p.participant_id[i])) > -1) { + idx = i; + break; + } + } + + if (idx >= 0) { + for (const agekey of ["age", "age_scan", "age_at_scan"]) { + if (age >= 0) break; + if (p[agekey]) { + age = p[agekey][idx]; + break; + } else if (p[agekey.toUpperCase()]) { + age = p[agekey.toUpperCase()][idx]; + break; + } else { + const cap = agekey.charAt(0).toUpperCase() + agekey.slice(1); + if (p[cap]) { + age = p[cap]; // matches upstream view (drops [idx] here) + break; + } + } + } + if (age < 0) { + for (const pfield of Object.keys(p)) { + if (pfield.toLowerCase().indexOf("age") >= 0) { + age = p[pfield][idx]; + } + } + } + for (const sexkey of ["sex", "gender"]) { + if (gender !== "N") break; + if (p[sexkey]) { + gender = p[sexkey][idx]; + break; + } else if (p[sexkey.toUpperCase()]) { + gender = p[sexkey.toUpperCase()][idx]; + break; + } else { + const cap = sexkey.charAt(0).toUpperCase() + sexkey.slice(1); + if (p[cap]) { + gender = p[cap]; // matches upstream view (drops [idx] here) + break; + } + } + } + if (gender === "N") { + for (const pfield of Object.keys(p)) { + if (pfield.toLowerCase().indexOf("sex") >= 0) { + gender = p[pfield][idx]; + } + } + } + if (gender === "N") { + for (const pfield of Object.keys(p)) { + if (pfield.toLowerCase().indexOf("gender") >= 0) { + gender = p[pfield][idx]; + } + } + } + } + } + + const subjDoc = doc[subj] || {}; + const parseFiles = (container) => { + for (const filename of Object.keys(container || {})) { + for (const task of filename.split("_")) { + if (task.indexOf("run-") === 0) { + if (runlist.indexOf(task.substring(4)) === -1) { + runlist.push(task.substring(4)); + } + } else if (task.indexOf("task-") === 0) { + if (tasklist.indexOf(task.substring(5)) === -1) { + tasklist.push(task.substring(5)); + } + } else if (task.indexOf(".") > 0) { + const tmp = task.substring(0, task.indexOf(".")); + if (filetype.indexOf(tmp) === -1) filetype.push(tmp); + } + } + } + }; + + for (const modal of Object.keys(subjDoc)) { + if (modal.indexOf("ses-") === 0) { + if (sessionlist.indexOf(modal.substring(4)) === -1) { + sessionlist.push(modal.substring(4)); + } + for (const modname of Object.keys(subjDoc[modal] || {})) { + if ( + modname.indexOf(".") === -1 && + modalitylist.indexOf(modname) === -1 + ) { + modalitylist.push(modname); + } + parseFiles(subjDoc[modal][modname]); + } + } else if ( + modal.indexOf(".") === -1 && + modalitylist.indexOf(modal) === -1 + ) { + modalitylist.push(modal); + parseFiles(subjDoc[modal]); + } + } + + if (typeof gender === "string") { + gender = gender.substring(0, 1).toUpperCase(); + } else { + gender = gender + ""; + } + if (typeof age === "string" && isNaN(+age)) age = -0.001; + if (typeof age === "string") age = +age; + if (age < 0) age = -0.01; + age = Math.floor(age * 100); + + results.push({ + id: doc._id, + key: [ + ("0000" + age).slice(-5), + ("000" + gender).slice(-4), + ("000" + sessionlist.length).slice(-4), + ("000" + modalitylist.length).slice(-4), + ("000" + tasklist.length).slice(-4), + ("000" + runlist.length).slice(-4), + subj.substring(4), + ], + value: { + sessions: sessionlist, + modalities: modalitylist, + tasks: tasklist, + runs: runlist, + types: filetype, + }, + }); + } + + return results; +} + +function transformLinks(doc) { + const results = []; + const filenameRe = /file=([^\/]*\/)*([^&\/\.]+)(\.[^.&%:]+(\.gz)*)([&:].*)*$/; + const filesizeRe = /size=(\d+)/; + const jsonpathRe = /:(\$[^&]+)/; + const urlhash = {}; + + function traverse(obj, level, rootpath) { + if (level > 10) return; + if (obj === null || typeof obj !== "object") return; + + for (const subkey of Object.keys(obj)) { + const v = obj[subkey]; + if ( + subkey === "_DataLink_" && + typeof v === "string" && + v.indexOf("http") !== -1 + ) { + const url = v; + const uniqurl = url.split(":$")[0]; + if (!Object.prototype.hasOwnProperty.call(urlhash, uniqurl)) { + const fname = url.match(filenameRe); + const fsize = url.match(filesizeRe); + let jpath = url.match(jsonpathRe); + if (jpath !== null && jpath.length) jpath = jpath[1]; + urlhash[uniqurl] = 1; + if (fname && fsize) { + results.push({ + id: doc._id, + key: [fname[3], parseInt(fsize[1], 10)], + value: { + path: rootpath, + url: uniqurl, + file: fname[2] + fname[3], + suffix: fname[3], + ref: jpath, + }, + }); + } + } + } + if (typeof v === "object" && v !== null) { + traverse(v, level + 1, rootpath + "." + subkey); + } + } + } + + traverse(doc, 1, "$"); + return results; +} + +// === DB helpers (each accepts an optional transaction) === + async function getLastSeq(dbname) { try { const result = await sequelize.query( "SELECT last_seq FROM sync_state WHERE dbname = :dbname", - { - replacements: { dbname }, - type: sequelize.QueryTypes.SELECT, - } + { replacements: { dbname }, type: sequelize.QueryTypes.SELECT } ); return result[0]?.last_seq || "0"; } catch (err) { @@ -55,19 +315,17 @@ async function getLastSeq(dbname) { } } -// save latest sequence number after sync async function saveLastSeq(dbname, seq) { await sequelize.query( `INSERT INTO sync_state (dbname, last_seq, synced_at) VALUES (:dbname, :seq, NOW()) ON CONFLICT (dbname) DO UPDATE SET last_seq = :seq, synced_at = NOW()`, - { replacements: { dbname, seq } } + { replacements: { dbname, seq: String(seq) } } ); } -// upsert a row into ioviews -async function upsertIoview(dbname, dsname, subj, view, json) { +async function upsertIoview(dbname, dsname, subj, view, json, transaction) { await sequelize.query( `INSERT INTO ioviews (dbname, dsname, subj, view, json, search_vector, updated_at) VALUES (:dbname, :dsname, :subj, :view, :json, to_tsvector('english', :text), NOW()) @@ -84,12 +342,12 @@ async function upsertIoview(dbname, dsname, subj, view, json) { json: JSON.stringify(json), text: JSON.stringify(json), }, + transaction, } ); } -// insert a row into iolinks -async function insertIolink(dbname, dsname, subj, view, json) { +async function insertIolink(dbname, dsname, subj, view, json, transaction) { await sequelize.query( `INSERT INTO iolinks (dbname, dsname, subj, view, json) VALUES (:dbname, :dsname, :subj, :view, :json)`, @@ -101,28 +359,27 @@ async function insertIolink(dbname, dsname, subj, view, json) { view, json: JSON.stringify(json), }, + transaction, } ); } -// delete all records for a dataset -async function deleteDataset(dbname, dsname) { +async function deleteDataset(dbname, dsname, transaction) { await sequelize.query( "DELETE FROM ioviews WHERE dbname = :dbname AND dsname = :dsname", - { replacements: { dbname, dsname } } + { replacements: { dbname, dsname }, transaction } ); await sequelize.query( "DELETE FROM iolinks WHERE dbname = :dbname AND dsname = :dsname", - { replacements: { dbname, dsname } } + { replacements: { dbname, dsname }, transaction } ); - console.log(` Deleted ${dbname}/${dsname}`); } -// first time sync - fetch from CouchDB views directly +// === First-time sync (fetch all three views once) === + async function firstSync(dbname) { console.log(` ${dbname}: first sync, fetching all views...`); - // fetch dbinfo view const dbinfoRes = await axios.get( `${COUCHDB_URL}/${dbname}/_design/qq/_view/dbinfo` ); @@ -133,7 +390,6 @@ async function firstSync(dbname) { } console.log(` ${dbname}: dbinfo synced (${dbinfoRows.length} rows)`); - // fetch subjects view const subjectsRes = await axios.get( `${COUCHDB_URL}/${dbname}/_design/qq/_view/subjects` ); @@ -147,7 +403,6 @@ async function firstSync(dbname) { } console.log(` ${dbname}: subjects synced (${subjectRows.length} rows)`); - // fetch links view const linksRes = await axios.get( `${COUCHDB_URL}/${dbname}/_design/qq/_view/links` ); @@ -163,89 +418,169 @@ async function firstSync(dbname) { console.log(` ${dbname}: links synced (${linkRows.length} rows)`); } -// incremental sync - only fetch changes since last sync -async function incrementalSync(dbname, lastSeq) { - const { data } = await axios.get( - `${COUCHDB_URL}/${dbname}/_changes?since=${lastSeq}&include_docs=true` - ); +// === Process one changed dataset (Option A: 2 HTTP requests + local transforms) === - if (data.results.length === 0) { - console.log(` ${dbname}: no changes since last sync`); - return data.last_seq; +async function processDatasetUpdate(dbname, dsname) { + // dbinfo view supports key filtering; raw doc carries everything else. + const keyParam = encodeURIComponent(JSON.stringify(dsname)); + const [dbinfoRes, rawDocRes] = await Promise.all([ + axios.get( + `${COUCHDB_URL}/${dbname}/_design/qq/_view/dbinfo?key=${keyParam}` + ), + axios.get(`${COUCHDB_URL}/${dbname}/${encodeURIComponent(dsname)}`), + ]); + + const dbinfoRow = (dbinfoRes.data.rows || [])[0]; + if (!dbinfoRow) { + console.warn(` ${dbname}/${dsname}: no dbinfo row, skipping`); + return; } + const dbinfoValue = dbinfoRow.value; + const doc = rawDocRes.data; - console.log(` ${dbname}: ${data.results.length} changes found`); + const subjectRows = transformSubjects(doc); + const linkRows = transformLinks(doc); - for (const change of data.results) { - if (change.deleted) { - await deleteDataset(dbname, change.id); - continue; - } + // Rule 1: wrap all writes for this dataset in one transaction. + await sequelize.transaction(async (t) => { + const subjCount = String(dbinfoValue?.subj?.length || 0); + await upsertIoview(dbname, dsname, subjCount, "dbinfo", dbinfoValue, t); - const doc = change.doc; - if (!doc?.value) continue; - - // upsert dbinfo - if (doc.value.subj && Array.isArray(doc.value.subj)) { - const subj = String(doc.value.subj.length); - await upsertIoview(dbname, change.id, subj, "dbinfo", doc.value); + // Rule 2: empty-subjs guard. NOT IN (NULL) silently matches nothing. + const currentSubjs = Array.isArray(dbinfoValue?.subj) + ? dbinfoValue.subj + : []; + if (currentSubjs.length > 0) { + // subjects view stores subj without "sub-"/"Sub-" prefix + // (key[6] = subj.substring(4) in upstream map). + const currentSubjIds = currentSubjs.map((s) => s.substring(4)); + await sequelize.query( + `DELETE FROM ioviews + WHERE dbname = :dbname AND dsname = :dsname AND view = 'subjects' + AND subj NOT IN (:subjs)`, + { + replacements: { dbname, dsname, subjs: currentSubjIds }, + transaction: t, + } + ); } - // upsert subjects - if (doc.value.subjects) { - for (const [subjId, subjData] of Object.entries(doc.value.subjects)) { - await upsertIoview(dbname, change.id, subjId, "subjects", { - key: subjData.key, - value: subjData.value, - }); - } + for (const row of subjectRows) { + const subj = String(row.key?.[6] || ""); + await upsertIoview( + dbname, + dsname, + subj, + "subjects", + { key: row.key, value: row.value }, + t + ); } - // upsert links - if (doc.value.links) { - for (const link of doc.value.links) { - const fileType = link.key?.[0]; - const subjId = String(link.key?.[1] || ""); - await insertIolink(dbname, change.id, subjId, fileType, link); - } + // iolinks: no usable upsert key, so delete + reinsert per dataset. + await sequelize.query( + "DELETE FROM iolinks WHERE dbname = :dbname AND dsname = :dsname", + { replacements: { dbname, dsname }, transaction: t } + ); + for (const row of linkRows) { + const fileType = row.key?.[0]; + const subjId = String(row.key?.[1] || ""); + await insertIolink( + dbname, + dsname, + subjId, + fileType, + { key: row.key, value: row.value }, + t + ); } + }); +} + +// === Incremental sync === + +async function incrementalSync(dbname, lastSeq) { + // No include_docs=true: we fetch the raw doc per dataset so the _changes + // payload stays small and per-dataset work runs in parallel. + const { data } = await axios.get( + `${COUCHDB_URL}/${dbname}/_changes?since=${encodeURIComponent(lastSeq)}` + ); + + if (!data.results || data.results.length === 0) { + console.log(` ${dbname}: no changes since last sync`); + return data.last_seq; } + const changes = data.results.filter( + (c) => c.id && !c.id.startsWith("_design/") + ); + console.log( + ` ${dbname}: ${changes.length} dataset changes (raw=${data.results.length})` + ); + + // Rule 3: bounded concurrency + per-dataset try/catch. + for (let i = 0; i < changes.length; i += CONCURRENCY) { + const chunk = changes.slice(i, i + CONCURRENCY); + await Promise.all( + chunk.map(async (change) => { + try { + if (change.deleted) { + await sequelize.transaction((t) => + deleteDataset(dbname, change.id, t) + ); + console.log(` ${dbname}/${change.id}: deleted`); + } else { + await processDatasetUpdate(dbname, change.id); + } + } catch (err) { + console.error( + ` ${dbname}/${change.id}: failed - ${err.message}` + ); + } + }) + ); + } + + // Rule 4: return last_seq from THIS response. Never re-fetch update_seq + // afterward (writes during sync would be silently skipped). return data.last_seq; } -// sync a single database +// === Sync a single database === + async function syncDatabase(dbname) { console.log(`\nSyncing ${dbname}...`); const lastSeq = await getLastSeq(dbname); try { + let nextSeq; if (lastSeq === "0") { + // Rule 5: capture update_seq BEFORE firstSync. Writes during firstSync + // get picked up by the next incremental run. + const { data: info } = await axios.get(`${COUCHDB_URL}/${dbname}`); + const seqAtStart = String(info.update_seq); await firstSync(dbname); + nextSeq = seqAtStart; } else { - await incrementalSync(dbname, lastSeq); + nextSeq = await incrementalSync(dbname, lastSeq); } - // get and save the latest seq number - const { data: info } = await axios.get(`${COUCHDB_URL}/${dbname}`); - await saveLastSeq(dbname, String(info.update_seq)); + await saveLastSeq(dbname, String(nextSeq)); console.log(` ${dbname}: sync complete ✓`); } catch (err) { console.error(` ${dbname}: sync failed - ${err.message}`); } } -// main function +// === Main === + async function runSync() { console.log("=== Starting NeuroJSON sync ==="); console.log(new Date().toISOString()); console.log(`CouchDB: ${COUCHDB_URL}`); - console.log(`Databases: ${DATABASES.length}`); - // change to getDatabases() when ready for full sync - // const databases = await getDatabases(); + // change to await getDatabases() when ready for full sync const databases = ["bfnirs"]; // testing with small database first - console.log(`Databases: ${databases.length}`); for (const db of databases) { From 9d69223b51c297bdde62abc409c6860e5601d258 Mon Sep 17 00:00:00 2001 From: elainefan331 Date: Fri, 8 May 2026 16:57:45 -0400 Subject: [PATCH 04/29] feat: add ioviews unique constraint migration and disable sequelize logging in dev --- backend/config/config.js | 2 +- ...508195500-add-ioviews-unique-constraint.js | 20 +++++++++++++++++++ 2 files changed, 21 insertions(+), 1 deletion(-) create mode 100644 backend/migrations/20260508195500-add-ioviews-unique-constraint.js diff --git a/backend/config/config.js b/backend/config/config.js index d117965..0e7d92a 100644 --- a/backend/config/config.js +++ b/backend/config/config.js @@ -13,7 +13,7 @@ module.exports = { database: "neurojson_dev", username: process.env.DB_USER_LOCAL, password: process.env.DB_PASSWORD_LOCAL, - logging: console.log, + logging: false, }, test: { dialect: "sqlite", diff --git a/backend/migrations/20260508195500-add-ioviews-unique-constraint.js b/backend/migrations/20260508195500-add-ioviews-unique-constraint.js new file mode 100644 index 0000000..757397d --- /dev/null +++ b/backend/migrations/20260508195500-add-ioviews-unique-constraint.js @@ -0,0 +1,20 @@ +"use strict"; + +/** @type {import('sequelize-cli').Migration} */ +module.exports = { + async up(queryInterface, Sequelize) { + // Required by upsertIoview's ON CONFLICT (dbname, dsname, subj, view). + await queryInterface.addConstraint("ioviews", { + fields: ["dbname", "dsname", "subj", "view"], + type: "unique", + name: "ioviews_dbname_dsname_subj_view_unique", + }); + }, + + async down(queryInterface, Sequelize) { + await queryInterface.removeConstraint( + "ioviews", + "ioviews_dbname_dsname_subj_view_unique" + ); + }, +}; From 627e59845b982d8fc2d9ea85df507ed410fdcd73 Mon Sep 17 00:00:00 2001 From: elainefan331 Date: Mon, 11 May 2026 10:55:01 -0400 Subject: [PATCH 05/29] feat: replace CGI search with PostgreSQL query in searchAllDatabases controller --- backend/src/controllers/couchdb.controller.js | 269 ++++++++++++++---- backend/sync/incrementalSync.js | 7 +- 2 files changed, 218 insertions(+), 58 deletions(-) diff --git a/backend/src/controllers/couchdb.controller.js b/backend/src/controllers/couchdb.controller.js index e913ee3..f6c54c5 100644 --- a/backend/src/controllers/couchdb.controller.js +++ b/backend/src/controllers/couchdb.controller.js @@ -1,4 +1,5 @@ const axios = require("axios"); +const { sequelize } = require("../config/database"); // const COUCHDB_BASE_URL = // process.env.COUCHDB_BASE_URL || // "https://cors.redoc.ly/https://neurojson.io:7777"; @@ -38,72 +39,226 @@ const getDbStats = async (req, res) => { } }; -// cross-database search +// cross-database search — old version proxied to https://neurojson.org/io/search.cgi +// kept for reference; replaced by the Postgres-backed version below. +// const searchAllDatabases = async (req, res) => { +// try { +// const formData = req.body; +// const map = { +// keyword: "keyword", +// age_min: "agemin", +// age_max: "agemax", +// task_min: "taskmin", +// task_max: "taskmax", +// run_min: "runmin", +// run_max: "runmax", +// sess_min: "sessmin", +// sess_max: "sessmax", +// modality: "modality", +// run_name: "run", +// type_name: "type", +// session_name: "session", +// task_name: "task", +// limit: "limit", +// skip: "skip", +// count: "count", +// unique: "unique", +// gender: "gender", +// database: "dbname", +// dataset: "dsname", +// subject: "subname", +// }; +// +// const params = new URLSearchParams(); +// params.append("_get", "dbname, dsname, json"); +// +// Object.keys(formData).forEach((key) => { +// let val = formData[key]; +// if (val === "" || val === "any" || val === undefined || val === null) { +// return; +// } +// +// const queryKey = map[key]; +// if (!queryKey) return; +// +// if (key.startsWith("age")) { +// params.append(queryKey, String(Math.floor(val * 100)).padStart(5, "0")); +// } else if (key === "gender") { +// params.append(queryKey, val[0]); +// } else if (key === "modality") { +// params.append(queryKey, val.replace(/.*\(/, "").replace(/\).*/, "")); +// } else { +// params.append(queryKey, val.toString()); +// } +// }); +// +// const queryString = `?${params.toString()}`; +// const response = await axios.get( +// `https://cors.redoc.ly/https://neurojson.org/io/search.cgi${queryString}`, +// { +// headers: { +// Origin: "https://neurojson.io", +// "X-Requested-With": "XMLHttpRequest", +// }, +// } +// ); +// res.status(200).json(response.data); +// } catch (error) { +// console.error("Error searching all databases:", error.message); +// res.status(error.response?.status || 500).json({ +// message: "Error searching databases", +// error: error.message, +// }); +// } +// }; + +// helpers for the Postgres-backed search +function isFilter(v) { + return v !== "" && v !== "any" && v !== undefined && v !== null; +} +function pad4(n) { + return String(n).padStart(4, "0"); +} +function pad5(n) { + return String(n).padStart(5, "0"); +} + +// cross-database search — Postgres-backed (queries ioviews) const searchAllDatabases = async (req, res) => { try { - const formData = req.body; - const map = { - keyword: "keyword", - age_min: "agemin", - age_max: "agemax", - task_min: "taskmin", - task_max: "taskmax", - run_min: "runmin", - run_max: "runmax", - sess_min: "sessmin", - sess_max: "sessmax", - modality: "modality", - run_name: "run", - type_name: "type", - session_name: "session", - task_name: "task", - limit: "limit", - skip: "skip", - count: "count", - unique: "unique", - gender: "gender", - database: "dbname", - dataset: "dsname", - subject: "subname", - }; + const f = req.body || {}; + const where = []; + const repl = {}; - const params = new URLSearchParams(); - params.append("_get", "dbname, dsname, json"); + // Pick which view to search. + // Subject-level filters → subjects view; otherwise dbinfo. + const subjectFilterKeys = [ + "age_min", "age_max", "gender", + "task_min", "task_max", "task_name", + "run_min", "run_max", "run_name", + "sess_min", "sess_max", "session_name", + "type_name", "modality", "subject", + ]; + const isSubjectSearch = subjectFilterKeys.some((k) => isFilter(f[k])); + where.push(`view = :view`); + repl.view = isSubjectSearch ? "subjects" : "dbinfo"; - Object.keys(formData).forEach((key) => { - let val = formData[key]; - if (val === "" || val === "any" || val === undefined || val === null) { - return; - } + // Range filters compare against zero-padded key components. + // json->'key' = [age, gender, sess, mod, task, run, subjId] + if (isFilter(f.age_min)) { + where.push(`(json->'key'->>0) >= :ageMin`); + repl.ageMin = pad5(Math.floor(Number(f.age_min) * 100)); + } + if (isFilter(f.age_max)) { + where.push(`(json->'key'->>0) <= :ageMax`); + repl.ageMax = pad5(Math.floor(Number(f.age_max) * 100)); + } + if (isFilter(f.sess_min)) { + where.push(`(json->'key'->>2) >= :sessMin`); + repl.sessMin = pad4(f.sess_min); + } + if (isFilter(f.sess_max)) { + where.push(`(json->'key'->>2) <= :sessMax`); + repl.sessMax = pad4(f.sess_max); + } + if (isFilter(f.task_min)) { + where.push(`(json->'key'->>4) >= :taskMin`); + repl.taskMin = pad4(f.task_min); + } + if (isFilter(f.task_max)) { + where.push(`(json->'key'->>4) <= :taskMax`); + repl.taskMax = pad4(f.task_max); + } + if (isFilter(f.run_min)) { + where.push(`(json->'key'->>5) >= :runMin`); + repl.runMin = pad4(f.run_min); + } + if (isFilter(f.run_max)) { + where.push(`(json->'key'->>5) <= :runMax`); + repl.runMax = pad4(f.run_max); + } + if (isFilter(f.gender)) { + // stored as one upper-case char left-padded to 4 chars + where.push(`(json->'key'->>1) LIKE :gender`); + repl.gender = `%${String(f.gender)[0].toUpperCase()}`; + } - const queryKey = map[key]; - if (!queryKey) return; - - if (key.startsWith("age")) { - params.append(queryKey, String(Math.floor(val * 100)).padStart(5, "0")); - } else if (key === "gender") { - params.append(queryKey, val[0]); - } else if (key === "modality") { - params.append(queryKey, val.replace(/.*\(/, "").replace(/\).*/, "")); - } else { - params.append(queryKey, val.toString()); - } + // Name filters — jsonb ? checks if a string is an element of the array. + if (isFilter(f.task_name)) { + where.push(`json->'value'->'tasks' ? :taskName`); + repl.taskName = String(f.task_name); + } + if (isFilter(f.run_name)) { + where.push(`json->'value'->'runs' ? :runName`); + repl.runName = String(f.run_name); + } + if (isFilter(f.session_name)) { + where.push(`json->'value'->'sessions' ? :sessName`); + repl.sessName = String(f.session_name); + } + if (isFilter(f.type_name)) { + where.push(`json->'value'->'types' ? :typeName`); + repl.typeName = String(f.type_name); + } + if (isFilter(f.modality)) { + // form sometimes wraps as "fNIRS (nirs)" — pull text inside parens + const mod = String(f.modality).replace(/.*\(/, "").replace(/\).*/, ""); + where.push(`json->'value'->'modalities' ? :modality`); + repl.modality = mod; + } + + // db / ds / subj filters + if (isFilter(f.database)) { + where.push(`dbname = :dbname`); + repl.dbname = String(f.database); + } + if (isFilter(f.dataset)) { + where.push(`dsname = :dsname`); + repl.dsname = String(f.dataset); + } + if (isFilter(f.subject)) { + where.push(`subj = :subj`); + repl.subj = String(f.subject); + } + + // Keyword full-text search + if (isFilter(f.keyword)) { + where.push(`search_vector @@ websearch_to_tsquery('english', :keyword)`); + repl.keyword = String(f.keyword); + } + + const limit = Math.min(parseInt(f.limit) || 100, 1000); + const offset = parseInt(f.skip) || 0; + repl.limit = limit; + repl.offset = offset; + + // dbinfo was stored flat ({name, subj, ...}); subjects was stored wrapped + // ({key, value}). Frontend expects parsed.value.subj for datasets, so we + // wrap dbinfo on the way out. + const sql = ` + SELECT + dbname, + dsname, + subj, + CASE + WHEN view = 'dbinfo' THEN jsonb_build_object('value', json)::text + ELSE json::text + END AS json + FROM ioviews + WHERE ${where.join(" AND ")} + ORDER BY dbname, dsname, subj + LIMIT :limit OFFSET :offset + `; + + const rows = await sequelize.query(sql, { + replacements: repl, + type: sequelize.QueryTypes.SELECT, }); - const queryString = `?${params.toString()}`; - const response = await axios.get( - `https://cors.redoc.ly/https://neurojson.org/io/search.cgi${queryString}`, - { - headers: { - Origin: "https://neurojson.io", - "X-Requested-With": "XMLHttpRequest", - }, - } - ); - res.status(200).json(response.data); + res.status(200).json(rows); } catch (error) { console.error("Error searching all databases:", error.message); - res.status(error.response?.status || 500).json({ + res.status(500).json({ message: "Error searching databases", error: error.message, }); diff --git a/backend/sync/incrementalSync.js b/backend/sync/incrementalSync.js index 5a73c42..c6ab7c7 100644 --- a/backend/sync/incrementalSync.js +++ b/backend/sync/incrementalSync.js @@ -580,7 +580,12 @@ async function runSync() { console.log(`CouchDB: ${COUCHDB_URL}`); // change to await getDatabases() when ready for full sync - const databases = ["bfnirs"]; // testing with small database first + const databases = [ + "bfnirs", // NIRS — .snirf, .jdb + "brainmeshlibrary", // mesh + atlas — .jmsh, .jnii (318 datasets) + "cotilab", // JData — small (6 datasets) + "abide", // BIDS MRI — .nii.gz, .tsv, .json (25 datasets) + ]; console.log(`Databases: ${databases.length}`); for (const db of databases) { From 207c6340728045f5853fc6ce457cc68473927c9c Mon Sep 17 00:00:00 2001 From: elainefan331 Date: Mon, 11 May 2026 11:25:31 -0400 Subject: [PATCH 06/29] fix: widen ioviews and iolinks text columns to handle longer values --- ...5900-widen-ioviews-iolinks-text-columns.js | 60 +++++++++++++++++++ backend/sync/incrementalSync.js | 32 ++++++---- 2 files changed, 80 insertions(+), 12 deletions(-) create mode 100644 backend/migrations/20260511145900-widen-ioviews-iolinks-text-columns.js diff --git a/backend/migrations/20260511145900-widen-ioviews-iolinks-text-columns.js b/backend/migrations/20260511145900-widen-ioviews-iolinks-text-columns.js new file mode 100644 index 0000000..c6cbe81 --- /dev/null +++ b/backend/migrations/20260511145900-widen-ioviews-iolinks-text-columns.js @@ -0,0 +1,60 @@ +"use strict"; + +/** @type {import('sequelize-cli').Migration} */ +module.exports = { + async up(queryInterface, Sequelize) { + // VARCHAR(n) → TEXT is a metadata-only change in Postgres (no table rewrite, + // no need to drop the unique constraint or indexes). + await queryInterface.changeColumn("ioviews", "dbname", { + type: Sequelize.TEXT, + allowNull: true, + }); + await queryInterface.changeColumn("ioviews", "dsname", { + type: Sequelize.TEXT, + allowNull: true, + }); + await queryInterface.changeColumn("ioviews", "subj", { + type: Sequelize.TEXT, + allowNull: true, + }); + await queryInterface.changeColumn("iolinks", "dbname", { + type: Sequelize.TEXT, + allowNull: true, + }); + await queryInterface.changeColumn("iolinks", "dsname", { + type: Sequelize.TEXT, + allowNull: true, + }); + await queryInterface.changeColumn("sync_state", "dbname", { + type: Sequelize.TEXT, + allowNull: false, + }); + }, + + async down(queryInterface, Sequelize) { + await queryInterface.changeColumn("ioviews", "dbname", { + type: Sequelize.STRING(30), + allowNull: true, + }); + await queryInterface.changeColumn("ioviews", "dsname", { + type: Sequelize.STRING(30), + allowNull: true, + }); + await queryInterface.changeColumn("ioviews", "subj", { + type: Sequelize.STRING(12), + allowNull: true, + }); + await queryInterface.changeColumn("iolinks", "dbname", { + type: Sequelize.STRING(30), + allowNull: true, + }); + await queryInterface.changeColumn("iolinks", "dsname", { + type: Sequelize.STRING(30), + allowNull: true, + }); + await queryInterface.changeColumn("sync_state", "dbname", { + type: Sequelize.STRING(30), + allowNull: false, + }); + }, +}; diff --git a/backend/sync/incrementalSync.js b/backend/sync/incrementalSync.js index c6ab7c7..b9e497a 100644 --- a/backend/sync/incrementalSync.js +++ b/backend/sync/incrementalSync.js @@ -377,23 +377,34 @@ async function deleteDataset(dbname, dsname, transaction) { // === First-time sync (fetch all three views once) === +// Fetch a view, treating 404 as "view doesn't exist on this DB" (returns []). +// Non-BIDS DBs (e.g. brainmeshlibrary) only have the dbinfo view. +async function fetchView(dbname, viewName) { + try { + const res = await axios.get( + `${COUCHDB_URL}/${dbname}/_design/qq/_view/${viewName}` + ); + return res.data.rows || []; + } catch (err) { + if (err.response?.status === 404) { + console.log(` ${dbname}: view '${viewName}' not present, skipping`); + return []; + } + throw err; + } +} + async function firstSync(dbname) { console.log(` ${dbname}: first sync, fetching all views...`); - const dbinfoRes = await axios.get( - `${COUCHDB_URL}/${dbname}/_design/qq/_view/dbinfo` - ); - const dbinfoRows = dbinfoRes.data.rows || []; + const dbinfoRows = await fetchView(dbname, "dbinfo"); for (const row of dbinfoRows) { const subj = String(row.value?.subj?.length || 0); await upsertIoview(dbname, row.id, subj, "dbinfo", row.value); } console.log(` ${dbname}: dbinfo synced (${dbinfoRows.length} rows)`); - const subjectsRes = await axios.get( - `${COUCHDB_URL}/${dbname}/_design/qq/_view/subjects` - ); - const subjectRows = subjectsRes.data.rows || []; + const subjectRows = await fetchView(dbname, "subjects"); for (const row of subjectRows) { const subj = String(row.key?.[6] || ""); await upsertIoview(dbname, row.id, subj, "subjects", { @@ -403,10 +414,7 @@ async function firstSync(dbname) { } console.log(` ${dbname}: subjects synced (${subjectRows.length} rows)`); - const linksRes = await axios.get( - `${COUCHDB_URL}/${dbname}/_design/qq/_view/links` - ); - const linkRows = linksRes.data.rows || []; + const linkRows = await fetchView(dbname, "links"); for (const row of linkRows) { const fileType = row.key?.[0]; const subjId = String(row.key?.[1] || ""); From 4490af6c2060d0b732cb8de106ae158cfbdde77e Mon Sep 17 00:00:00 2001 From: elainefan331 Date: Mon, 11 May 2026 11:44:02 -0400 Subject: [PATCH 07/29] fix: make suggested databases refresh on Search click --- src/pages/SearchPage.tsx | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/src/pages/SearchPage.tsx b/src/pages/SearchPage.tsx index 8e35494..b0479af 100644 --- a/src/pages/SearchPage.tsx +++ b/src/pages/SearchPage.tsx @@ -91,9 +91,10 @@ const SearchPage: React.FC = () => { const placement = upMd ? "right" : "top"; - // for database card - const keywordInput = String(formData?.keyword ?? "").trim(); - const selectedDbId = String(formData?.database ?? "").trim(); + // inputs for the "Suggested databases" memo — read from appliedFilters so + // the suggestion list refreshes only on Search click, matching the results. + const keywordInput = String(appliedFilters?.keyword ?? "").trim(); + const selectedDbId = String(appliedFilters?.database ?? "").trim(); const registryMatches: RegistryItem[] = React.useMemo(() => { if (!Array.isArray(registry)) return []; @@ -635,10 +636,9 @@ const SearchPage: React.FC = () => { }} title={ - Live preview based on your keyword or selected database. - This list updates as you type or change the dropdown. - It’s separate from the results—you’ll - see datasets/subjects after you click Search. + Databases that match your keyword or selected database + filter. This list refreshes when you click{" "} + Search, alongside the datasets/subjects below. } > @@ -668,7 +668,7 @@ const SearchPage: React.FC = () => { datasets={db.datasets} modalities={db.datatype} logo={db.logo} - keyword={formData.keyword} // for keyword highlight + keyword={appliedFilters.keyword} // highlight the searched keyword, not the live input onChipClick={handleChipClick} /> ))} From 5966b0957e5e02fec0c83bcf1ff2e00514ea64ea Mon Sep 17 00:00:00 2001 From: elainefan331 Date: Mon, 11 May 2026 13:54:32 -0400 Subject: [PATCH 08/29] feat: modality-aware combobox for Data type filter --- .../widgets/TypeAutocompleteWidget.tsx | 27 +++++++++++++++ src/pages/SearchPage.tsx | 5 +++ .../SearchPageFunctions/generateUiSchema.ts | 15 +++++--- .../SearchPageFunctions/searchformSchema.ts | 2 +- .../SearchPageFunctions/typesByModality.ts | 34 +++++++++++++++++++ 5 files changed, 78 insertions(+), 5 deletions(-) create mode 100644 src/components/SearchPage/widgets/TypeAutocompleteWidget.tsx create mode 100644 src/utils/SearchPageFunctions/typesByModality.ts diff --git a/src/components/SearchPage/widgets/TypeAutocompleteWidget.tsx b/src/components/SearchPage/widgets/TypeAutocompleteWidget.tsx new file mode 100644 index 0000000..b0573fc --- /dev/null +++ b/src/components/SearchPage/widgets/TypeAutocompleteWidget.tsx @@ -0,0 +1,27 @@ +import { Autocomplete, TextField } from "@mui/material"; +import { WidgetProps } from "@rjsf/utils"; + +// Combobox: type freely OR pick from a modality-specific suggestion list. +export const TypeAutocompleteWidget = (props: WidgetProps) => { + const { value, onChange, options, label, placeholder } = props; + const suggestions = (options.suggestions as string[]) || []; + + return ( + onChange(typeof v === "string" ? v : "")} + onInputChange={(_, v) => onChange(v || "")} + renderInput={(params) => ( + + )} + /> + ); +}; diff --git a/src/pages/SearchPage.tsx b/src/pages/SearchPage.tsx index b0479af..cfd7fd7 100644 --- a/src/pages/SearchPage.tsx +++ b/src/pages/SearchPage.tsx @@ -22,6 +22,7 @@ import ClickTooltip from "components/SearchPage/ClickTooltip"; import DatabaseCard from "components/SearchPage/DatabaseCard"; import DatasetCard from "components/SearchPage/DatasetCard"; import SubjectCard from "components/SearchPage/SubjectCard"; +import { TypeAutocompleteWidget } from "components/SearchPage/widgets/TypeAutocompleteWidget"; import { Colors } from "design/theme"; import { useAppDispatch } from "hooks/useAppDispatch"; import { useAppSelector } from "hooks/useAppSelector"; @@ -212,6 +213,9 @@ const SearchPage: React.FC = () => { [formData, showSubjectFilters, showDatasetFilters] ); + // Custom RJSF widgets — combobox for the "Data type keywords" field. + const customWidgets = { typeAutocomplete: TypeAutocompleteWidget }; + // Create the "Subject-level Filters" button as a custom field const customFields = { subjectFiltersToggle: () => ( @@ -401,6 +405,7 @@ const SearchPage: React.FC = () => { onChange={({ formData }) => setFormData(formData)} uiSchema={uiSchema} fields={customFields} + widgets={customWidgets} /> ); diff --git a/src/utils/SearchPageFunctions/generateUiSchema.ts b/src/utils/SearchPageFunctions/generateUiSchema.ts index f352a23..7b54428 100644 --- a/src/utils/SearchPageFunctions/generateUiSchema.ts +++ b/src/utils/SearchPageFunctions/generateUiSchema.ts @@ -1,4 +1,5 @@ import { Colors } from "design/theme"; +import { getTypeSuggestions } from "./typesByModality"; // Controls the background highlight of selected fields // Controls the visibility of subject-level filters @@ -52,6 +53,7 @@ export const generateUiSchema = ( "keyword", "subject_filters_toggle", "modality", + "type_name", // sits right after modality — its options depend on it "gender", "age_min", "age_max", @@ -62,7 +64,6 @@ export const generateUiSchema = ( "run_min", "run_max", "task_name", - "type_name", "session_name", "run_name", "limit", @@ -156,9 +157,15 @@ export const generateUiSchema = ( : {} : hiddenStyle, type_name: showSubjectFilters - ? formData["type_name"] - ? activeStyle - : {} + ? { + "ui:widget": "typeAutocomplete", + "ui:options": { + suggestions: getTypeSuggestions(formData.modality), + ...(formData["type_name"] + ? { style: { backgroundColor: Colors.lightBlue } } + : {}), + }, + } : hiddenStyle, session_name: showSubjectFilters ? formData["session_name"] diff --git a/src/utils/SearchPageFunctions/searchformSchema.ts b/src/utils/SearchPageFunctions/searchformSchema.ts index 7fc71b5..ead7855 100644 --- a/src/utils/SearchPageFunctions/searchformSchema.ts +++ b/src/utils/SearchPageFunctions/searchformSchema.ts @@ -118,7 +118,7 @@ export const baseSchema: JSONSchema7 = { type: "string", }, type_name: { - title: "Data type keywords", + title: "Data type", type: "string", }, session_name: { diff --git a/src/utils/SearchPageFunctions/typesByModality.ts b/src/utils/SearchPageFunctions/typesByModality.ts new file mode 100644 index 0000000..090475d --- /dev/null +++ b/src/utils/SearchPageFunctions/typesByModality.ts @@ -0,0 +1,34 @@ +// Common BIDS suffixes grouped by modality. +// Extend the lists as you find missing values in your data. +export const TYPES_BY_MODALITY: Record = { + anat: ["T1w", "T2w", "FLAIR", "T2star", "PD", "angio", "defacemask"], + func: ["bold", "sbref", "events", "physio", "stim"], + dwi: ["dwi", "sbref"], + fmap: ["phasediff", "magnitude1", "magnitude2", "fieldmap", "epi"], + meg: ["meg", "channels", "coordsystem", "headshape", "events"], + eeg: ["eeg", "channels", "electrodes", "coordsystem", "events"], + ieeg: ["ieeg", "channels", "electrodes", "coordsystem", "events"], + pet: ["pet", "blood", "events"], + nirs: ["nirs", "channels", "optodes", "coordsystem", "events"], + beh: ["beh", "events"], + motion: ["motion", "channels", "events"], + perf: ["asl", "m0scan"], + micr: ["TEM", "SEM", "MRM"], +}; + +// The modality form field stores values like "fMRI (func)" — extract the +// suffix inside the parens so we can look it up in TYPES_BY_MODALITY. +export function getModalityKey(modalityValue?: string): string | null { + if (!modalityValue || modalityValue === "any") return null; + const m = modalityValue.match(/\(([^)]+)\)/); + return m ? m[1] : modalityValue; +} + +export function getTypeSuggestions(modalityValue?: string): string[] { + const key = getModalityKey(modalityValue); + if (!key) { + // No modality picked → show all suffixes deduped and sorted. + return Array.from(new Set(Object.values(TYPES_BY_MODALITY).flat())).sort(); + } + return TYPES_BY_MODALITY[key] || []; +} From d101e5cd64b0f6c4ce582c4cf4eec1ae2cf340b7 Mon Sep 17 00:00:00 2001 From: elainefan331 Date: Mon, 11 May 2026 14:54:30 -0400 Subject: [PATCH 09/29] feat(search): add draggable age-range slider at top of subject filters --- src/pages/SearchPage.tsx | 58 +++++++++++++++++++ .../SearchPageFunctions/generateUiSchema.ts | 19 +++--- .../SearchPageFunctions/searchformSchema.ts | 4 ++ 3 files changed, 71 insertions(+), 10 deletions(-) diff --git a/src/pages/SearchPage.tsx b/src/pages/SearchPage.tsx index cfd7fd7..e9ac04d 100644 --- a/src/pages/SearchPage.tsx +++ b/src/pages/SearchPage.tsx @@ -13,6 +13,7 @@ import { Tooltip, IconButton, Alert, + Slider, } from "@mui/material"; import { useTheme } from "@mui/material/styles"; import useMediaQuery from "@mui/material/useMediaQuery"; @@ -46,6 +47,60 @@ type RegistryItem = { logo?: string; }; +// Module-scope so the component identity is stable across SearchPage renders. +// An inline arrow function inside customFields was getting a new identity each +// render, which made RJSF remount the slider mid-drag. +const AGE_MIN_BOUND = 0; +const AGE_MAX_BOUND = 100; + +const AgeRangeSliderField = (props: any) => { + const ctx = props?.registry?.formContext as + | { + formData: Record; + setFormData: React.Dispatch>>; + } + | undefined; + if (!ctx) return null; + const { formData, setFormData } = ctx; + const lo = + typeof formData.age_min === "number" ? formData.age_min : AGE_MIN_BOUND; + const hi = + typeof formData.age_max === "number" ? formData.age_max : AGE_MAX_BOUND; + const isAny = lo === AGE_MIN_BOUND && hi === AGE_MAX_BOUND; + return ( + + + Age: {isAny ? "Any" : `${lo} – ${hi}`} + + { + const [newLo, newHi] = v as number[]; + setFormData((prev) => { + const atFull = + newLo === AGE_MIN_BOUND && newHi === AGE_MAX_BOUND; + const next = { ...prev }; + if (atFull) { + delete next.age_min; + delete next.age_max; + } else { + next.age_min = newLo; + next.age_max = newHi; + } + return next; + }); + }} + valueLabelDisplay="auto" + min={AGE_MIN_BOUND} + max={AGE_MAX_BOUND} + step={1} + disableSwap + sx={{ color: Colors.purple }} + /> + + ); +}; + const matchesKeyword = (item: RegistryItem, keyword: string) => { if (!keyword) return false; const needle = keyword.toLowerCase(); @@ -252,6 +307,7 @@ const SearchPage: React.FC = () => { ), + ageRangeSlider: AgeRangeSliderField, }; // determine the results are subject-level or dataset-level @@ -406,6 +462,7 @@ const SearchPage: React.FC = () => { uiSchema={uiSchema} fields={customFields} widgets={customWidgets} + formContext={{ formData, setFormData }} /> ); @@ -847,6 +904,7 @@ const SearchPage: React.FC = () => { {...item} parsedJson={parsedJson} onChipClick={handleChipClick} + age={parsedJson?.key?.[0]} /> ); } catch (e) { diff --git a/src/utils/SearchPageFunctions/generateUiSchema.ts b/src/utils/SearchPageFunctions/generateUiSchema.ts index 7b54428..d604b5f 100644 --- a/src/utils/SearchPageFunctions/generateUiSchema.ts +++ b/src/utils/SearchPageFunctions/generateUiSchema.ts @@ -52,10 +52,11 @@ export const generateUiSchema = ( "database", "keyword", "subject_filters_toggle", + "age_range_slider", // top of subject filters — range slider for age "modality", "type_name", // sits right after modality — its options depend on it "gender", - "age_min", + "age_min", // hidden via invisibleStyle; written by the slider above "age_max", "sess_min", "sess_max", @@ -101,16 +102,14 @@ export const generateUiSchema = ( : {} : hiddenStyle, - age_min: showSubjectFilters - ? formData["age_min"] - ? activeStyle - : {} - : hiddenStyle, - age_max: showSubjectFilters - ? formData["age_max"] - ? activeStyle - : {} + // Age range — slider lives inside the form via the AgeRangeSliderField + // stable component. age_min/age_max stay in the schema (so the backend + // gets them on submit) but their default numeric inputs are hidden. + age_range_slider: showSubjectFilters + ? { "ui:field": "ageRangeSlider" } : hiddenStyle, + age_min: invisibleStyle, + age_max: invisibleStyle, gender: showSubjectFilters ? formData["gender"] && formData["gender"] !== "any" diff --git a/src/utils/SearchPageFunctions/searchformSchema.ts b/src/utils/SearchPageFunctions/searchformSchema.ts index ead7855..bcba6d7 100644 --- a/src/utils/SearchPageFunctions/searchformSchema.ts +++ b/src/utils/SearchPageFunctions/searchformSchema.ts @@ -64,6 +64,10 @@ export const baseSchema: JSONSchema7 = { enum: ["male", "female", "unknown", "any"], default: "any", }, + age_range_slider: { + type: "null", + title: "Age range", + }, age_min: { title: "Minimum age", type: "number", From 88474cc4caf485da90a6a5ea366f69f208210ef3 Mon Sep 17 00:00:00 2001 From: elainefan331 Date: Mon, 11 May 2026 15:05:54 -0400 Subject: [PATCH 10/29] feat(search): add placeholder hints to task/session/run keyword fields --- .../SearchPageFunctions/generateUiSchema.ts | 21 +++++++++++-------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/src/utils/SearchPageFunctions/generateUiSchema.ts b/src/utils/SearchPageFunctions/generateUiSchema.ts index d604b5f..e27d02c 100644 --- a/src/utils/SearchPageFunctions/generateUiSchema.ts +++ b/src/utils/SearchPageFunctions/generateUiSchema.ts @@ -151,9 +151,10 @@ export const generateUiSchema = ( : hiddenStyle, task_name: showSubjectFilters - ? formData["task_name"] - ? activeStyle - : {} + ? { + "ui:placeholder": "e.g. rest, motor", + ...(formData["task_name"] ? activeStyle : {}), + } : hiddenStyle, type_name: showSubjectFilters ? { @@ -167,14 +168,16 @@ export const generateUiSchema = ( } : hiddenStyle, session_name: showSubjectFilters - ? formData["session_name"] - ? activeStyle - : {} + ? { + "ui:placeholder": "e.g. 01, pre, baseline", + ...(formData["session_name"] ? activeStyle : {}), + } : hiddenStyle, run_name: showSubjectFilters - ? formData["run_name"] - ? activeStyle - : {} + ? { + "ui:placeholder": "e.g. 01, 02", + ...(formData["run_name"] ? activeStyle : {}), + } : hiddenStyle, "ui:submitButtonOptions": { From b9be29f248103c4c44e743c7274cb501d5c1b407 Mon Sep 17 00:00:00 2001 From: elainefan331 Date: Mon, 11 May 2026 15:18:23 -0400 Subject: [PATCH 11/29] feat(search): pair min/max count fields on one row and tighten gaps(sessions, tasks, and runs) --- src/pages/SearchPage.tsx | 60 +++++++++++++++ .../SearchPageFunctions/generateUiSchema.ts | 76 +++++++++++-------- .../SearchPageFunctions/searchformSchema.ts | 3 + 3 files changed, 106 insertions(+), 33 deletions(-) diff --git a/src/pages/SearchPage.tsx b/src/pages/SearchPage.tsx index e9ac04d..ca5d1c4 100644 --- a/src/pages/SearchPage.tsx +++ b/src/pages/SearchPage.tsx @@ -14,6 +14,8 @@ import { IconButton, Alert, Slider, + Stack, + TextField, } from "@mui/material"; import { useTheme } from "@mui/material/styles"; import useMediaQuery from "@mui/material/useMediaQuery"; @@ -101,6 +103,63 @@ const AgeRangeSliderField = (props: any) => { ); }; +// Pairs a "_min" + "_max" into a single row of two number inputs. +// Reads target field names + label from uiSchema's ui:options: +// { minKey: "sess_min", maxKey: "sess_max", label: "sessions" } +const CountRangePairField = (props: any) => { + const ctx = props?.registry?.formContext as + | { + formData: Record; + setFormData: React.Dispatch>>; + } + | undefined; + const opts = props?.uiSchema?.["ui:options"] || {}; + const minKey = opts.minKey as string; + const maxKey = opts.maxKey as string; + const label = (opts.label as string) || ""; + if (!ctx || !minKey || !maxKey) return null; + const { formData, setFormData } = ctx; + const minVal = formData[minKey] ?? ""; + const maxVal = formData[maxKey] ?? ""; + + const update = (key: string, raw: string) => { + setFormData((prev) => { + const next = { ...prev }; + if (raw === "" || raw === undefined) { + delete next[key]; + } else { + const n = Number(raw); + if (Number.isNaN(n)) delete next[key]; + else next[key] = n; + } + return next; + }); + }; + + return ( + + update(minKey, e.target.value)} + fullWidth + inputProps={{ min: 0 }} + /> + update(maxKey, e.target.value)} + fullWidth + inputProps={{ min: 0 }} + /> + + ); +}; + const matchesKeyword = (item: RegistryItem, keyword: string) => { if (!keyword) return false; const needle = keyword.toLowerCase(); @@ -308,6 +367,7 @@ const SearchPage: React.FC = () => { ), ageRangeSlider: AgeRangeSliderField, + countRangePair: CountRangePairField, }; // determine the results are subject-level or dataset-level diff --git a/src/utils/SearchPageFunctions/generateUiSchema.ts b/src/utils/SearchPageFunctions/generateUiSchema.ts index e27d02c..872accb 100644 --- a/src/utils/SearchPageFunctions/generateUiSchema.ts +++ b/src/utils/SearchPageFunctions/generateUiSchema.ts @@ -16,13 +16,11 @@ export const generateUiSchema = ( }, }; - // hide subject-level filter + // Fully remove a field from the rendered DOM (keeps its value in formData). + // Using ui:widget: "hidden" produces just an , so no + // empty Grid row + margin is left behind — fixes the big gap between rows. const invisibleStyle = { - "ui:options": { - style: { - display: "none", - }, - }, + "ui:widget": "hidden", }; const hiddenStyle = { @@ -58,10 +56,13 @@ export const generateUiSchema = ( "gender", "age_min", // hidden via invisibleStyle; written by the slider above "age_max", + "sess_count_range", // sessions min/max on one row "sess_min", "sess_max", + "task_count_range", // tasks min/max on one row "task_min", "task_max", + "run_count_range", // runs min/max on one row "run_min", "run_max", "task_name", @@ -117,38 +118,47 @@ export const generateUiSchema = ( : {} : hiddenStyle, - sess_min: showSubjectFilters - ? formData["sess_min"] - ? activeStyle - : {} - : hiddenStyle, - sess_max: showSubjectFilters - ? formData["sess_max"] - ? activeStyle - : {} + // Session / task / run min+max pairs are rendered by a single + // CountRangePairField each. The raw integer inputs are hidden but stay in + // formData so the backend still receives them on submit. + sess_count_range: showSubjectFilters + ? { + "ui:field": "countRangePair", + "ui:options": { + minKey: "sess_min", + maxKey: "sess_max", + label: "sessions", + }, + } : hiddenStyle, + sess_min: invisibleStyle, + sess_max: invisibleStyle, - task_min: showSubjectFilters - ? formData["task_min"] - ? activeStyle - : {} - : hiddenStyle, - task_max: showSubjectFilters - ? formData["task_max"] - ? activeStyle - : {} + task_count_range: showSubjectFilters + ? { + "ui:field": "countRangePair", + "ui:options": { + minKey: "task_min", + maxKey: "task_max", + label: "tasks", + }, + } : hiddenStyle, + task_min: invisibleStyle, + task_max: invisibleStyle, - run_min: showSubjectFilters - ? formData["run_min"] - ? activeStyle - : {} - : hiddenStyle, - run_max: showSubjectFilters - ? formData["run_max"] - ? activeStyle - : {} + run_count_range: showSubjectFilters + ? { + "ui:field": "countRangePair", + "ui:options": { + minKey: "run_min", + maxKey: "run_max", + label: "runs", + }, + } : hiddenStyle, + run_min: invisibleStyle, + run_max: invisibleStyle, task_name: showSubjectFilters ? { diff --git a/src/utils/SearchPageFunctions/searchformSchema.ts b/src/utils/SearchPageFunctions/searchformSchema.ts index bcba6d7..34abc4a 100644 --- a/src/utils/SearchPageFunctions/searchformSchema.ts +++ b/src/utils/SearchPageFunctions/searchformSchema.ts @@ -81,6 +81,7 @@ export const baseSchema: JSONSchema7 = { maximum: 1000, }, + sess_count_range: { type: "null", title: "Sessions" }, sess_min: { title: "Minimum session count", type: "integer", @@ -93,6 +94,7 @@ export const baseSchema: JSONSchema7 = { minimum: 0, maximum: 1000, }, + task_count_range: { type: "null", title: "Tasks" }, task_min: { title: "Minimum task count", type: "integer", @@ -105,6 +107,7 @@ export const baseSchema: JSONSchema7 = { minimum: 0, maximum: 1000, }, + run_count_range: { type: "null", title: "Runs" }, run_min: { title: "Minimum runs", type: "integer", From df73e1cbc0e2f61fe1c41c006a6dd08088d1871c Mon Sep 17 00:00:00 2001 From: elainefan331 Date: Mon, 11 May 2026 15:31:47 -0400 Subject: [PATCH 12/29] feat(search): show run count in subject card --- src/components/SearchPage/SubjectCard.tsx | 9 ++++++++- src/utils/SearchPageFunctions/searchformSchema.ts | 2 +- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/src/components/SearchPage/SubjectCard.tsx b/src/components/SearchPage/SubjectCard.tsx index c66ebce..f805b48 100644 --- a/src/components/SearchPage/SubjectCard.tsx +++ b/src/components/SearchPage/SubjectCard.tsx @@ -16,6 +16,7 @@ interface SubjectCardProps { modalities?: string[]; tasks?: string[]; sessions?: string[]; + runs?: string[]; types?: string[]; }; }; @@ -32,7 +33,8 @@ const SubjectCard: React.FC = ({ index, onChipClick, }) => { - const { modalities, tasks, sessions, types } = parsedJson.value; + const { modalities, tasks, sessions, runs, types } = parsedJson.value; + const runCount = Array.isArray(runs) ? runs.length : 0; const subjectLink = `${RoutesEnum.DATABASES}/${dbname}/${dsname}`; const formattedSubj = /^sub-/i.test(subj) ? subj : `sub-${String(subj)}`; @@ -229,6 +231,11 @@ const SubjectCard: React.FC = ({ {sessions?.length === 0 ? 1 : sessions?.length} + + + Runs: {runCount} + + diff --git a/src/utils/SearchPageFunctions/searchformSchema.ts b/src/utils/SearchPageFunctions/searchformSchema.ts index 34abc4a..4cb4ef6 100644 --- a/src/utils/SearchPageFunctions/searchformSchema.ts +++ b/src/utils/SearchPageFunctions/searchformSchema.ts @@ -61,7 +61,7 @@ export const baseSchema: JSONSchema7 = { gender: { title: "Subject gender", type: "string", - enum: ["male", "female", "unknown", "any"], + enum: ["male", "female", "any"], default: "any", }, age_range_slider: { From 44635bd8a4e67c909306c43d68a0c8743bf24d2c Mon Sep 17 00:00:00 2001 From: elainefan331 Date: Tue, 12 May 2026 10:55:48 -0400 Subject: [PATCH 13/29] feat(backend): add file type filter to search using iolinks table --- backend/src/controllers/couchdb.controller.js | 36 +++++++++++++++++++ backend/src/routes/dbs.routes.js | 6 ++++ 2 files changed, 42 insertions(+) diff --git a/backend/src/controllers/couchdb.controller.js b/backend/src/controllers/couchdb.controller.js index f6c54c5..edb9957 100644 --- a/backend/src/controllers/couchdb.controller.js +++ b/backend/src/controllers/couchdb.controller.js @@ -227,6 +227,20 @@ const searchAllDatabases = async (req, res) => { repl.keyword = String(f.keyword); } + // File-type filter — array of extensions like [".jdb", ".snirf"]. + // Dataset-level: include rows whose (dbname, dsname) has at least one + // iolinks file with a matching view (extension). Per-subject filtering + // isn't possible here because iolinks.subj stores file size, not subj id. + if (Array.isArray(f.file_type) && f.file_type.length > 0) { + where.push(`EXISTS ( + SELECT 1 FROM iolinks l + WHERE l.dbname = ioviews.dbname + AND l.dsname = ioviews.dsname + AND l.view = ANY(:fileTypes) + )`); + repl.fileTypes = f.file_type.map((t) => String(t)); + } + const limit = Math.min(parseInt(f.limit) || 100, 1000); const offset = parseInt(f.skip) || 0; repl.limit = limit; @@ -392,6 +406,27 @@ const getDatasetMeta = async (req, res) => { // } +// distinct file extensions present in iolinks across all synced DBs. +// Drives the multi-select "File types" filter on the search page. +const getFileTypes = async (req, res) => { + try { + const rows = await sequelize.query( + `SELECT DISTINCT view AS type + FROM iolinks + WHERE view IS NOT NULL AND view <> '' + ORDER BY view`, + { type: sequelize.QueryTypes.SELECT } + ); + res.status(200).json(rows.map((r) => r.type)); + } catch (error) { + console.error("Error fetching file types:", error.message); + res.status(500).json({ + message: "Error fetching file types", + error: error.message, + }); + } +}; + module.exports = { getDbList, getDbStats, @@ -400,4 +435,5 @@ module.exports = { searchAllDatabases, getDatasetDetail, getDatasetMeta, + getFileTypes, }; diff --git a/backend/src/routes/dbs.routes.js b/backend/src/routes/dbs.routes.js index c8a57fa..621b32f 100644 --- a/backend/src/routes/dbs.routes.js +++ b/backend/src/routes/dbs.routes.js @@ -6,6 +6,7 @@ const { getDbInfo, getDbDatasets, searchAllDatabases, + getFileTypes, // searchDatabase, } = require("../controllers/couchdb.controller"); @@ -15,6 +16,11 @@ const router = express.Router(); router.get("/", getDbList); router.get("/stats", getDbStats); +// distinct file extensions across all iolinks rows (drives the file-type +// filter on the search page). Must come BEFORE the /:dbName route, otherwise +// Express treats "file-types" as a dbName. +router.get("/file-types", getFileTypes); + // cross-database search router.post("/search", searchAllDatabases); From 1b0aa6576ecddb1c225adb3d8b0c543ea8069bb9 Mon Sep 17 00:00:00 2001 From: elainefan331 Date: Tue, 12 May 2026 11:37:12 -0400 Subject: [PATCH 14/29] feat(search): add multi-select File types filter (dataset-level) --- backend/src/controllers/couchdb.controller.js | 4 +- .../widgets/FileTypeAutocompleteWidget.tsx | 40 +++++++++++++++++++ src/pages/SearchPage.tsx | 27 +++++++++++-- src/redux/neurojson/neurojson.action.ts | 13 ++++++ src/redux/neurojson/neurojson.slice.ts | 13 ++++++ .../neurojson/types/neurojson.interface.ts | 1 + src/services/neurojson.service.ts | 7 ++++ .../SearchPageFunctions/generateUiSchema.ts | 18 ++++++++- .../SearchPageFunctions/searchformSchema.ts | 6 +++ 9 files changed, 123 insertions(+), 6 deletions(-) create mode 100644 src/components/SearchPage/widgets/FileTypeAutocompleteWidget.tsx diff --git a/backend/src/controllers/couchdb.controller.js b/backend/src/controllers/couchdb.controller.js index edb9957..928a61b 100644 --- a/backend/src/controllers/couchdb.controller.js +++ b/backend/src/controllers/couchdb.controller.js @@ -231,12 +231,14 @@ const searchAllDatabases = async (req, res) => { // Dataset-level: include rows whose (dbname, dsname) has at least one // iolinks file with a matching view (extension). Per-subject filtering // isn't possible here because iolinks.subj stores file size, not subj id. + // Use IN (:array) — Sequelize replacements expand arrays as 'a','b','c', + // which fits IN(...) but NOT ANY(...). if (Array.isArray(f.file_type) && f.file_type.length > 0) { where.push(`EXISTS ( SELECT 1 FROM iolinks l WHERE l.dbname = ioviews.dbname AND l.dsname = ioviews.dsname - AND l.view = ANY(:fileTypes) + AND l.view IN (:fileTypes) )`); repl.fileTypes = f.file_type.map((t) => String(t)); } diff --git a/src/components/SearchPage/widgets/FileTypeAutocompleteWidget.tsx b/src/components/SearchPage/widgets/FileTypeAutocompleteWidget.tsx new file mode 100644 index 0000000..eb941ad --- /dev/null +++ b/src/components/SearchPage/widgets/FileTypeAutocompleteWidget.tsx @@ -0,0 +1,40 @@ +import { Autocomplete, Chip, TextField } from "@mui/material"; +import { WidgetProps } from "@rjsf/utils"; + +// Multi-select combobox for file extensions (e.g. ".jdb", ".snirf"). +// Options come from uiSchema's ui:options.fileTypes, fetched once by the +// parent SearchPage from /api/v1/dbs/file-types. +export const FileTypeAutocompleteWidget = (props: WidgetProps) => { + const { value, onChange, options, label } = props; + const fileTypes = (options.fileTypes as string[]) || []; + const current: string[] = Array.isArray(value) ? value : []; + + return ( + onChange(v as string[])} + renderTags={(items, getTagProps) => + items.map((item, index) => ( + + )) + } + renderInput={(params) => ( + + )} + /> + ); +}; diff --git a/src/pages/SearchPage.tsx b/src/pages/SearchPage.tsx index ca5d1c4..e8e8da0 100644 --- a/src/pages/SearchPage.tsx +++ b/src/pages/SearchPage.tsx @@ -25,6 +25,7 @@ import ClickTooltip from "components/SearchPage/ClickTooltip"; import DatabaseCard from "components/SearchPage/DatabaseCard"; import DatasetCard from "components/SearchPage/DatasetCard"; import SubjectCard from "components/SearchPage/SubjectCard"; +import { FileTypeAutocompleteWidget } from "components/SearchPage/widgets/FileTypeAutocompleteWidget"; import { TypeAutocompleteWidget } from "components/SearchPage/widgets/TypeAutocompleteWidget"; import { Colors } from "design/theme"; import { useAppDispatch } from "hooks/useAppDispatch"; @@ -33,6 +34,7 @@ import pako from "pako"; import React from "react"; import { useState, useEffect, useMemo } from "react"; import { + fetchFileTypes, fetchMetadataSearchResults, fetchRegistry, } from "redux/neurojson/neurojson.action"; @@ -186,6 +188,9 @@ const SearchPage: React.FC = () => { const registry = useAppSelector( (state: RootState) => state.neurojson.registry ); + const fileTypes = useAppSelector( + (state: RootState) => state.neurojson.fileTypes + ); const loading = useAppSelector((state: RootState) => state.neurojson.loading); const [formData, setFormData] = useState>({}); @@ -323,12 +328,21 @@ const SearchPage: React.FC = () => { // form UI const uiSchema = useMemo( - () => generateUiSchema(formData, showSubjectFilters, showDatasetFilters), - [formData, showSubjectFilters, showDatasetFilters] + () => + generateUiSchema( + formData, + showSubjectFilters, + showDatasetFilters, + fileTypes || [] + ), + [formData, showSubjectFilters, showDatasetFilters, fileTypes] ); - // Custom RJSF widgets — combobox for the "Data type keywords" field. - const customWidgets = { typeAutocomplete: TypeAutocompleteWidget }; + // Custom RJSF widgets — comboboxes for the Data type and File types fields. + const customWidgets = { + typeAutocomplete: TypeAutocompleteWidget, + fileTypeAutocomplete: FileTypeAutocompleteWidget, + }; // Create the "Subject-level Filters" button as a custom field const customFields = { @@ -387,6 +401,11 @@ const SearchPage: React.FC = () => { dispatch(fetchRegistry()); }, [dispatch]); + // get the distinct file extensions for the "File types" multi-select. + useEffect(() => { + dispatch(fetchFileTypes()); + }, [dispatch]); + // dynamically add database enum to schema const schema = useMemo(() => { const dbList = registry?.length diff --git a/src/redux/neurojson/neurojson.action.ts b/src/redux/neurojson/neurojson.action.ts index 35cf6c1..f8f08c2 100644 --- a/src/redux/neurojson/neurojson.action.ts +++ b/src/redux/neurojson/neurojson.action.ts @@ -109,6 +109,19 @@ export const fetchMetadataSearchResults = createAsyncThunk( } ); +// distinct iolinks file extensions — populates the "File types" multi-select +export const fetchFileTypes = createAsyncThunk( + "neurojson/fetchFileTypes", + async (_, { rejectWithValue }) => { + try { + const data = await NeurojsonService.getFileTypes(); + return data; + } catch (error: any) { + return rejectWithValue("Failed to fetch file types"); + } + } +); + // fetch data for metadata panel in dataset detail page export const fetchDbInfoByDatasetId = createAsyncThunk( "neurojson/fetchDbInfoByDatasetId", diff --git a/src/redux/neurojson/neurojson.slice.ts b/src/redux/neurojson/neurojson.slice.ts index 90722c7..cfafc6a 100644 --- a/src/redux/neurojson/neurojson.slice.ts +++ b/src/redux/neurojson/neurojson.slice.ts @@ -7,6 +7,7 @@ import { fetchDbStats, fetchMetadataSearchResults, fetchDbInfoByDatasetId, + fetchFileTypes, } from "./neurojson.action"; import { DBDatafields, INeuroJsonState } from "./types/neurojson.interface"; import { createSlice, PayloadAction } from "@reduxjs/toolkit"; @@ -26,6 +27,7 @@ const initialState: INeuroJsonState = { dbStats: null, searchResults: null, datasetViewInfo: null, + fileTypes: null, }; const neurojsonSlice = createSlice({ @@ -155,6 +157,17 @@ const neurojsonSlice = createSlice({ state.loading = false; state.error = action.payload as string; }) + // fetchFileTypes runs once on mount; no pending case so it doesn't + // clobber the shared `loading` spinner used by the search button. + .addCase( + fetchFileTypes.fulfilled, + (state, action: PayloadAction) => { + state.fileTypes = action.payload; + } + ) + .addCase(fetchFileTypes.rejected, (state, action) => { + state.error = action.payload as string; + }) .addCase(fetchDbInfoByDatasetId.pending, (state) => { state.loading = true; state.error = null; diff --git a/src/redux/neurojson/types/neurojson.interface.ts b/src/redux/neurojson/types/neurojson.interface.ts index 365566d..01c8273 100644 --- a/src/redux/neurojson/types/neurojson.interface.ts +++ b/src/redux/neurojson/types/neurojson.interface.ts @@ -13,6 +13,7 @@ export interface INeuroJsonState { dbStats: DbStatsItem[] | null; // for dbStats on landing page searchResults: any[] | { status: string; msg: string } | null; datasetViewInfo: any | null; + fileTypes: string[] | null; } export interface DBParticulars { diff --git a/src/services/neurojson.service.ts b/src/services/neurojson.service.ts index 008e960..ed1f70d 100644 --- a/src/services/neurojson.service.ts +++ b/src/services/neurojson.service.ts @@ -150,6 +150,13 @@ export const NeurojsonService = { return response.data; }, + // GET /api/v1/dbs/file-types → distinct iolinks.view values + // Drives the multi-select "File types" filter on the search page. + getFileTypes: async (): Promise => { + const response = await api.get(`/dbs/file-types`); + return response.data; + }, + // getDbInfoByDatasetId: async (dbName: string, dsId: string): Promise => { // const response = await api.get( // `${baseURL}/${dbName}/_design/qq/_view/dbinfo`, diff --git a/src/utils/SearchPageFunctions/generateUiSchema.ts b/src/utils/SearchPageFunctions/generateUiSchema.ts index 872accb..152a056 100644 --- a/src/utils/SearchPageFunctions/generateUiSchema.ts +++ b/src/utils/SearchPageFunctions/generateUiSchema.ts @@ -6,7 +6,8 @@ import { getTypeSuggestions } from "./typesByModality"; export const generateUiSchema = ( formData: Record, showSubjectFilters: boolean, - showDatasetFilters: boolean + showDatasetFilters: boolean, + fileTypeOptions: string[] = [] ) => { const activeStyle = { "ui:options": { @@ -49,6 +50,7 @@ export const generateUiSchema = ( "dataset_filters_toggle", // button first "database", "keyword", + "file_type", // dataset-level: filters by file extensions in iolinks "subject_filters_toggle", "age_range_slider", // top of subject filters — range slider for age "modality", @@ -90,6 +92,20 @@ export const generateUiSchema = ( // dataset: formData["dataset"] ? activeStyle : {}, // limit: formData["limit"] ? activeStyle : {}, // skip: formData["skip"] ? activeStyle : {}, + // File-type filter — dataset-level. Multi-select of file extensions + // present in iolinks (fetched dynamically via /api/v1/dbs/file-types). + file_type: showDatasetFilters + ? { + "ui:widget": "fileTypeAutocomplete", + "ui:options": { + fileTypes: fileTypeOptions, + ...(Array.isArray(formData["file_type"]) && + formData["file_type"].length > 0 + ? { style: { backgroundColor: Colors.lightBlue } } + : {}), + }, + } + : datasetHiddenStyle, limit: invisibleStyle, skip: invisibleStyle, diff --git a/src/utils/SearchPageFunctions/searchformSchema.ts b/src/utils/SearchPageFunctions/searchformSchema.ts index 4cb4ef6..e122725 100644 --- a/src/utils/SearchPageFunctions/searchformSchema.ts +++ b/src/utils/SearchPageFunctions/searchformSchema.ts @@ -128,6 +128,12 @@ export const baseSchema: JSONSchema7 = { title: "Data type", type: "string", }, + file_type: { + title: "File types", + type: "array", + items: { type: "string" }, + uniqueItems: true, + }, session_name: { title: "Session keywords", type: "string", From 9f826d4c4e959b9553c191ea8a52afd1585f4ea1 Mon Sep 17 00:00:00 2001 From: elainefan331 Date: Tue, 12 May 2026 12:12:11 -0400 Subject: [PATCH 15/29] fix(search): age slider per-handle clearing + non-BIDS file-type warning --- src/pages/SearchPage.tsx | 64 ++++++++++++++++++++++++++++++++-------- 1 file changed, 51 insertions(+), 13 deletions(-) diff --git a/src/pages/SearchPage.tsx b/src/pages/SearchPage.tsx index e8e8da0..260931c 100644 --- a/src/pages/SearchPage.tsx +++ b/src/pages/SearchPage.tsx @@ -81,16 +81,15 @@ const AgeRangeSliderField = (props: any) => { onChange={(_, v) => { const [newLo, newHi] = v as number[]; setFormData((prev) => { - const atFull = - newLo === AGE_MIN_BOUND && newHi === AGE_MAX_BOUND; const next = { ...prev }; - if (atFull) { - delete next.age_min; - delete next.age_max; - } else { - next.age_min = newLo; - next.age_max = newHi; - } + // Each handle is its own filter. A handle at the bound means + // "no constraint on that side", so we leave it out of formData + // (otherwise age_min=0 silently excludes unknown-age subjects + // whose stored key is "000-1", lexicographically below "00000"). + if (newLo === AGE_MIN_BOUND) delete next.age_min; + else next.age_min = newLo; + if (newHi === AGE_MAX_BOUND) delete next.age_max; + else next.age_max = newHi; return next; }); }} @@ -560,6 +559,35 @@ const SearchPage: React.FC = () => { !loading && // !hasDbMatches && (!hasDatasetMatches || backendEmpty); + + // Tailored empty-state message: when the user combined a file_type filter + // with any subject-level filter and got nothing back, it's almost certainly + // because the file extension lives in non-BIDS datasets (which have no + // subject rows in ioviews). The generic "adjust filters" message hides this. + const SUBJECT_FILTER_KEYS = [ + "age_min", + "age_max", + "gender", + "task_min", + "task_max", + "task_name", + "run_min", + "run_max", + "run_name", + "sess_min", + "sess_max", + "session_name", + "type_name", + "modality", + "subject", + ]; + const isAppliedFilter = (v: any) => + v !== "" && v !== "any" && v !== undefined && v !== null; + const showFileTypeNonBidsHint = + showNoResults && + Array.isArray(appliedFilters.file_type) && + appliedFilters.file_type.length > 0 && + SUBJECT_FILTER_KEYS.some((k) => isAppliedFilter(appliedFilters[k])); return ( { Search Results - - No datasets or subjects found. Please adjust the - filters and try again. - + {showFileTypeNonBidsHint ? ( + + No matching subjects found. The selected file type + may only exist in non-BIDS datasets (e.g. mesh or + atlas libraries), which have no subject-level + records. Try removing subject-level filters + (modality, age, gender, etc.) and search again. + + ) : ( + + No datasets or subjects found. Please adjust the + filters and try again. + + )} )} From 38fc1cbb83131613137d0386f55a4d6b2e993772 Mon Sep 17 00:00:00 2001 From: elainefan331 Date: Tue, 12 May 2026 14:26:36 -0400 Subject: [PATCH 16/29] fix(search): highlight each word of multi-word keyword independently --- backend/src/controllers/couchdb.controller.js | 13 +++- src/components/SearchPage/DatasetCard.tsx | 75 +++++++++++++------ 2 files changed, 65 insertions(+), 23 deletions(-) diff --git a/backend/src/controllers/couchdb.controller.js b/backend/src/controllers/couchdb.controller.js index 928a61b..df1532b 100644 --- a/backend/src/controllers/couchdb.controller.js +++ b/backend/src/controllers/couchdb.controller.js @@ -221,10 +221,19 @@ const searchAllDatabases = async (req, res) => { repl.subj = String(f.subject); } - // Keyword full-text search + // Keyword search — match anywhere relevant. + // tsquery covers stemmed tokens inside the JSON content (name, readme, + // info, modality, subj). ILIKE on dbname/dsname adds substring matching + // so "fnirs" finds "bfnirs", "openfnirs", and any dataset id containing it. + // The whole group is parenthesised so it ANDs cleanly with other filters. if (isFilter(f.keyword)) { - where.push(`search_vector @@ websearch_to_tsquery('english', :keyword)`); + where.push(`( + search_vector @@ websearch_to_tsquery('english', :keyword) + OR dbname ILIKE :keywordLike + OR dsname ILIKE :keywordLike + )`); repl.keyword = String(f.keyword); + repl.keywordLike = `%${String(f.keyword)}%`; } // File-type filter — array of extensions like [".jdb", ".snirf"]. diff --git a/src/components/SearchPage/DatasetCard.tsx b/src/components/SearchPage/DatasetCard.tsx index 42f6646..5790ee5 100644 --- a/src/components/SearchPage/DatasetCard.tsx +++ b/src/components/SearchPage/DatasetCard.tsx @@ -35,11 +35,21 @@ const normalize = (s: string) => ?.replace(/[\u201C\u201D\u2033]/g, '"') ?? // curly → straight ""; +// Multi-word keyword support: backend tsquery treats "head brain" as AND of +// independent tokens. Highlighting should match the same logic — split on +// whitespace and treat each word independently. +const splitKeyword = (kw?: string): string[] => { + if (!kw) return []; + return normalize(kw).trim().split(/\s+/).filter(Boolean); +}; + +const escapeRegex = (s: string) => s.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); + const containsKeyword = (text?: string, kw?: string) => { if (!text || !kw) return false; const t = normalize(text).toLowerCase(); - const k = normalize(kw).toLowerCase(); - return t.includes(k); + const words = splitKeyword(kw.toLowerCase()); + return words.some((w) => t.includes(w)); }; /** Find a short snippet in secondary fields if not already visible */ @@ -62,24 +72,41 @@ function findMatchSnippet( ["ReferencesAndLinks", (v) => v?.info?.ReferencesAndLinks], ]; - const k = normalize(kw).toLowerCase(); + const words = splitKeyword(kw.toLowerCase()); + if (words.length === 0) return null; for (const [label, getter] of CANDIDATE_FIELDS) { const raw = getter(v); // v = parsedJson.value if (!raw) continue; const text = normalize(String(raw)); - const i = text.toLowerCase().indexOf(k); // k is the lowercase version of keyword - if (i >= 0) { - const start = Math.max(0, i - 40); - const end = Math.min(text.length, i + k.length + 40); - const before = text.slice(start, i); - const hit = text.slice(i, i + k.length); - const after = text.slice(i + k.length, end); - const html = `${ - start > 0 ? "…" : "" - }${before}${hit}${after}${end < text.length ? "…" : ""}`; - return { label, html }; + const lower = text.toLowerCase(); + + // Find the earliest occurrence of ANY matching word — that's the snippet anchor. + let anchor = -1; + let anchorLen = 0; + for (const w of words) { + const i = lower.indexOf(w); + if (i >= 0 && (anchor < 0 || i < anchor)) { + anchor = i; + anchorLen = w.length; + } } + if (anchor < 0) continue; + + const start = Math.max(0, anchor - 40); + const end = Math.min(text.length, anchor + anchorLen + 40); + const slice = text.slice(start, end); + + // Highlight every matching word inside the snippet, not just the first. + const regex = new RegExp( + `(${words.map(escapeRegex).join("|")})`, + "gi" + ); + const highlighted = slice.replace(regex, "$1"); + const html = `${start > 0 ? "…" : ""}${highlighted}${ + end < text.length ? "…" : "" + }`; + return { label, html }; } return null; } @@ -122,19 +149,25 @@ const DatasetCard: React.FC = ({ [parsedJson.value, keyword, visibleHasKeyword] ); - // keyword highlight functional component (only for visible fields) + // keyword highlight functional component (only for visible fields). + // Splits the keyword on whitespace and highlights each word independently + // so "head brain" highlights both words wherever they appear. const highlightKeyword = (text: string, keyword?: string) => { - if (!keyword || !text?.toLowerCase().includes(keyword.toLowerCase())) { - return text; - } - - const regex = new RegExp(`(${keyword})`, "gi"); // for case-insensitive and global + const words = splitKeyword(keyword); + if (words.length === 0 || !text) return text; + const lowerWordSet = new Set(words.map((w) => w.toLowerCase())); + const regex = new RegExp( + `(${words.map(escapeRegex).join("|")})`, + "gi" + ); + if (!regex.test(text)) return text; + // Reset lastIndex because test() advances on /g regexes; safer to use split. const parts = text.split(regex); return ( <> {parts.map((part, i) => - part.toLowerCase() === keyword.toLowerCase() ? ( + lowerWordSet.has(part.toLowerCase()) ? ( Date: Tue, 12 May 2026 16:31:53 -0400 Subject: [PATCH 17/29] fix(search): keyword highlight reads from appliedFilters, not formData --- src/pages/SearchPage.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pages/SearchPage.tsx b/src/pages/SearchPage.tsx index 260931c..ca63bed 100644 --- a/src/pages/SearchPage.tsx +++ b/src/pages/SearchPage.tsx @@ -1002,7 +1002,7 @@ const SearchPage: React.FC = () => { dsname={item.dsname} parsedJson={parsedJson} onChipClick={handleChipClick} - keyword={formData.keyword} // for keyword highlight + keyword={appliedFilters.keyword} // highlight what was searched, not the live form /> ) : ( Date: Wed, 13 May 2026 15:26:18 -0400 Subject: [PATCH 18/29] feat: add file download endpoints for dataset search results --- backend/src/controllers/couchdb.controller.js | 76 ++++++++++++++++++- backend/src/routes/dbs.routes.js | 5 ++ 2 files changed, 80 insertions(+), 1 deletion(-) diff --git a/backend/src/controllers/couchdb.controller.js b/backend/src/controllers/couchdb.controller.js index df1532b..db642da 100644 --- a/backend/src/controllers/couchdb.controller.js +++ b/backend/src/controllers/couchdb.controller.js @@ -257,6 +257,32 @@ const searchAllDatabases = async (req, res) => { repl.limit = limit; repl.offset = offset; + // When file_type filter is active, also return a sample of the actual + // matching iolinks rows (filename, url, path, suffix) per dataset, plus + // a total count. Frontend shows up to 20 as clickable filenames and a + // "Download manifest" button for the full list via a separate endpoint. + const matchingFilesActive = + Array.isArray(f.file_type) && f.file_type.length > 0; + const matchingFilesColumn = matchingFilesActive + ? `, + COALESCE(( + SELECT jsonb_agg(t.json) + FROM ( + SELECT l.json + FROM iolinks l + WHERE l.dbname = ioviews.dbname + AND l.dsname = ioviews.dsname + AND l.view IN (:fileTypes) + ORDER BY l.id + LIMIT 20 + ) t + ), '[]'::jsonb)::text AS matching_files, + (SELECT COUNT(*) FROM iolinks l + WHERE l.dbname = ioviews.dbname + AND l.dsname = ioviews.dsname + AND l.view IN (:fileTypes))::int AS matching_files_total` + : ""; + // dbinfo was stored flat ({name, subj, ...}); subjects was stored wrapped // ({key, value}). Frontend expects parsed.value.subj for datasets, so we // wrap dbinfo on the way out. @@ -268,7 +294,7 @@ const searchAllDatabases = async (req, res) => { CASE WHEN view = 'dbinfo' THEN jsonb_build_object('value', json)::text ELSE json::text - END AS json + END AS json${matchingFilesColumn} FROM ioviews WHERE ${where.join(" AND ")} ORDER BY dbname, dsname, subj @@ -417,6 +443,53 @@ const getDatasetMeta = async (req, res) => { // } +// Plain-text manifest of every matching iolinks URL for a dataset, served +// as a downloadable .txt. The user pipes it into wget/aria2c to fetch +// everything: `wget -i manifest.txt`. Avoids server-side zipping and gives +// resumable, parallel downloads. +const getDatasetFilesManifest = async (req, res) => { + try { + const { dbName, dsName } = req.params; + const rawExt = req.query.ext; + const exts = Array.isArray(rawExt) + ? rawExt + : typeof rawExt === "string" && rawExt.length > 0 + ? rawExt.split(",") + : []; + + if (exts.length === 0) { + res.status(400).send("ext query parameter required (e.g. ?ext=.jdb)"); + return; + } + + const rows = await sequelize.query( + `SELECT json->'value'->>'url' AS url + FROM iolinks + WHERE dbname = :dbname + AND dsname = :dsname + AND view IN (:exts) + ORDER BY id`, + { + replacements: { dbname: dbName, dsname: dsName, exts }, + type: sequelize.QueryTypes.SELECT, + } + ); + + const urls = rows.map((r) => r.url).filter(Boolean); + const filename = `${dbName}_${dsName}_${exts.join("_")}_manifest.txt`; + + res.setHeader("Content-Type", "text/plain; charset=utf-8"); + res.setHeader( + "Content-Disposition", + `attachment; filename="${filename}"` + ); + res.send(urls.join("\n") + "\n"); + } catch (error) { + console.error("Error generating manifest:", error.message); + res.status(500).send(`Error generating manifest: ${error.message}`); + } +}; + // distinct file extensions present in iolinks across all synced DBs. // Drives the multi-select "File types" filter on the search page. const getFileTypes = async (req, res) => { @@ -447,4 +520,5 @@ module.exports = { getDatasetDetail, getDatasetMeta, getFileTypes, + getDatasetFilesManifest, }; diff --git a/backend/src/routes/dbs.routes.js b/backend/src/routes/dbs.routes.js index 621b32f..45979ac 100644 --- a/backend/src/routes/dbs.routes.js +++ b/backend/src/routes/dbs.routes.js @@ -7,6 +7,7 @@ const { getDbDatasets, searchAllDatabases, getFileTypes, + getDatasetFilesManifest, // searchDatabase, } = require("../controllers/couchdb.controller"); @@ -24,6 +25,10 @@ router.get("/file-types", getFileTypes); // cross-database search router.post("/search", searchAllDatabases); +// downloadable manifest (plain text) of all iolinks URLs for a dataset +// filtered by extension(s). e.g. /dbs/bfnirs/Motion-Yucel2014-I/files/manifest?ext=.jdb +router.get("/:dbName/:dsName/files/manifest", getDatasetFilesManifest); + // Specific database routes router.get("/:dbName", getDbInfo); router.get("/:dbName/datasets", getDbDatasets); From 51960cf2274aacf95c4864da675809b74aadd925 Mon Sep 17 00:00:00 2001 From: elainefan331 Date: Wed, 13 May 2026 15:45:06 -0400 Subject: [PATCH 19/29] feat(search): show matching files in dataset card with selective and manifest download --- backend/src/controllers/couchdb.controller.js | 4 +- src/components/SearchPage/DatasetCard.tsx | 145 +++++++++++++++++- src/pages/SearchPage.tsx | 7 + 3 files changed, 153 insertions(+), 3 deletions(-) diff --git a/backend/src/controllers/couchdb.controller.js b/backend/src/controllers/couchdb.controller.js index db642da..20a7cae 100644 --- a/backend/src/controllers/couchdb.controller.js +++ b/backend/src/controllers/couchdb.controller.js @@ -259,7 +259,7 @@ const searchAllDatabases = async (req, res) => { // When file_type filter is active, also return a sample of the actual // matching iolinks rows (filename, url, path, suffix) per dataset, plus - // a total count. Frontend shows up to 20 as clickable filenames and a + // a total count. Frontend shows up to 10 as clickable filenames and a // "Download manifest" button for the full list via a separate endpoint. const matchingFilesActive = Array.isArray(f.file_type) && f.file_type.length > 0; @@ -274,7 +274,7 @@ const searchAllDatabases = async (req, res) => { AND l.dsname = ioviews.dsname AND l.view IN (:fileTypes) ORDER BY l.id - LIMIT 20 + LIMIT 10 ) t ), '[]'::jsonb)::text AS matching_files, (SELECT COUNT(*) FROM iolinks l diff --git a/src/components/SearchPage/DatasetCard.tsx b/src/components/SearchPage/DatasetCard.tsx index 5790ee5..da4e0c7 100644 --- a/src/components/SearchPage/DatasetCard.tsx +++ b/src/components/SearchPage/DatasetCard.tsx @@ -1,10 +1,31 @@ -import { Typography, Card, CardContent, Stack, Chip } from "@mui/material"; +import DownloadIcon from "@mui/icons-material/Download"; +import { + Typography, + Card, + CardContent, + Stack, + Chip, + Button, + Link as MuiLink, +} from "@mui/material"; +import { baseURL } from "services/instance"; import { Colors } from "design/theme"; import React from "react"; import { useMemo } from "react"; import { Link } from "react-router-dom"; import RoutesEnum from "types/routes.enum"; +interface MatchingFile { + key?: any; + value?: { + file?: string; + url?: string; + path?: string; + suffix?: string; + ref?: string; + }; +} + interface DatasetCardProps { dbname: string; dsname: string; @@ -26,6 +47,9 @@ interface DatasetCardProps { index: number; onChipClick: (key: string, value: string) => void; keyword?: string; // for keyword highlight + matchingFiles?: MatchingFile[]; // sample of iolinks rows matching file_type + matchingFilesTotal?: number; // total count across all matches + fileTypes?: string[]; // the active file_type filter, used to build manifest URL } /** ---------- utility helpers ---------- **/ @@ -119,10 +143,41 @@ const DatasetCard: React.FC = ({ index, onChipClick, keyword, + matchingFiles, + matchingFilesTotal, + fileTypes, }) => { const { name, readme, modality, subj, info } = parsedJson.value; const datasetLink = `${RoutesEnum.DATABASES}/${dbname}/${dsname}`; + // Manifest URL — backend serves a plain-text list of all matching URLs. + const manifestUrl = useMemo(() => { + if (!fileTypes || fileTypes.length === 0) return null; + const ext = fileTypes + .map((e) => encodeURIComponent(e)) + .join(","); + return `${baseURL}/dbs/${encodeURIComponent( + dbname + )}/${encodeURIComponent(dsname)}/files/manifest?ext=${ext}`; + }, [dbname, dsname, fileTypes]); + + // Extract a short "sub-XXX" tag from a BIDS path like + // "$.sub-019.ses-1.nirs.sub-019_ses-1_task-MA_run-01_nirs.snirf.SNIRFData..." + const subjectFromPath = (p?: string): string => { + if (!p) return ""; + const m = p.match(/sub-[^.]+/); + return m ? m[0] : ""; + }; + + // File size stored in key[1] of each iolinks row (bytes). Format for humans. + const formatBytes = (n?: number): string => { + if (typeof n !== "number" || !Number.isFinite(n) || n < 0) return ""; + if (n < 1024) return `${n} B`; + if (n < 1024 * 1024) return `${(n / 1024).toFixed(1)} KB`; + if (n < 1024 * 1024 * 1024) return `${(n / 1024 / 1024).toFixed(1)} MB`; + return `${(n / 1024 / 1024 / 1024).toFixed(2)} GB`; + }; + // prepare DOI URL const rawDOI = info?.DatasetDOI?.replace(/^doi:/, ""); const doiLink = rawDOI ? `https://doi.org/${rawDOI}` : null; @@ -338,6 +393,94 @@ const DatasetCard: React.FC = ({ )} + + {/* Matching files section — only shown when file_type filter is active */} + {Array.isArray(matchingFiles) && matchingFiles.length > 0 && ( + + + + Matching files + {typeof matchingFilesTotal === "number" && + ` (${ + matchingFiles.length < matchingFilesTotal + ? `${matchingFiles.length} of ${matchingFilesTotal}` + : matchingFilesTotal + })`} + + {manifestUrl && ( + + )} + + + {matchingFiles.slice(0, 10).map((f, i) => { + const v = f.value || {}; + const subjTag = subjectFromPath(v.path); + const sizeBytes = + Array.isArray(f.key) && typeof f.key[1] === "number" + ? f.key[1] + : undefined; + const sizeTag = formatBytes(sizeBytes); + const meta = [subjTag, sizeTag].filter(Boolean).join(" · "); + return ( +
  • + + {v.file || v.url} + + {meta && ( + + ({meta}) + + )} +
    • + ); + })} +     +     + )} diff --git a/src/pages/SearchPage.tsx b/src/pages/SearchPage.tsx index ca63bed..397e63a 100644 --- a/src/pages/SearchPage.tsx +++ b/src/pages/SearchPage.tsx @@ -1003,6 +1003,13 @@ const SearchPage: React.FC = () => { parsedJson={parsedJson} onChipClick={handleChipClick} keyword={appliedFilters.keyword} // highlight what was searched, not the live form + matchingFiles={ + item.matching_files + ? JSON.parse(item.matching_files) + : undefined + } + matchingFilesTotal={item.matching_files_total} + fileTypes={appliedFilters.file_type} /> ) : ( Date: Wed, 13 May 2026 16:14:45 -0400 Subject: [PATCH 20/29] fix(search): use plainto_tsquery + normalize ILIKE separators for keyword --- backend/src/controllers/couchdb.controller.js | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/backend/src/controllers/couchdb.controller.js b/backend/src/controllers/couchdb.controller.js index 20a7cae..cdcedd3 100644 --- a/backend/src/controllers/couchdb.controller.js +++ b/backend/src/controllers/couchdb.controller.js @@ -222,18 +222,23 @@ const searchAllDatabases = async (req, res) => { } // Keyword search — match anywhere relevant. - // tsquery covers stemmed tokens inside the JSON content (name, readme, - // info, modality, subj). ILIKE on dbname/dsname adds substring matching - // so "fnirs" finds "bfnirs", "openfnirs", and any dataset id containing it. + // plainto_tsquery treats input as plain words AND'd together; ignores + // operator chars like "-" and "OR" so dataset names with hyphens + // (e.g. "ABIDE - CMU_a") don't get parsed as NOT clauses. + // ILIKE on dbname/dsname adds substring matching so "fnirs" finds + // "bfnirs", "openfnirs", and any dataset id containing it. + // ILIKE pattern normalizes whitespace/hyphens to % wildcards so the + // user's "ABIDE - CMU_a" matches stored names like "abide_cmu_a" or + // "ABIDE_-_CMU_a" regardless of separator style. // The whole group is parenthesised so it ANDs cleanly with other filters. if (isFilter(f.keyword)) { where.push(`( - search_vector @@ websearch_to_tsquery('english', :keyword) + search_vector @@ plainto_tsquery('english', :keyword) OR dbname ILIKE :keywordLike OR dsname ILIKE :keywordLike )`); repl.keyword = String(f.keyword); - repl.keywordLike = `%${String(f.keyword)}%`; + repl.keywordLike = `%${String(f.keyword).replace(/[\s-]+/g, "%")}%`; } // File-type filter — array of extensions like [".jdb", ".snirf"]. From bf1abe34995b9e28692165aa38b07718fdce90c6 Mon Sep 17 00:00:00 2001 From: elainefan331 Date: Wed, 13 May 2026 16:31:14 -0400 Subject: [PATCH 21/29] fix(search): match dataset display name via json->>'name' ILIKE --- backend/src/controllers/couchdb.controller.js | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/backend/src/controllers/couchdb.controller.js b/backend/src/controllers/couchdb.controller.js index cdcedd3..88e96f1 100644 --- a/backend/src/controllers/couchdb.controller.js +++ b/backend/src/controllers/couchdb.controller.js @@ -227,15 +227,19 @@ const searchAllDatabases = async (req, res) => { // (e.g. "ABIDE - CMU_a") don't get parsed as NOT clauses. // ILIKE on dbname/dsname adds substring matching so "fnirs" finds // "bfnirs", "openfnirs", and any dataset id containing it. - // ILIKE pattern normalizes whitespace/hyphens to % wildcards so the - // user's "ABIDE - CMU_a" matches stored names like "abide_cmu_a" or - // "ABIDE_-_CMU_a" regardless of separator style. + // ILIKE on json->>'name' covers the human-readable name from + // dataset_description.json (e.g. "ABIDE - CMU_a"), which is where the + // user-visible dataset titles live — dsname column often stores just + // an opaque id like "CMU_a" without the prefix. + // ILIKE pattern normalizes whitespace/hyphens to % wildcards so + // "ABIDE - CMU_a" matches stored names regardless of separator style. // The whole group is parenthesised so it ANDs cleanly with other filters. if (isFilter(f.keyword)) { where.push(`( search_vector @@ plainto_tsquery('english', :keyword) OR dbname ILIKE :keywordLike OR dsname ILIKE :keywordLike + OR (json->>'name') ILIKE :keywordLike )`); repl.keyword = String(f.keyword); repl.keywordLike = `%${String(f.keyword).replace(/[\s-]+/g, "%")}%`; From 6fce0633ef5b639c863302ff50c659386d0ce15b Mon Sep 17 00:00:00 2001 From: elainefan331 Date: Thu, 14 May 2026 16:56:12 -0400 Subject: [PATCH 22/29] feat(search): add Mac/Linux and Windows script options to file download --- backend/src/controllers/couchdb.controller.js | 86 ++++++++- src/components/SearchPage/DatasetCard.tsx | 177 +++++++++++++++--- 2 files changed, 229 insertions(+), 34 deletions(-) diff --git a/backend/src/controllers/couchdb.controller.js b/backend/src/controllers/couchdb.controller.js index 88e96f1..c990fe7 100644 --- a/backend/src/controllers/couchdb.controller.js +++ b/backend/src/controllers/couchdb.controller.js @@ -452,14 +452,18 @@ const getDatasetMeta = async (req, res) => { // } -// Plain-text manifest of every matching iolinks URL for a dataset, served -// as a downloadable .txt. The user pipes it into wget/aria2c to fetch -// everything: `wget -i manifest.txt`. Avoids server-side zipping and gives -// resumable, parallel downloads. +// Downloadable list of every matching iolinks URL for a dataset. +// Three formats via ?format=: +// - txt (default) → plain URL list (use with `wget -i`) +// - sh → bash script with curl commands (Mac/Linux) +// - bat → Windows batch script with curl commands +// All three avoid server-side zipping — the user's machine pulls files +// directly from neurojson.org/io, so this Express server stays light. const getDatasetFilesManifest = async (req, res) => { try { const { dbName, dsName } = req.params; const rawExt = req.query.ext; + const format = String(req.query.format || "txt").toLowerCase(); const exts = Array.isArray(rawExt) ? rawExt : typeof rawExt === "string" && rawExt.length > 0 @@ -472,7 +476,8 @@ const getDatasetFilesManifest = async (req, res) => { } const rows = await sequelize.query( - `SELECT json->'value'->>'url' AS url + `SELECT json->'value'->>'url' AS url, + json->'value'->>'file' AS file FROM iolinks WHERE dbname = :dbname AND dsname = :dsname @@ -484,15 +489,78 @@ const getDatasetFilesManifest = async (req, res) => { } ); - const urls = rows.map((r) => r.url).filter(Boolean); - const filename = `${dbName}_${dsName}_${exts.join("_")}_manifest.txt`; + const files = rows.filter((r) => r.url); + const urls = files.map((r) => r.url); + const baseName = `${dbName}_${dsName}_${exts.join("_")}`; + const extLabel = exts.join(", "); + + // Strip any path separators or quote chars from the parsed filename + // before using it in shell commands — file names come from iolinks + // and are usually content hashes, but defensive belt-and-suspenders. + const safeName = (s) => + (s || "").replace(/["\\\/\r\n]/g, "").trim(); + + let body; + let contentType; + let filename; + + if (format === "sh") { + // Bash script — curl is preinstalled on macOS and most Linux distros. + // -L follows redirects, -C - resumes interrupted downloads, -o saves + // with our parsed filename (the URL is a CGI query — using -O would + // save files as literal `stat.cgi?...`). + body = + `#!/bin/bash\n` + + `# Downloads ${extLabel} files from ${dbName}/${dsName}\n` + + `# Usage: bash ${baseName}_download.sh\n` + + `set -e\n` + + `mkdir -p "neurojson_downloads"\n` + + `cd "neurojson_downloads" || exit 1\n` + + files + .map((r) => { + const fn = safeName(r.file); + return fn + ? `curl -L -C - -o "${fn}" "${r.url}"` + : `curl -L -C - -O "${r.url}"`; + }) + .join("\n") + + `\necho "Done. Files saved to $(pwd)"\n`; + contentType = "application/x-sh; charset=utf-8"; + filename = `${baseName}_download.sh`; + } else if (format === "bat") { + // Windows batch — curl ships with Windows 10+. Uses CRLF line endings + // for proper rendering in CMD. /d on cd handles cross-drive paths. + body = + `@echo off\r\n` + + `REM Downloads ${extLabel} files from ${dbName}/${dsName}\r\n` + + `REM Usage: double-click or run ${baseName}_download.bat\r\n` + + `if not exist "neurojson_downloads" mkdir "neurojson_downloads"\r\n` + + `cd /d "neurojson_downloads"\r\n` + + files + .map((r) => { + const fn = safeName(r.file); + return fn + ? `curl -L -C - -o "${fn}" "${r.url}"` + : `curl -L -C - -O "${r.url}"`; + }) + .join("\r\n") + + `\r\necho Done. Files saved to %cd%\r\n` + + `pause\r\n`; + contentType = "text/plain; charset=utf-8"; + filename = `${baseName}_download.bat`; + } else { + // Default: plain URL list, one per line (advanced users with wget). + body = urls.join("\n") + "\n"; + contentType = "text/plain; charset=utf-8"; + filename = `${baseName}_manifest.txt`; + } - res.setHeader("Content-Type", "text/plain; charset=utf-8"); + res.setHeader("Content-Type", contentType); res.setHeader( "Content-Disposition", `attachment; filename="${filename}"` ); - res.send(urls.join("\n") + "\n"); + res.send(body); } catch (error) { console.error("Error generating manifest:", error.message); res.status(500).send(`Error generating manifest: ${error.message}`); diff --git a/src/components/SearchPage/DatasetCard.tsx b/src/components/SearchPage/DatasetCard.tsx index da4e0c7..4243119 100644 --- a/src/components/SearchPage/DatasetCard.tsx +++ b/src/components/SearchPage/DatasetCard.tsx @@ -1,4 +1,5 @@ import DownloadIcon from "@mui/icons-material/Download"; +import KeyboardArrowDownIcon from "@mui/icons-material/KeyboardArrowDown"; import { Typography, Card, @@ -7,11 +8,16 @@ import { Chip, Button, Link as MuiLink, + Menu, + MenuItem, + Box, + Snackbar, + Alert, } from "@mui/material"; import { baseURL } from "services/instance"; import { Colors } from "design/theme"; import React from "react"; -import { useMemo } from "react"; +import { useMemo, useState } from "react"; import { Link } from "react-router-dom"; import RoutesEnum from "types/routes.enum"; @@ -150,16 +156,44 @@ const DatasetCard: React.FC = ({ const { name, readme, modality, subj, info } = parsedJson.value; const datasetLink = `${RoutesEnum.DATABASES}/${dbname}/${dsname}`; - // Manifest URL — backend serves a plain-text list of all matching URLs. - const manifestUrl = useMemo(() => { + // Build manifest URL for any of the three formats. Backend serves + // text/plain for .txt, application/x-sh for .sh, text/plain for .bat — + // each with a Content-Disposition header so the browser saves them. + const buildManifestUrl = (format: "txt" | "sh" | "bat") => { if (!fileTypes || fileTypes.length === 0) return null; - const ext = fileTypes - .map((e) => encodeURIComponent(e)) - .join(","); + const ext = fileTypes.map((e) => encodeURIComponent(e)).join(","); return `${baseURL}/dbs/${encodeURIComponent( dbname - )}/${encodeURIComponent(dsname)}/files/manifest?ext=${ext}`; - }, [dbname, dsname, fileTypes]); + )}/${encodeURIComponent( + dsname + )}/files/manifest?ext=${ext}&format=${format}`; + }; + + const hasManifest = Array.isArray(fileTypes) && fileTypes.length > 0; + + // Dropdown state for the download format menu. + const [downloadMenuEl, setDownloadMenuEl] = useState( + null + ); + // Post-download instruction snackbar. Stays open until user dismisses it + // (no autoHideDuration) so researchers have time to read multi-step + // instructions. + const [downloadHint, setDownloadHint] = useState< + "sh" | "bat" | "txt" | null + >(null); + const handleDownload = (format: "txt" | "sh" | "bat") => { + const url = buildManifestUrl(format); + setDownloadMenuEl(null); + if (!url) return; + // Programmatic anchor click triggers the browser's normal download flow + // without leaving the current page (window.location would navigate away). + const a = document.createElement("a"); + a.href = url; + document.body.appendChild(a); + a.click(); + document.body.removeChild(a); + setDownloadHint(format); + }; // Extract a short "sub-XXX" tag from a BIDS path like // "$.sub-019.ses-1.nirs.sub-019_ses-1_task-MA_run-01_nirs.snirf.SNIRFData..." @@ -421,23 +455,40 @@ const DatasetCard: React.FC = ({ : matchingFilesTotal })`} - {manifestUrl && ( - + {hasManifest && ( + <> + + setDownloadMenuEl(null)} + > + handleDownload("sh")}> + For Mac / Linux (.sh) + + handleDownload("bat")}> + For Windows (.bat) + + handleDownload("txt")}> + URL list (.txt, advanced) + + + )} @@ -483,6 +534,82 @@ const DatasetCard: React.FC = ({ )} + + {/* Post-download instructions. No auto-hide so users can read at their + * own pace; dismiss with the ✕ when finished. */} + setDownloadHint(null)} + anchorOrigin={{ vertical: "bottom", horizontal: "center" }} + > + setDownloadHint(null)} + sx={{ maxWidth: 520 }} + > + {downloadHint === "sh" && ( + + + Downloaded the Mac / Linux script + + + To fetch your data files: + + +
  • Open Terminal
    • +
  • Go to the folder where the script was saved
    • +
  • + Run:{" "} + + bash <script-name>.sh + +
    • +     +     + )} + {downloadHint === "bat" && ( + + + Downloaded the Windows script + + + Open the folder where the script was saved and{" "} + double-click the .bat file. A command window + opens and the files download next to it. + + + )} + {downloadHint === "txt" && ( + + + Downloaded the URL list + + + In Terminal (Mac/Linux) or PowerShell (Windows), run:{" "} + + wget -i <file-name>.txt + + + + )} +     +     ); }; From 8be3fa1af653de11aedc926130f7a054a2d09bf2 Mon Sep 17 00:00:00 2001 From: elainefan331 Date: Fri, 15 May 2026 14:15:49 -0400 Subject: [PATCH 23/29] fix(dataset-detail): keep all downloaded files in one visible folder --- src/pages/UpdatedDatasetDetailPage.tsx | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/src/pages/UpdatedDatasetDetailPage.tsx b/src/pages/UpdatedDatasetDetailPage.tsx index f94a751..734c104 100644 --- a/src/pages/UpdatedDatasetDetailPage.tsx +++ b/src/pages/UpdatedDatasetDetailPage.tsx @@ -577,8 +577,17 @@ const UpdatedDatasetDetailPage: React.FC = () => { // }); // setJsonSize(blob.size); - // Construct download script dynamically - let script = `curl -L --create-dirs "https://neurojson.io:7777/${dbName}/${docId}" -o "${docId}.json"\n`; + // Construct download script dynamically — everything lands in a + // .// folder next to where the user runs the script, so the + // JSON and the data files stay together (was split between cwd and + // ~/.neurojson/io/... previously, hard to find). + let script = `#!/bin/bash\n`; + script += `# Downloads ${docId} from ${dbName}\n`; + script += `# Usage: bash ${docId}.sh\n`; + script += `set -e\n`; + script += `mkdir -p "${docId}"\n`; + script += `cd "${docId}" || exit 1\n`; + script += `curl -L -C - -o "${docId}.json" "https://neurojson.io:7777/${dbName}/${docId}"\n`; links.forEach((link) => { const url = link.url; @@ -594,10 +603,10 @@ const UpdatedDatasetDetailPage: React.FC = () => { })() : `file-${link.index}`; - const outputPath = `$HOME/.neurojson/io/${dbName}/${docId}/${filename}`; - - script += `curl -L --create-dirs "${url}" -o "${outputPath}"\n`; + script += `curl -L -C - -o "${filename}" "${url}"\n`; }); + + script += `echo "Done. Files saved to $(pwd)"\n`; setDownloadScript(script); // Calculate and set script size const scriptBlob = new Blob([script], { type: "text/plain" }); From e21c661760d3061f2b18540043c8952c60a259fa Mon Sep 17 00:00:00 2001 From: elainefan331 Date: Fri, 15 May 2026 14:24:22 -0400 Subject: [PATCH 24/29] feat(dataset-detail): three script formats for download all files button --- src/pages/UpdatedDatasetDetailPage.tsx | 223 ++++++++++++++++++++----- 1 file changed, 183 insertions(+), 40 deletions(-) diff --git a/src/pages/UpdatedDatasetDetailPage.tsx b/src/pages/UpdatedDatasetDetailPage.tsx index 734c104..835a6b5 100644 --- a/src/pages/UpdatedDatasetDetailPage.tsx +++ b/src/pages/UpdatedDatasetDetailPage.tsx @@ -7,6 +7,7 @@ import ExpandLess from "@mui/icons-material/ExpandLess"; import ExpandMore from "@mui/icons-material/ExpandMore"; import HomeIcon from "@mui/icons-material/Home"; import InfoOutlinedIcon from "@mui/icons-material/InfoOutlined"; +import KeyboardArrowDownIcon from "@mui/icons-material/KeyboardArrowDown"; import { Box, Typography, @@ -17,6 +18,9 @@ import { Collapse, Tooltip, IconButton, + Menu, + MenuItem, + Snackbar, } from "@mui/material"; import DatasetActions from "components/DatasetDetailPage/DatasetAction"; import FileTree from "components/DatasetDetailPage/FileTree/FileTree"; @@ -267,8 +271,23 @@ const UpdatedDatasetDetailPage: React.FC = () => { const [externalLinks, setExternalLinks] = useState([]); const [internalLinks, setInternalLinks] = useState([]); const [isInternalExpanded, setIsInternalExpanded] = useState(true); - const [downloadScript, setDownloadScript] = useState(""); + // Three script formats generated client-side: bash (Mac/Linux), batch + // (Windows), and a plain URL list. Same files in all three; only the + // wrapper syntax differs. + const [downloadScripts, setDownloadScripts] = useState<{ + sh: string; + bat: string; + txt: string; + }>({ sh: "", bat: "", txt: "" }); const [downloadScriptSize, setDownloadScriptSize] = useState(0); + // Dropdown state for the download format menu. + const [downloadMenuEl, setDownloadMenuEl] = useState( + null + ); + // Post-download instruction snackbar. Stays open until user dismisses. + const [downloadHint, setDownloadHint] = useState< + "sh" | "bat" | "txt" | null + >(null); const [totalFileSize, setTotalFileSize] = useState(0); const [previewIsInternal, setPreviewIsInternal] = useState(false); const [isExternalExpanded, setIsExternalExpanded] = useState(true); @@ -577,40 +596,62 @@ const UpdatedDatasetDetailPage: React.FC = () => { // }); // setJsonSize(blob.size); - // Construct download script dynamically — everything lands in a - // .// folder next to where the user runs the script, so the - // JSON and the data files stay together (was split between cwd and - // ~/.neurojson/io/... previously, hard to find). - let script = `#!/bin/bash\n`; - script += `# Downloads ${docId} from ${dbName}\n`; - script += `# Usage: bash ${docId}.sh\n`; - script += `set -e\n`; - script += `mkdir -p "${docId}"\n`; - script += `cd "${docId}" || exit 1\n`; - script += `curl -L -C - -o "${docId}.json" "https://neurojson.io:7777/${dbName}/${docId}"\n`; - - links.forEach((link) => { - const url = link.url; - const match = url.match(/file=([^&]+)/); - - const filename = match - ? (() => { - try { - return decodeURIComponent(match[1]); - } catch { - return match[1]; // fallback if decode fails - } - })() - : `file-${link.index}`; - - script += `curl -L -C - -o "${filename}" "${url}"\n`; - }); - - script += `echo "Done. Files saved to $(pwd)"\n`; - setDownloadScript(script); - // Calculate and set script size - const scriptBlob = new Blob([script], { type: "text/plain" }); - setDownloadScriptSize(scriptBlob.size); + // Construct download scripts (three formats) dynamically — everything + // lands in a .// folder next to where the user runs the script. + // JSON metadata and data files stay together (was split between cwd + // and ~/.neurojson/io/... previously, hard to find). + const docUrl = `https://neurojson.io:7777/${dbName}/${docId}`; + type DlItem = { url: string; filename: string }; + const items: DlItem[] = [ + { url: docUrl, filename: `${docId}.json` }, + ...links.map((link) => { + const match = link.url.match(/file=([^&]+)/); + const filename = match + ? (() => { + try { + return decodeURIComponent(match[1]); + } catch { + return match[1]; + } + })() + : `file-${link.index}`; + return { url: link.url, filename }; + }), + ]; + + // Bash script (Mac/Linux) + const sh = + `#!/bin/bash\n` + + `# Downloads ${docId} from ${dbName}\n` + + `# Usage: bash ${docId}.sh\n` + + `set -e\n` + + `mkdir -p "${docId}"\n` + + `cd "${docId}" || exit 1\n` + + items + .map((it) => `curl -L -C - -o "${it.filename}" "${it.url}"`) + .join("\n") + + `\necho "Done. Files saved to $(pwd)"\n`; + + // Batch script (Windows) — curl ships with Windows 10+. CRLF endings. + const bat = + `@echo off\r\n` + + `REM Downloads ${docId} from ${dbName}\r\n` + + `REM Usage: double-click or run ${docId}.bat\r\n` + + `if not exist "${docId}" mkdir "${docId}"\r\n` + + `cd /d "${docId}"\r\n` + + items + .map((it) => `curl -L -C - -o "${it.filename}" "${it.url}"`) + .join("\r\n") + + `\r\necho Done. Files saved to %cd%\r\n` + + `pause\r\n`; + + // Plain URL list — for advanced users with wget. + const txt = items.map((it) => it.url).join("\n") + "\n"; + + setDownloadScripts({ sh, bat, txt }); + // Size shown on the button is the .sh script size (representative). + const shBlob = new Blob([sh], { type: "text/plain" }); + setDownloadScriptSize(shBlob.size); } }, [datasetDocument, docId]); @@ -636,14 +677,25 @@ const UpdatedDatasetDetailPage: React.FC = () => { document.body.removeChild(link); }; - const handleDownloadScript = () => { - const blob = new Blob([downloadScript], { type: "text/plain" }); + // Trigger download of the selected script format. Programmatic anchor + // click triggers the browser's normal download flow without navigating. + const handleDownloadScript = (format: "sh" | "bat" | "txt") => { + const content = downloadScripts[format]; + if (!content) return; + const mime = + format === "sh" ? "application/x-sh" : "text/plain"; + const filename = + format === "txt" ? `${docId}_manifest.txt` : `${docId}.${format}`; + const blob = new Blob([content], { type: `${mime}; charset=utf-8` }); const link = document.createElement("a"); link.href = URL.createObjectURL(blob); - link.download = `${docId}.sh`; + link.download = filename; document.body.appendChild(link); link.click(); document.body.removeChild(link); + URL.revokeObjectURL(link.href); + setDownloadMenuEl(null); + setDownloadHint(format); }; const handlePreview = ( @@ -1127,20 +1179,35 @@ const UpdatedDatasetDetailPage: React.FC = () => { + setDownloadMenuEl(null)} + > + handleDownloadScript("sh")}> + For Mac / Linux (.sh) + + handleDownloadScript("bat")}> + For Windows (.bat) + + handleDownloadScript("txt")}> + URL list (.txt, advanced) + + @@ -1667,6 +1734,82 @@ const UpdatedDatasetDetailPage: React.FC = () => { key={`${previewIndex}-${previewOpen}`} // react will destroy the existing component and create a new one for mount /> + + {/* Post-download instructions. No auto-hide so users can read at + * their own pace; dismiss with the ✕ when finished. */} + setDownloadHint(null)} + anchorOrigin={{ vertical: "bottom", horizontal: "center" }} + > + setDownloadHint(null)} + sx={{ maxWidth: 520 }} + > + {downloadHint === "sh" && ( + + + Downloaded the Mac / Linux script + + + To fetch your data files: + + +
  • Open Terminal
    • +
  • Go to the folder where the script was saved
    • +
  • + Run:{" "} + + bash <script-name>.sh + +
    • +     +     + )} + {downloadHint === "bat" && ( + + + Downloaded the Windows script + + + Open the folder where the script was saved and{" "} + double-click the .bat file. A command + window opens and the files download next to it. + + + )} + {downloadHint === "txt" && ( + + + Downloaded the URL list + + + In Terminal (Mac/Linux) or PowerShell (Windows), run:{" "} + + wget -i <file-name>.txt + + + + )} +     +     ); }; From 909cd36172343cf1b86492bae3c5d70174ce2558 Mon Sep 17 00:00:00 2001 From: elainefan331 Date: Wed, 20 May 2026 16:39:16 -0400 Subject: [PATCH 25/29] feat(sync): pull database list from registry --- backend/sync/incrementalSync.js | 48 +++++++-------------------------- 1 file changed, 9 insertions(+), 39 deletions(-) diff --git a/backend/sync/incrementalSync.js b/backend/sync/incrementalSync.js index b9e497a..55c5e90 100644 --- a/backend/sync/incrementalSync.js +++ b/backend/sync/incrementalSync.js @@ -8,34 +8,13 @@ const COUCHDB_URL = process.env.COUCHDB_URL || "https://neurojson.io:7777"; const CONCURRENCY = 5; // fetch database list dynamically from registry +// registry doc shape: { database: [{ id, name, ... }, ...] } async function getDatabases() { - try { - const response = await axios.get(`${COUCHDB_URL}/sys/registry`); - const databases = response.data - .map((db) => db.id) - .filter((id) => id && id !== "sys"); - console.log(`Found ${databases.length} databases in registry`); - return databases; - } catch (err) { - console.error("Failed to fetch registry:", err.message); - return [ - "openneuro", - "abide", - "abide2", - "datalad-registry", - "adhd200", - "bfnirs", - "mcx", - "mmc", - "ucl-4d-neonatal-head-model", - "unc-012-infant-atlas", - "unc-infant-cortical-surface-atlas", - "cotilab", - "emnist", - "nemo-bids", - "openfnirs", - ]; - } + const response = await axios.get(`${COUCHDB_URL}/sys/registry`); + const entries = response.data?.database || []; + const databases = entries.map((db) => db.id).filter(Boolean); + console.log(`Found ${databases.length} databases in registry`); + return databases; } // === Local ports of CouchDB _design/qq map functions === @@ -43,8 +22,7 @@ async function getDatabases() { // these drift silently. function transformDbinfo(doc) { - const txt = - doc["README"] || doc["README.md"] || doc["README.rst"] || ""; + const txt = doc["README"] || doc["README.md"] || doc["README.rst"] || ""; const rawtext = JSON.stringify(doc); const datainfo = doc["dataset_description.json"] || { Name: doc._id }; const subjlist = []; @@ -541,9 +519,7 @@ async function incrementalSync(dbname, lastSeq) { await processDatasetUpdate(dbname, change.id); } } catch (err) { - console.error( - ` ${dbname}/${change.id}: failed - ${err.message}` - ); + console.error(` ${dbname}/${change.id}: failed - ${err.message}`); } }) ); @@ -587,13 +563,7 @@ async function runSync() { console.log(new Date().toISOString()); console.log(`CouchDB: ${COUCHDB_URL}`); - // change to await getDatabases() when ready for full sync - const databases = [ - "bfnirs", // NIRS — .snirf, .jdb - "brainmeshlibrary", // mesh + atlas — .jmsh, .jnii (318 datasets) - "cotilab", // JData — small (6 datasets) - "abide", // BIDS MRI — .nii.gz, .tsv, .json (25 datasets) - ]; + const databases = await getDatabases(); console.log(`Databases: ${databases.length}`); for (const db of databases) { From 0cee8b6c221002d6d49c3f7748f0159d62906585 Mon Sep 17 00:00:00 2001 From: elainefan331 Date: Thu, 21 May 2026 11:43:56 -0400 Subject: [PATCH 26/29] feat(search): add dataset-level modality filter with AND/OR mode --- backend/src/controllers/couchdb.controller.js | 24 +++++ backend/sync/incrementalSync.js | 14 ++- src/pages/SearchPage.tsx | 97 ++++++++++++++++++- .../SearchPageFunctions/generateUiSchema.ts | 8 ++ .../SearchPageFunctions/searchformSchema.ts | 15 +++ 5 files changed, 151 insertions(+), 7 deletions(-) diff --git a/backend/src/controllers/couchdb.controller.js b/backend/src/controllers/couchdb.controller.js index c990fe7..3700fc8 100644 --- a/backend/src/controllers/couchdb.controller.js +++ b/backend/src/controllers/couchdb.controller.js @@ -207,6 +207,30 @@ const searchAllDatabases = async (req, res) => { repl.modality = mod; } + // Dataset-level modality filter (multi-select + AND/OR). + // Queries json->'modality' on dbinfo rows, not subjects rows. + if (Array.isArray(f.modalities) && f.modalities.length > 0) { + const op = f.modality_mode === "and" ? " AND " : " OR "; + const parts = f.modalities.map((m, i) => { + repl[`dmod${i}`] = String(m); + return isSubjectSearch + ? `dsi.json->'modality' ? :dmod${i}` + : `json->'modality' ? :dmod${i}`; + }); + const condition = `(${parts.join(op)})`; + if (isSubjectSearch) { + where.push(`EXISTS ( + SELECT 1 FROM ioviews dsi + WHERE dsi.dbname = ioviews.dbname + AND dsi.dsname = ioviews.dsname + AND dsi.view = 'dbinfo' + AND ${condition} + )`); + } else { + where.push(condition); + } + } + // db / ds / subj filters if (isFilter(f.database)) { where.push(`dbname = :dbname`); diff --git a/backend/sync/incrementalSync.js b/backend/sync/incrementalSync.js index 55c5e90..50fbad5 100644 --- a/backend/sync/incrementalSync.js +++ b/backend/sync/incrementalSync.js @@ -303,7 +303,15 @@ async function saveLastSeq(dbname, seq) { ); } +// Postgres jsonb rejects the null-byte escape with "unsupported Unicode +// escape sequence", so strip it from the serialized JSON before insert. +// Seen in openneuro README/TSV fields containing stray null bytes. +function safeStringify(obj) { + return JSON.stringify(obj).replace(/\\u0000/g, ""); +} + async function upsertIoview(dbname, dsname, subj, view, json, transaction) { + const payload = safeStringify(json); await sequelize.query( `INSERT INTO ioviews (dbname, dsname, subj, view, json, search_vector, updated_at) VALUES (:dbname, :dsname, :subj, :view, :json, to_tsvector('english', :text), NOW()) @@ -317,8 +325,8 @@ async function upsertIoview(dbname, dsname, subj, view, json, transaction) { dsname, subj: String(subj), view, - json: JSON.stringify(json), - text: JSON.stringify(json), + json: payload, + text: payload, }, transaction, } @@ -335,7 +343,7 @@ async function insertIolink(dbname, dsname, subj, view, json, transaction) { dsname, subj: String(subj), view, - json: JSON.stringify(json), + json: safeStringify(json), }, transaction, } diff --git a/src/pages/SearchPage.tsx b/src/pages/SearchPage.tsx index 397e63a..ddd5137 100644 --- a/src/pages/SearchPage.tsx +++ b/src/pages/SearchPage.tsx @@ -16,6 +16,11 @@ import { Slider, Stack, TextField, + ToggleButton, + ToggleButtonGroup, + FormGroup, + FormControlLabel, + Checkbox, } from "@mui/material"; import { useTheme } from "@mui/material/styles"; import useMediaQuery from "@mui/material/useMediaQuery"; @@ -57,6 +62,80 @@ type RegistryItem = { const AGE_MIN_BOUND = 0; const AGE_MAX_BOUND = 100; +const DATASET_MODALITIES = [ + "anat", "func", "dwi", "fmap", "perf", + "meg", "eeg", "ieeg", "beh", "pet", + "micr", "nirs", "motion", "ephys", "atlas", + "JMesh", "JNIFTI", "JSNIRF", "JData", +]; + +const DatasetModalityFilterField = (props: any) => { + const ctx = props?.registry?.formContext as + | { formData: Record; setFormData: React.Dispatch>> } + | undefined; + if (!ctx) return null; + const { formData, setFormData } = ctx; + const selected: string[] = Array.isArray(formData.modalities) ? formData.modalities : []; + const mode: string = formData.modality_mode || "or"; + + const toggle = (code: string) => { + setFormData((prev) => { + const cur: string[] = Array.isArray(prev.modalities) ? prev.modalities : []; + const next = cur.includes(code) ? cur.filter((m) => m !== code) : [...cur, code]; + const updated = { ...prev }; + if (next.length === 0) { + delete updated.modalities; + delete updated.modality_mode; + } else { + updated.modalities = next; + if (!updated.modality_mode) updated.modality_mode = "or"; + } + return updated; + }); + }; + + const handleModeChange = (_: any, val: string | null) => { + if (!val) return; + setFormData((prev) => ({ ...prev, modality_mode: val })); + }; + + return ( + 0 ? "#e8f4fd" : "transparent" }}> + + Dataset modalities + + + {DATASET_MODALITIES.map((code) => ( + toggle(code)} + sx={{ py: 0.25 }} + /> + } + label={{code}} + sx={{ mr: 1 }} + /> + ))} + + {selected.length > 1 && ( + + + OR + AND + + + {mode === "and" ? "must have all selected" : "must have any selected"} + + + )} + + ); +}; + const AgeRangeSliderField = (props: any) => { const ctx = props?.registry?.formContext as | { @@ -239,10 +318,12 @@ const SearchPage: React.FC = () => { ([key, value]) => key !== "skip" && key !== "limit" && + key !== "modality_mode" && value !== undefined && value !== null && value !== "" && - value !== "any" + value !== "any" && + !(Array.isArray(value) && value.length === 0) ); useEffect(() => { @@ -381,6 +462,7 @@ const SearchPage: React.FC = () => { ), ageRangeSlider: AgeRangeSliderField, countRangePair: CountRangePairField, + datasetModalityFilter: DatasetModalityFilterField, }; // determine the results are subject-level or dataset-level @@ -685,10 +767,16 @@ const SearchPage: React.FC = () => { mt: 1, }} > - {activeFilters.map(([key, value]) => ( + {activeFilters.map(([key, value]) => { + let label = `${String(key)}: ${String(value)}`; + if (key === "modalities" && Array.isArray(value)) { + const mode = appliedFilters.modality_mode || "or"; + label = `modalities (${mode}): ${value.join(", ")}`; + } + return ( { } }} /> - ))} + ); + })} )} diff --git a/src/utils/SearchPageFunctions/generateUiSchema.ts b/src/utils/SearchPageFunctions/generateUiSchema.ts index 152a056..46dc215 100644 --- a/src/utils/SearchPageFunctions/generateUiSchema.ts +++ b/src/utils/SearchPageFunctions/generateUiSchema.ts @@ -51,6 +51,9 @@ export const generateUiSchema = ( "database", "keyword", "file_type", // dataset-level: filters by file extensions in iolinks + "dataset_modality_filter", // dataset-level: modality multi-select + AND/OR + "modalities", + "modality_mode", "subject_filters_toggle", "age_range_slider", // top of subject filters — range slider for age "modality", @@ -106,6 +109,11 @@ export const generateUiSchema = ( }, } : datasetHiddenStyle, + dataset_modality_filter: showDatasetFilters + ? { "ui:field": "datasetModalityFilter" } + : datasetHiddenStyle, + modalities: invisibleStyle, + modality_mode: invisibleStyle, limit: invisibleStyle, skip: invisibleStyle, diff --git a/src/utils/SearchPageFunctions/searchformSchema.ts b/src/utils/SearchPageFunctions/searchformSchema.ts index e122725..3674cdd 100644 --- a/src/utils/SearchPageFunctions/searchformSchema.ts +++ b/src/utils/SearchPageFunctions/searchformSchema.ts @@ -134,6 +134,21 @@ export const baseSchema: JSONSchema7 = { items: { type: "string" }, uniqueItems: true, }, + dataset_modality_filter: { + type: "null", + title: "", + }, + modalities: { + type: "array", + title: "Dataset modalities", + items: { type: "string" }, + uniqueItems: true, + }, + modality_mode: { + type: "string", + title: "Modality match mode", + default: "or", + }, session_name: { title: "Session keywords", type: "string", From 601d90be7e248ed6e1b977883a6f456f715ab8e3 Mon Sep 17 00:00:00 2001 From: elainefan331 Date: Fri, 22 May 2026 10:10:44 -0400 Subject: [PATCH 27/29] feat(search): add tooltip to Subject-Level Filters explaining result type --- src/pages/SearchPage.tsx | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/src/pages/SearchPage.tsx b/src/pages/SearchPage.tsx index ddd5137..765e308 100644 --- a/src/pages/SearchPage.tsx +++ b/src/pages/SearchPage.tsx @@ -427,7 +427,7 @@ const SearchPage: React.FC = () => { // Create the "Subject-level Filters" button as a custom field const customFields = { subjectFiltersToggle: () => ( - + + + + + + ), datasetFiltersToggle: () => ( From 5aab8089f330c52d3ce592af07ba8f2152f04815 Mon Sep 17 00:00:00 2001 From: elainefan331 Date: Fri, 22 May 2026 10:24:34 -0400 Subject: [PATCH 28/29] fix(sync): reject malformed file extensions from CouchDB links view --- backend/sync/incrementalSync.js | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/backend/sync/incrementalSync.js b/backend/sync/incrementalSync.js index 50fbad5..766f1a5 100644 --- a/backend/sync/incrementalSync.js +++ b/backend/sync/incrementalSync.js @@ -310,6 +310,19 @@ function safeStringify(obj) { return JSON.stringify(obj).replace(/\\u0000/g, ""); } +// A valid file type is a dot-prefixed extension with no slashes and +// a reasonable length. Some CouchDB links view rows (e.g. openneuro) +// emit paths like ".0/libraries/FID-A/..." where the version number +// gets parsed as a fake extension — reject those. +function isValidFileType(ext) { + return ( + typeof ext === "string" && + ext.startsWith(".") && + !ext.includes("/") && + ext.length <= 20 + ); +} + async function upsertIoview(dbname, dsname, subj, view, json, transaction) { const payload = safeStringify(json); await sequelize.query( @@ -401,15 +414,18 @@ async function firstSync(dbname) { console.log(` ${dbname}: subjects synced (${subjectRows.length} rows)`); const linkRows = await fetchView(dbname, "links"); + let linkCount = 0; for (const row of linkRows) { const fileType = row.key?.[0]; + if (!isValidFileType(fileType)) continue; const subjId = String(row.key?.[1] || ""); await insertIolink(dbname, row.id, subjId, fileType, { key: row.key, value: row.value, }); + linkCount++; } - console.log(` ${dbname}: links synced (${linkRows.length} rows)`); + console.log(` ${dbname}: links synced (${linkCount}/${linkRows.length} rows)`); } // === Process one changed dataset (Option A: 2 HTTP requests + local transforms) === From f834a4740a5d912203d3b8ebe791049de23f3abe Mon Sep 17 00:00:00 2001 From: elainefan331 Date: Mon, 25 May 2026 07:34:16 -0400 Subject: [PATCH 29/29] style(search): match dataset modality filter UI to file types field --- src/pages/SearchPage.tsx | 54 +++++++++++++++++++++------------------- 1 file changed, 28 insertions(+), 26 deletions(-) diff --git a/src/pages/SearchPage.tsx b/src/pages/SearchPage.tsx index 765e308..df49761 100644 --- a/src/pages/SearchPage.tsx +++ b/src/pages/SearchPage.tsx @@ -18,9 +18,7 @@ import { TextField, ToggleButton, ToggleButtonGroup, - FormGroup, - FormControlLabel, - Checkbox, + Autocomplete, } from "@mui/material"; import { useTheme } from "@mui/material/styles"; import useMediaQuery from "@mui/material/useMediaQuery"; @@ -78,10 +76,8 @@ const DatasetModalityFilterField = (props: any) => { const selected: string[] = Array.isArray(formData.modalities) ? formData.modalities : []; const mode: string = formData.modality_mode || "or"; - const toggle = (code: string) => { + const handleChange = (_: any, next: string[]) => { setFormData((prev) => { - const cur: string[] = Array.isArray(prev.modalities) ? prev.modalities : []; - const next = cur.includes(code) ? cur.filter((m) => m !== code) : [...cur, code]; const updated = { ...prev }; if (next.length === 0) { delete updated.modalities; @@ -100,27 +96,33 @@ const DatasetModalityFilterField = (props: any) => { }; return ( - 0 ? "#e8f4fd" : "transparent" }}> - - Dataset modalities - - - {DATASET_MODALITIES.map((code) => ( - toggle(code)} - sx={{ py: 0.25 }} - /> - } - label={{code}} - sx={{ mr: 1 }} + + + items.map((item, index) => ( + + )) + } + renderInput={(params) => ( + - ))} - + )} + /> {selected.length > 1 && (