diff --git a/.github/workflows/build-deploy-zodiac.yml b/.github/workflows/build-deploy-zodiac.yml index 57a049d..d6d86e6 100644 --- a/.github/workflows/build-deploy-zodiac.yml +++ b/.github/workflows/build-deploy-zodiac.yml @@ -18,6 +18,9 @@ jobs: - name: Check out the repository uses: actions/checkout@v3 + - name: Check IP + run: curl https://api.ipify.org + - name: Install dependencies run: yarn install --frozen-lockfile diff --git a/backend/config/config.js b/backend/config/config.js index 7c15ed9..0e7d92a 100644 --- a/backend/config/config.js +++ b/backend/config/config.js @@ -1,10 +1,19 @@ require("dotenv").config(); module.exports = { + // development: { + // dialect: "sqlite", + // storage: "./database.sqlite", + // logging: console.log, + // }, development: { - dialect: "sqlite", - storage: "./database.sqlite", - logging: console.log, + dialect: "postgres", + host: "localhost", + port: 5432, + database: "neurojson_dev", + username: process.env.DB_USER_LOCAL, + password: process.env.DB_PASSWORD_LOCAL, + logging: false, }, test: { dialect: "sqlite", diff --git a/backend/migrations/20260507145253-create-search-tables.js b/backend/migrations/20260507145253-create-search-tables.js new file mode 100644 index 0000000..d2c9deb --- /dev/null +++ b/backend/migrations/20260507145253-create-search-tables.js @@ -0,0 +1,130 @@ +"use strict"; + +/** @type {import('sequelize-cli').Migration} */ +module.exports = { + async up(queryInterface, Sequelize) { + // ioviews table + await queryInterface.createTable("ioviews", { + id: { + type: Sequelize.INTEGER, + autoIncrement: true, + primaryKey: true, + allowNull: false, + }, + dbname: { + type: Sequelize.STRING(30), + allowNull: true, + }, + dsname: { + type: Sequelize.STRING(30), + allowNull: true, + }, + subj: { + type: Sequelize.STRING(12), + allowNull: true, + }, + view: { + type: Sequelize.STRING(12), + allowNull: true, + }, + json: { + type: Sequelize.JSONB, + allowNull: true, + }, + search_vector: { + type: Sequelize.DataTypes.TSVECTOR, + allowNull: true, + }, + updated_at: { + type: Sequelize.DATE, + allowNull: false, + defaultValue: Sequelize.literal("CURRENT_TIMESTAMP"), + }, + }); + + // ioviews indexes + await queryInterface.addIndex("ioviews", ["view"], { + name: "idx_ioviews_view", + }); + await queryInterface.addIndex("ioviews", ["dbname"], { + name: "idx_ioviews_dbname", + }); + await queryInterface.addIndex("ioviews", ["updated_at"], { + name: "idx_ioviews_updated_at", + }); + + // GIN indexes need raw query (not supported by addIndex) + await queryInterface.sequelize.query(` + CREATE INDEX IF NOT EXISTS idx_ioviews_search + ON ioviews USING GIN(search_vector); + CREATE INDEX IF NOT EXISTS idx_ioviews_json + ON ioviews USING GIN(json); + `); + + // iolinks table + await queryInterface.createTable("iolinks", { + id: { + type: Sequelize.INTEGER, + autoIncrement: true, + primaryKey: true, + allowNull: false, + }, + dbname: { + type: Sequelize.STRING(30), + allowNull: true, + }, + dsname: { + type: Sequelize.STRING(30), + allowNull: true, + }, + subj: { + type: Sequelize.TEXT, + allowNull: true, + }, + view: { + type: Sequelize.TEXT, + allowNull: true, + }, + json: { + type: Sequelize.JSONB, + allowNull: true, + }, + }); + + // iolinks indexes + await queryInterface.addIndex("iolinks", ["view"], { + name: "idx_iolinks_view", + }); + await queryInterface.addIndex("iolinks", ["dbname"], { + name: "idx_iolinks_dbname", + }); + await queryInterface.sequelize.query(` + CREATE INDEX IF NOT EXISTS idx_iolinks_json + ON iolinks USING GIN(json); + `); + + // sync_state table + await queryInterface.createTable("sync_state", { + dbname: { + type: Sequelize.STRING(30), + primaryKey: true, + allowNull: false, + }, + last_seq: { + type: Sequelize.TEXT, + allowNull: true, + }, + synced_at: { + type: Sequelize.DATE, + allowNull: false, + defaultValue: Sequelize.literal("CURRENT_TIMESTAMP"), + }, + }); + }, + + async down(queryInterface, Sequelize) { + await queryInterface.dropTable("ioviews"); + await queryInterface.dropTable("iolinks"); + await queryInterface.dropTable("sync_state"); + }, +}; diff --git a/backend/migrations/20260508195500-add-ioviews-unique-constraint.js b/backend/migrations/20260508195500-add-ioviews-unique-constraint.js new file mode 100644 index 0000000..757397d --- /dev/null +++ b/backend/migrations/20260508195500-add-ioviews-unique-constraint.js @@ -0,0 +1,20 @@ +"use strict"; + +/** @type {import('sequelize-cli').Migration} */ +module.exports = { + async up(queryInterface, Sequelize) { + // Required by upsertIoview's ON CONFLICT (dbname, dsname, subj, view). + await queryInterface.addConstraint("ioviews", { + fields: ["dbname", "dsname", "subj", "view"], + type: "unique", + name: "ioviews_dbname_dsname_subj_view_unique", + }); + }, + + async down(queryInterface, Sequelize) { + await queryInterface.removeConstraint( + "ioviews", + "ioviews_dbname_dsname_subj_view_unique" + ); + }, +}; diff --git a/backend/migrations/20260511145900-widen-ioviews-iolinks-text-columns.js b/backend/migrations/20260511145900-widen-ioviews-iolinks-text-columns.js new file mode 100644 index 0000000..c6cbe81 --- /dev/null +++ b/backend/migrations/20260511145900-widen-ioviews-iolinks-text-columns.js @@ -0,0 +1,60 @@ +"use strict"; + +/** @type {import('sequelize-cli').Migration} */ +module.exports = { + async up(queryInterface, Sequelize) { + // VARCHAR(n) → TEXT is a metadata-only change in Postgres (no table rewrite, + // no need to drop the unique constraint or indexes). + await queryInterface.changeColumn("ioviews", "dbname", { + type: Sequelize.TEXT, + allowNull: true, + }); + await queryInterface.changeColumn("ioviews", "dsname", { + type: Sequelize.TEXT, + allowNull: true, + }); + await queryInterface.changeColumn("ioviews", "subj", { + type: Sequelize.TEXT, + allowNull: true, + }); + await queryInterface.changeColumn("iolinks", "dbname", { + type: Sequelize.TEXT, + allowNull: true, + }); + await queryInterface.changeColumn("iolinks", "dsname", { + type: Sequelize.TEXT, + allowNull: true, + }); + await queryInterface.changeColumn("sync_state", "dbname", { + type: Sequelize.TEXT, + allowNull: false, + }); + }, + + async down(queryInterface, Sequelize) { + await queryInterface.changeColumn("ioviews", "dbname", { + type: Sequelize.STRING(30), + allowNull: true, + }); + await queryInterface.changeColumn("ioviews", "dsname", { + type: Sequelize.STRING(30), + allowNull: true, + }); + await queryInterface.changeColumn("ioviews", "subj", { + type: Sequelize.STRING(12), + allowNull: true, + }); + await queryInterface.changeColumn("iolinks", "dbname", { + type: Sequelize.STRING(30), + allowNull: true, + }); + await queryInterface.changeColumn("iolinks", "dsname", { + type: Sequelize.STRING(30), + allowNull: true, + }); + await queryInterface.changeColumn("sync_state", "dbname", { + type: Sequelize.STRING(30), + allowNull: false, + }); + }, +}; diff --git a/backend/package-lock.json b/backend/package-lock.json index 70f2f74..9bb2c58 100644 --- a/backend/package-lock.json +++ b/backend/package-lock.json @@ -15,6 +15,7 @@ "cors": "^2.8.5", "dotenv": "^17.2.3", "express": "^5.1.0", + "express-rate-limit": "^8.5.2", "jsonwebtoken": "^9.0.2", "nanoid": "^3.3.11", "nodemailer": "^7.0.11", @@ -1181,6 +1182,24 @@ "url": "https://opencollective.com/express" } }, + "node_modules/express-rate-limit": { + "version": "8.5.2", + "resolved": "https://registry.npmjs.org/express-rate-limit/-/express-rate-limit-8.5.2.tgz", + "integrity": "sha512-5Kb34ipNX694DH48vN9irak1Qx30nb0PLYHXfJgw4YEjiC3ZEmZJhwOp+VfiCYwFzvFTdB9QkArYS5kXa2cx2A==", + "license": "MIT", + "dependencies": { + "ip-address": "^10.2.0" + }, + "engines": { + "node": ">= 16" + }, + "funding": { + "url": "https://github.com/sponsors/express-rate-limit" + }, + "peerDependencies": { + "express": ">= 4.11" + } + }, "node_modules/file-uri-to-path": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/file-uri-to-path/-/file-uri-to-path-1.0.0.tgz", @@ -1742,11 +1761,10 @@ "license": "ISC" }, "node_modules/ip-address": { - "version": "10.0.1", - "resolved": "https://registry.npmjs.org/ip-address/-/ip-address-10.0.1.tgz", - "integrity": "sha512-NWv9YLW4PoW2B7xtzaS3NCot75m6nK7Icdv0o3lfMceJVRfSoQwqD4wEH5rLwoKJwUiZ/rfpiVBhnaF0FK4HoA==", + "version": "10.2.0", + "resolved": "https://registry.npmjs.org/ip-address/-/ip-address-10.2.0.tgz", + "integrity": "sha512-/+S6j4E9AHvW9SWMSEY9Xfy66O5PWvVEJ08O0y5JGyEKQpojb0K0GKpz/v5HJ/G0vi3D2sjGK78119oXZeE0qA==", "license": "MIT", - "optional": true, "engines": { "node": ">= 12" } diff --git a/backend/package.json b/backend/package.json index 1dde6de..45f386a 100644 --- a/backend/package.json +++ b/backend/package.json @@ -11,7 +11,8 @@ "migrate:undo:all": "npx sequelize-cli db:migrate:undo:all", "seed": "npx sequelize-cli db:seed:all", "seed:undo": "npx sequelize-cli db:seed:undo:all", - "db:reset": "npx sequelize-cli db:migrate:undo:all && npx sequelize-cli db:migrate" + "db:reset": "npx sequelize-cli db:migrate:undo:all && npx sequelize-cli db:migrate", + "sync": "node sync/incrementalSync.js" }, "keywords": [ "express", @@ -28,6 +29,7 @@ "cors": "^2.8.5", "dotenv": "^17.2.3", "express": "^5.1.0", + "express-rate-limit": "^8.5.2", "jsonwebtoken": "^9.0.2", "nanoid": "^3.3.11", "nodemailer": "^7.0.11", diff --git a/backend/src/controllers/couchdb.controller.js b/backend/src/controllers/couchdb.controller.js index ac75807..ab12d0f 100644 --- a/backend/src/controllers/couchdb.controller.js +++ b/backend/src/controllers/couchdb.controller.js @@ -1,8 +1,9 @@ const axios = require("axios"); -const COUCHDB_BASE_URL = - process.env.COUCHDB_BASE_URL || - "https://cors.redoc.ly/https://neurojson.io:7777"; - +const { sequelize } = require("../config/database"); +// const COUCHDB_BASE_URL = +// process.env.COUCHDB_BASE_URL || +// "https://cors.redoc.ly/https://neurojson.io:7777"; +const COUCHDB_BASE_URL = "https://neurojson.io:7777"; // get all dbs list (registry) const getDbList = async (req, res) => { try { @@ -38,72 +39,310 @@ const getDbStats = async (req, res) => { } }; -// cross-database search +// cross-database search — old version proxied to https://neurojson.org/io/search.cgi +// kept for reference; replaced by the Postgres-backed version below. +// const searchAllDatabases = async (req, res) => { +// try { +// const formData = req.body; +// const map = { +// keyword: "keyword", +// age_min: "agemin", +// age_max: "agemax", +// task_min: "taskmin", +// task_max: "taskmax", +// run_min: "runmin", +// run_max: "runmax", +// sess_min: "sessmin", +// sess_max: "sessmax", +// modality: "modality", +// run_name: "run", +// type_name: "type", +// session_name: "session", +// task_name: "task", +// limit: "limit", +// skip: "skip", +// count: "count", +// unique: "unique", +// gender: "gender", +// database: "dbname", +// dataset: "dsname", +// subject: "subname", +// }; +// +// const params = new URLSearchParams(); +// params.append("_get", "dbname, dsname, json"); +// +// Object.keys(formData).forEach((key) => { +// let val = formData[key]; +// if (val === "" || val === "any" || val === undefined || val === null) { +// return; +// } +// +// const queryKey = map[key]; +// if (!queryKey) return; +// +// if (key.startsWith("age")) { +// params.append(queryKey, String(Math.floor(val * 100)).padStart(5, "0")); +// } else if (key === "gender") { +// params.append(queryKey, val[0]); +// } else if (key === "modality") { +// params.append(queryKey, val.replace(/.*\(/, "").replace(/\).*/, "")); +// } else { +// params.append(queryKey, val.toString()); +// } +// }); +// +// const queryString = `?${params.toString()}`; +// const response = await axios.get( +// `https://cors.redoc.ly/https://neurojson.org/io/search.cgi${queryString}`, +// { +// headers: { +// Origin: "https://neurojson.io", +// "X-Requested-With": "XMLHttpRequest", +// }, +// } +// ); +// res.status(200).json(response.data); +// } catch (error) { +// console.error("Error searching all databases:", error.message); +// res.status(error.response?.status || 500).json({ +// message: "Error searching databases", +// error: error.message, +// }); +// } +// }; + +// helpers for the Postgres-backed search +function isFilter(v) { + return v !== "" && v !== "any" && v !== undefined && v !== null; +} +function pad4(n) { + return String(n).padStart(4, "0"); +} +function pad5(n) { + return String(n).padStart(5, "0"); +} + +// cross-database search — Postgres-backed (queries ioviews) const searchAllDatabases = async (req, res) => { try { - const formData = req.body; - const map = { - keyword: "keyword", - age_min: "agemin", - age_max: "agemax", - task_min: "taskmin", - task_max: "taskmax", - run_min: "runmin", - run_max: "runmax", - sess_min: "sessmin", - sess_max: "sessmax", - modality: "modality", - run_name: "run", - type_name: "type", - session_name: "session", - task_name: "task", - limit: "limit", - skip: "skip", - count: "count", - unique: "unique", - gender: "gender", - database: "dbname", - dataset: "dsname", - subject: "subname", - }; + const f = req.body || {}; + const where = []; + const repl = {}; - const params = new URLSearchParams(); - params.append("_get", "dbname, dsname, json"); + // Pick which view to search. + // Subject-level filters → subjects view; otherwise dbinfo. + const subjectFilterKeys = [ + "age_min", "age_max", "gender", + "task_min", "task_max", "task_name", + "run_min", "run_max", "run_name", + "sess_min", "sess_max", "session_name", + "type_name", "modality", "subject", + ]; + const isSubjectSearch = subjectFilterKeys.some((k) => isFilter(f[k])); + where.push(`view = :view`); + repl.view = isSubjectSearch ? "subjects" : "dbinfo"; - Object.keys(formData).forEach((key) => { - let val = formData[key]; - if (val === "" || val === "any" || val === undefined || val === null) { - return; - } + // Range filters compare against zero-padded key components. + // json->'key' = [age, gender, sess, mod, task, run, subjId] + if (isFilter(f.age_min)) { + where.push(`(json->'key'->>0) >= :ageMin`); + repl.ageMin = pad5(Math.floor(Number(f.age_min) * 100)); + } + if (isFilter(f.age_max)) { + where.push(`(json->'key'->>0) <= :ageMax`); + repl.ageMax = pad5(Math.floor(Number(f.age_max) * 100)); + } + if (isFilter(f.sess_min)) { + where.push(`(json->'key'->>2) >= :sessMin`); + repl.sessMin = pad4(f.sess_min); + } + if (isFilter(f.sess_max)) { + where.push(`(json->'key'->>2) <= :sessMax`); + repl.sessMax = pad4(f.sess_max); + } + if (isFilter(f.task_min)) { + where.push(`(json->'key'->>4) >= :taskMin`); + repl.taskMin = pad4(f.task_min); + } + if (isFilter(f.task_max)) { + where.push(`(json->'key'->>4) <= :taskMax`); + repl.taskMax = pad4(f.task_max); + } + if (isFilter(f.run_min)) { + where.push(`(json->'key'->>5) >= :runMin`); + repl.runMin = pad4(f.run_min); + } + if (isFilter(f.run_max)) { + where.push(`(json->'key'->>5) <= :runMax`); + repl.runMax = pad4(f.run_max); + } + if (isFilter(f.gender)) { + // stored as one upper-case char left-padded to 4 chars + where.push(`(json->'key'->>1) LIKE :gender`); + repl.gender = `%${String(f.gender)[0].toUpperCase()}`; + } - const queryKey = map[key]; - if (!queryKey) return; + // Name filters — jsonb ? checks if a string is an element of the array. + if (isFilter(f.task_name)) { + where.push(`json->'value'->'tasks' ? :taskName`); + repl.taskName = String(f.task_name); + } + if (isFilter(f.run_name)) { + where.push(`json->'value'->'runs' ? :runName`); + repl.runName = String(f.run_name); + } + if (isFilter(f.session_name)) { + where.push(`json->'value'->'sessions' ? :sessName`); + repl.sessName = String(f.session_name); + } + if (isFilter(f.type_name)) { + where.push(`json->'value'->'types' ? :typeName`); + repl.typeName = String(f.type_name); + } + if (isFilter(f.modality)) { + // form sometimes wraps as "fNIRS (nirs)" — pull text inside parens + const mod = String(f.modality).replace(/.*\(/, "").replace(/\).*/, ""); + where.push(`json->'value'->'modalities' ? :modality`); + repl.modality = mod; + } - if (key.startsWith("age")) { - params.append(queryKey, String(Math.floor(val * 100)).padStart(5, "0")); - } else if (key === "gender") { - params.append(queryKey, val[0]); - } else if (key === "modality") { - params.append(queryKey, val.replace(/.*\(/, "").replace(/\).*/, "")); + // Dataset-level modality filter (multi-select + AND/OR). + // Queries json->'modality' on dbinfo rows, not subjects rows. + if (Array.isArray(f.modalities) && f.modalities.length > 0) { + const op = f.modality_mode === "and" ? " AND " : " OR "; + const parts = f.modalities.map((m, i) => { + repl[`dmod${i}`] = String(m); + return isSubjectSearch + ? `dsi.json->'modality' ? :dmod${i}` + : `json->'modality' ? :dmod${i}`; + }); + const condition = `(${parts.join(op)})`; + if (isSubjectSearch) { + where.push(`EXISTS ( + SELECT 1 FROM ioviews dsi + WHERE dsi.dbname = ioviews.dbname + AND dsi.dsname = ioviews.dsname + AND dsi.view = 'dbinfo' + AND ${condition} + )`); } else { - params.append(queryKey, val.toString()); + where.push(condition); } + } + + // db / ds / subj filters + if (isFilter(f.database)) { + where.push(`dbname = :dbname`); + repl.dbname = String(f.database); + } + if (isFilter(f.dataset)) { + where.push(`dsname = :dsname`); + repl.dsname = String(f.dataset); + } + if (isFilter(f.subject)) { + where.push(`subj = :subj`); + repl.subj = String(f.subject); + } + + // Keyword search — match anywhere relevant. + // plainto_tsquery treats input as plain words AND'd together; ignores + // operator chars like "-" and "OR" so dataset names with hyphens + // (e.g. "ABIDE - CMU_a") don't get parsed as NOT clauses. + // ILIKE on dbname/dsname adds substring matching so "fnirs" finds + // "bfnirs", "openfnirs", and any dataset id containing it. + // ILIKE on json->>'name' covers the human-readable name from + // dataset_description.json (e.g. "ABIDE - CMU_a"), which is where the + // user-visible dataset titles live — dsname column often stores just + // an opaque id like "CMU_a" without the prefix. + // ILIKE pattern normalizes whitespace/hyphens to % wildcards so + // "ABIDE - CMU_a" matches stored names regardless of separator style. + // The whole group is parenthesised so it ANDs cleanly with other filters. + if (isFilter(f.keyword)) { + where.push(`( + search_vector @@ plainto_tsquery('english', :keyword) + OR dbname ILIKE :keywordLike + OR dsname ILIKE :keywordLike + OR (json->>'name') ILIKE :keywordLike + )`); + repl.keyword = String(f.keyword); + repl.keywordLike = `%${String(f.keyword).replace(/[\s-]+/g, "%")}%`; + } + + // File-type filter — array of extensions like [".jdb", ".snirf"]. + // Dataset-level: include rows whose (dbname, dsname) has at least one + // iolinks file with a matching view (extension). Per-subject filtering + // isn't possible here because iolinks.subj stores file size, not subj id. + // Use IN (:array) — Sequelize replacements expand arrays as 'a','b','c', + // which fits IN(...) but NOT ANY(...). + if (Array.isArray(f.file_type) && f.file_type.length > 0) { + where.push(`EXISTS ( + SELECT 1 FROM iolinks l + WHERE l.dbname = ioviews.dbname + AND l.dsname = ioviews.dsname + AND l.view IN (:fileTypes) + )`); + repl.fileTypes = f.file_type.map((t) => String(t)); + } + + const limit = Math.min(parseInt(f.limit) || 100, 1000); + const offset = parseInt(f.skip) || 0; + repl.limit = limit; + repl.offset = offset; + + // When file_type filter is active, also return a sample of the actual + // matching iolinks rows (filename, url, path, suffix) per dataset, plus + // a total count. Frontend shows up to 10 as clickable filenames and a + // "Download manifest" button for the full list via a separate endpoint. + const matchingFilesActive = + Array.isArray(f.file_type) && f.file_type.length > 0; + const matchingFilesColumn = matchingFilesActive + ? `, + COALESCE(( + SELECT jsonb_agg(t.json) + FROM ( + SELECT l.json + FROM iolinks l + WHERE l.dbname = ioviews.dbname + AND l.dsname = ioviews.dsname + AND l.view IN (:fileTypes) + ORDER BY l.id + LIMIT 10 + ) t + ), '[]'::jsonb)::text AS matching_files, + (SELECT COUNT(*) FROM iolinks l + WHERE l.dbname = ioviews.dbname + AND l.dsname = ioviews.dsname + AND l.view IN (:fileTypes))::int AS matching_files_total` + : ""; + + // dbinfo was stored flat ({name, subj, ...}); subjects was stored wrapped + // ({key, value}). Frontend expects parsed.value.subj for datasets, so we + // wrap dbinfo on the way out. + const sql = ` + SELECT + dbname, + dsname, + subj, + CASE + WHEN view = 'dbinfo' THEN jsonb_build_object('value', json)::text + ELSE json::text + END AS json${matchingFilesColumn} + FROM ioviews + WHERE ${where.join(" AND ")} + ORDER BY dbname, dsname, subj + LIMIT :limit OFFSET :offset + `; + + const rows = await sequelize.query(sql, { + replacements: repl, + type: sequelize.QueryTypes.SELECT, }); - const queryString = `?${params.toString()}`; - const response = await axios.get( - `https://cors.redoc.ly/https://neurojson.org/io/search.cgi${queryString}`, - { - headers: { - Origin: "https://neurojson.io", - "X-Requested-With": "XMLHttpRequest", - }, - } - ); - res.status(200).json(response.data); + res.status(200).json(rows); } catch (error) { console.error("Error searching all databases:", error.message); - res.status(error.response?.status || 500).json({ + res.status(500).json({ message: "Error searching databases", error: error.message, }); @@ -237,6 +476,146 @@ const getDatasetMeta = async (req, res) => { // } +// Downloadable list of every matching iolinks URL for a dataset. +// Three formats via ?format=: +// - txt (default) → plain URL list (use with `wget -i`) +// - sh → bash script with curl commands (Mac/Linux) +// - bat → Windows batch script with curl commands +// All three avoid server-side zipping — the user's machine pulls files +// directly from neurojson.org/io, so this Express server stays light. +const getDatasetFilesManifest = async (req, res) => { + try { + const { dbName, dsName } = req.params; + const rawExt = req.query.ext; + const format = String(req.query.format || "txt").toLowerCase(); + const exts = Array.isArray(rawExt) + ? rawExt + : typeof rawExt === "string" && rawExt.length > 0 + ? rawExt.split(",") + : []; + + if (exts.length === 0) { + res.status(400).send("ext query parameter required (e.g. ?ext=.jdb)"); + return; + } + + const rows = await sequelize.query( + `SELECT json->'value'->>'url' AS url, + json->'value'->>'file' AS file + FROM iolinks + WHERE dbname = :dbname + AND dsname = :dsname + AND view IN (:exts) + ORDER BY id`, + { + replacements: { dbname: dbName, dsname: dsName, exts }, + type: sequelize.QueryTypes.SELECT, + } + ); + + const resolveUrl = (url) => { + if (!url || url.startsWith("http")) return url; + return `https://neurojson.org/io/stat.cgi?action=get&db=${dbName}&doc=${dsName}&${url}`; + }; + const files = rows.filter((r) => r.url).map((r) => ({ ...r, url: resolveUrl(r.url) })); + const urls = files.map((r) => r.url); + const baseName = `${dbName}_${dsName}_${exts.join("_")}`; + const extLabel = exts.join(", "); + + // Strip any path separators or quote chars from the parsed filename + // before using it in shell commands — file names come from iolinks + // and are usually content hashes, but defensive belt-and-suspenders. + const safeName = (s) => + (s || "").replace(/["\\\/\r\n]/g, "").trim(); + + let body; + let contentType; + let filename; + + if (format === "sh") { + // Bash script — curl is preinstalled on macOS and most Linux distros. + // -L follows redirects, -C - resumes interrupted downloads, -o saves + // with our parsed filename (the URL is a CGI query — using -O would + // save files as literal `stat.cgi?...`). + body = + `#!/bin/bash\n` + + `# Downloads ${extLabel} files from ${dbName}/${dsName}\n` + + `# Usage: bash ${baseName}_download.sh\n` + + `set -e\n` + + `mkdir -p "neurojson_downloads"\n` + + `cd "neurojson_downloads" || exit 1\n` + + files + .map((r) => { + const fn = safeName(r.file); + return fn + ? `curl -L -C - -o "${fn}" "${r.url}"` + : `curl -L -C - -O "${r.url}"`; + }) + .join("\n") + + `\necho "Done. Files saved to $(pwd)"\n`; + contentType = "application/x-sh; charset=utf-8"; + filename = `${baseName}_download.sh`; + } else if (format === "bat") { + // Windows batch — curl ships with Windows 10+. Uses CRLF line endings + // for proper rendering in CMD. /d on cd handles cross-drive paths. + body = + `@echo off\r\n` + + `REM Downloads ${extLabel} files from ${dbName}/${dsName}\r\n` + + `REM Usage: double-click or run ${baseName}_download.bat\r\n` + + `if not exist "neurojson_downloads" mkdir "neurojson_downloads"\r\n` + + `cd /d "neurojson_downloads"\r\n` + + files + .map((r) => { + const fn = safeName(r.file); + return fn + ? `curl -L -C - -o "${fn}" "${r.url}"` + : `curl -L -C - -O "${r.url}"`; + }) + .join("\r\n") + + `\r\necho Done. Files saved to %cd%\r\n` + + `pause\r\n`; + contentType = "text/plain; charset=utf-8"; + filename = `${baseName}_download.bat`; + } else { + // Default: plain URL list, one per line (advanced users with wget). + body = urls.join("\n") + "\n"; + contentType = "text/plain; charset=utf-8"; + filename = `${baseName}_manifest.txt`; + } + + res.setHeader("Content-Type", contentType); + res.setHeader( + "Content-Disposition", + `attachment; filename="${filename}"` + ); + res.send(body); + } catch (error) { + console.error("Error generating manifest:", error.message); + res.status(500).send(`Error generating manifest: ${error.message}`); + } +}; + +// distinct file extensions present in iolinks across all synced DBs. +// Drives the multi-select "File types" filter on the search page. +const getFileTypes = async (req, res) => { + try { + const rows = await sequelize.query( + `SELECT DISTINCT view AS type + FROM iolinks + WHERE view IS NOT NULL AND view <> '' + ORDER BY view`, + { type: sequelize.QueryTypes.SELECT } + ); + res.status(200).json(rows.map((r) => r.type)); + } catch (error) { + console.error("Error fetching file types:", error.message); + res.status(500).json({ + message: "Error fetching file types", + error: error.message, + }); + } +}; + module.exports = { getDbList, getDbStats, @@ -245,4 +624,6 @@ module.exports = { searchAllDatabases, getDatasetDetail, getDatasetMeta, + getFileTypes, + getDatasetFilesManifest, }; diff --git a/backend/src/controllers/ollama.controller.js b/backend/src/controllers/ollama.controller.js index 84bb3bb..fe0c4e8 100644 --- a/backend/src/controllers/ollama.controller.js +++ b/backend/src/controllers/ollama.controller.js @@ -1,12 +1,13 @@ const OLLAMA_BASE_URL = "http://jin.neu.edu:11434"; +const OLLAMA_MODEL = "qwen3.6:27b"; const proxyChat = async (req, res) => { - console.log("🟣 [Ollama] proxyChat hit — model:", req.body.model); + console.log("🟣 [Ollama] proxyChat hit — model:", OLLAMA_MODEL); try { const response = await fetch(`${OLLAMA_BASE_URL}/v1/chat/completions`, { method: "POST", headers: { "Content-Type": "application/json" }, - body: JSON.stringify(req.body), + body: JSON.stringify({ ...req.body, model: OLLAMA_MODEL }), }); const data = await response.json(); diff --git a/backend/src/routes/dbs.routes.js b/backend/src/routes/dbs.routes.js index d11f970..45979ac 100644 --- a/backend/src/routes/dbs.routes.js +++ b/backend/src/routes/dbs.routes.js @@ -6,6 +6,8 @@ const { getDbInfo, getDbDatasets, searchAllDatabases, + getFileTypes, + getDatasetFilesManifest, // searchDatabase, } = require("../controllers/couchdb.controller"); @@ -15,8 +17,17 @@ const router = express.Router(); router.get("/", getDbList); router.get("/stats", getDbStats); +// distinct file extensions across all iolinks rows (drives the file-type +// filter on the search page). Must come BEFORE the /:dbName route, otherwise +// Express treats "file-types" as a dbName. +router.get("/file-types", getFileTypes); + // cross-database search -router.get("/search", searchAllDatabases); +router.post("/search", searchAllDatabases); + +// downloadable manifest (plain text) of all iolinks URLs for a dataset +// filtered by extension(s). e.g. /dbs/bfnirs/Motion-Yucel2014-I/files/manifest?ext=.jdb +router.get("/:dbName/:dsName/files/manifest", getDatasetFilesManifest); // Specific database routes router.get("/:dbName", getDbInfo); diff --git a/backend/src/routes/ollama.public.routes.js b/backend/src/routes/ollama.public.routes.js new file mode 100644 index 0000000..c395e51 --- /dev/null +++ b/backend/src/routes/ollama.public.routes.js @@ -0,0 +1,17 @@ +const express = require("express"); +const router = express.Router(); +const rateLimit = require("express-rate-limit"); +const { proxyChat, getTags } = require("../controllers/ollama.controller"); + +const dailyLimit = rateLimit({ + windowMs: 24 * 60 * 60 * 1000, // 24 hours + max: 20, + standardHeaders: true, + legacyHeaders: false, + message: { error: "Daily request limit reached. You can send up to 20 requests per day from this IP." }, +}); + +router.post("/chat", dailyLimit, proxyChat); +// router.get("/tags", dailyLimit, getTags); + +module.exports = router; diff --git a/backend/src/routes/ollama.routes.js b/backend/src/routes/ollama.routes.js index ff1fd94..be86d75 100644 --- a/backend/src/routes/ollama.routes.js +++ b/backend/src/routes/ollama.routes.js @@ -1,8 +1,9 @@ const express = require("express"); const router = express.Router(); const { proxyChat, getTags } = require("../controllers/ollama.controller"); +const { requireAuth } = require("../middleware/auth.middleware"); -router.post("/chat", proxyChat); -router.get("/tags", getTags); +router.post("/chat", requireAuth, proxyChat); +// router.get("/tags", requireAuth, getTags); module.exports = router; diff --git a/backend/src/server.js b/backend/src/server.js index 5e6cb6c..d0d043d 100644 --- a/backend/src/server.js +++ b/backend/src/server.js @@ -14,6 +14,7 @@ const datasetsRoutes = require("./routes/datasets.routes"); const collectionRoutes = require("./routes/collection.route"); const projectRoutes = require("./routes/projects.routes"); const ollamaRoutes = require("./routes/ollama.routes"); +const ollamaPublicRoutes = require("./routes/ollama.public.routes"); const app = express(); const PORT = process.env.PORT || 5000; @@ -51,6 +52,7 @@ app.use("/api/v1/datasets", datasetsRoutes); app.use("/api/v1/collections", collectionRoutes); app.use("/api/v1/projects", projectRoutes); app.use("/api/v1/ollama", ollamaRoutes); +app.use("/api/v1/ollama-public", ollamaPublicRoutes); // health check endpoint app.get("/api/health", async (req, res) => { diff --git a/backend/sync/incrementalSync.js b/backend/sync/incrementalSync.js new file mode 100644 index 0000000..766f1a5 --- /dev/null +++ b/backend/sync/incrementalSync.js @@ -0,0 +1,605 @@ +"use strict"; + +require("dotenv").config(); +const axios = require("axios"); +const { sequelize } = require("../src/config/database"); + +const COUCHDB_URL = process.env.COUCHDB_URL || "https://neurojson.io:7777"; +const CONCURRENCY = 5; + +// fetch database list dynamically from registry +// registry doc shape: { database: [{ id, name, ... }, ...] } +async function getDatabases() { + const response = await axios.get(`${COUCHDB_URL}/sys/registry`); + const entries = response.data?.database || []; + const databases = entries.map((db) => db.id).filter(Boolean); + console.log(`Found ${databases.length} databases in registry`); + return databases; +} + +// === Local ports of CouchDB _design/qq map functions === +// 1:1 ports of dbinfo / subjects / links views. If upstream views change, +// these drift silently. + +function transformDbinfo(doc) { + const txt = doc["README"] || doc["README.md"] || doc["README.rst"] || ""; + const rawtext = JSON.stringify(doc); + const datainfo = doc["dataset_description.json"] || { Name: doc._id }; + const subjlist = []; + const modalitylist = []; + + for (const item of Object.keys(doc)) { + if (item.indexOf("ub-") !== -1) { + subjlist.push(item); + for (const modal of Object.keys(doc[item] || {})) { + if (modal.indexOf("ses") === 0) { + for (const m of Object.keys(doc[item][modal] || {})) { + if (m.indexOf(".") === -1 && modalitylist.indexOf(m) === -1) { + modalitylist.push(m); + } + } + } else if ( + modal.indexOf(".") === -1 && + modalitylist.indexOf(modal) === -1 + ) { + modalitylist.push(modal); + } + } + } + } + + if (subjlist.length === 0) subjlist.push("nonbids"); + + if (modalitylist.length === 0) { + if (rawtext.indexOf('"MeshNode"') !== -1) modalitylist.push("JMesh"); + if (rawtext.indexOf('"NIFTIData"') !== -1) modalitylist.push("JNIFTI"); + if (rawtext.indexOf('"SNIRFData"') !== -1) modalitylist.push("JSNIRF"); + if (rawtext.indexOf('"_ArrayType_"') !== -1) modalitylist.push("JData"); + } + + return { + name: datainfo.Name, + length: rawtext.length, + readme: String(txt).substr(0, 256), + info: datainfo, + subj: subjlist, + modality: modalitylist, + }; +} + +function transformSubjects(doc) { + const results = []; + const skipIds = ["sidecards", "derivatives", "sourcedata", "code"]; + if (skipIds.indexOf(doc._id) !== -1) return results; + + for (const subj of Object.keys(doc)) { + if (!/^[sS]ub-/.test(subj)) continue; + + const sessionlist = []; + const modalitylist = []; + const tasklist = []; + const runlist = []; + const filetype = []; + let age = -0.01; + let gender = "N"; + + const p = doc["participants.tsv"]; + if (p && Array.isArray(p.participant_id)) { + let idx = -1; + for (let i = 0; i < p.participant_id.length; i++) { + if (subj.indexOf(String(p.participant_id[i])) > -1) { + idx = i; + break; + } + } + + if (idx >= 0) { + for (const agekey of ["age", "age_scan", "age_at_scan"]) { + if (age >= 0) break; + if (p[agekey]) { + age = p[agekey][idx]; + break; + } else if (p[agekey.toUpperCase()]) { + age = p[agekey.toUpperCase()][idx]; + break; + } else { + const cap = agekey.charAt(0).toUpperCase() + agekey.slice(1); + if (p[cap]) { + age = p[cap]; // matches upstream view (drops [idx] here) + break; + } + } + } + if (age < 0) { + for (const pfield of Object.keys(p)) { + if (pfield.toLowerCase().indexOf("age") >= 0) { + age = p[pfield][idx]; + } + } + } + for (const sexkey of ["sex", "gender"]) { + if (gender !== "N") break; + if (p[sexkey]) { + gender = p[sexkey][idx]; + break; + } else if (p[sexkey.toUpperCase()]) { + gender = p[sexkey.toUpperCase()][idx]; + break; + } else { + const cap = sexkey.charAt(0).toUpperCase() + sexkey.slice(1); + if (p[cap]) { + gender = p[cap]; // matches upstream view (drops [idx] here) + break; + } + } + } + if (gender === "N") { + for (const pfield of Object.keys(p)) { + if (pfield.toLowerCase().indexOf("sex") >= 0) { + gender = p[pfield][idx]; + } + } + } + if (gender === "N") { + for (const pfield of Object.keys(p)) { + if (pfield.toLowerCase().indexOf("gender") >= 0) { + gender = p[pfield][idx]; + } + } + } + } + } + + const subjDoc = doc[subj] || {}; + const parseFiles = (container) => { + for (const filename of Object.keys(container || {})) { + for (const task of filename.split("_")) { + if (task.indexOf("run-") === 0) { + if (runlist.indexOf(task.substring(4)) === -1) { + runlist.push(task.substring(4)); + } + } else if (task.indexOf("task-") === 0) { + if (tasklist.indexOf(task.substring(5)) === -1) { + tasklist.push(task.substring(5)); + } + } else if (task.indexOf(".") > 0) { + const tmp = task.substring(0, task.indexOf(".")); + if (filetype.indexOf(tmp) === -1) filetype.push(tmp); + } + } + } + }; + + for (const modal of Object.keys(subjDoc)) { + if (modal.indexOf("ses-") === 0) { + if (sessionlist.indexOf(modal.substring(4)) === -1) { + sessionlist.push(modal.substring(4)); + } + for (const modname of Object.keys(subjDoc[modal] || {})) { + if ( + modname.indexOf(".") === -1 && + modalitylist.indexOf(modname) === -1 + ) { + modalitylist.push(modname); + } + parseFiles(subjDoc[modal][modname]); + } + } else if ( + modal.indexOf(".") === -1 && + modalitylist.indexOf(modal) === -1 + ) { + modalitylist.push(modal); + parseFiles(subjDoc[modal]); + } + } + + if (typeof gender === "string") { + gender = gender.substring(0, 1).toUpperCase(); + } else { + gender = gender + ""; + } + if (typeof age === "string" && isNaN(+age)) age = -0.001; + if (typeof age === "string") age = +age; + if (age < 0) age = -0.01; + age = Math.floor(age * 100); + + results.push({ + id: doc._id, + key: [ + ("0000" + age).slice(-5), + ("000" + gender).slice(-4), + ("000" + sessionlist.length).slice(-4), + ("000" + modalitylist.length).slice(-4), + ("000" + tasklist.length).slice(-4), + ("000" + runlist.length).slice(-4), + subj.substring(4), + ], + value: { + sessions: sessionlist, + modalities: modalitylist, + tasks: tasklist, + runs: runlist, + types: filetype, + }, + }); + } + + return results; +} + +function transformLinks(doc) { + const results = []; + const filenameRe = /file=([^\/]*\/)*([^&\/\.]+)(\.[^.&%:]+(\.gz)*)([&:].*)*$/; + const filesizeRe = /size=(\d+)/; + const jsonpathRe = /:(\$[^&]+)/; + const urlhash = {}; + + function traverse(obj, level, rootpath) { + if (level > 10) return; + if (obj === null || typeof obj !== "object") return; + + for (const subkey of Object.keys(obj)) { + const v = obj[subkey]; + if ( + subkey === "_DataLink_" && + typeof v === "string" && + v.indexOf("http") !== -1 + ) { + const url = v; + const uniqurl = url.split(":$")[0]; + if (!Object.prototype.hasOwnProperty.call(urlhash, uniqurl)) { + const fname = url.match(filenameRe); + const fsize = url.match(filesizeRe); + let jpath = url.match(jsonpathRe); + if (jpath !== null && jpath.length) jpath = jpath[1]; + urlhash[uniqurl] = 1; + if (fname && fsize) { + results.push({ + id: doc._id, + key: [fname[3], parseInt(fsize[1], 10)], + value: { + path: rootpath, + url: uniqurl, + file: fname[2] + fname[3], + suffix: fname[3], + ref: jpath, + }, + }); + } + } + } + if (typeof v === "object" && v !== null) { + traverse(v, level + 1, rootpath + "." + subkey); + } + } + } + + traverse(doc, 1, "$"); + return results; +} + +// === DB helpers (each accepts an optional transaction) === + +async function getLastSeq(dbname) { + try { + const result = await sequelize.query( + "SELECT last_seq FROM sync_state WHERE dbname = :dbname", + { replacements: { dbname }, type: sequelize.QueryTypes.SELECT } + ); + return result[0]?.last_seq || "0"; + } catch (err) { + console.error(`Error getting last_seq for ${dbname}:`, err.message); + return "0"; + } +} + +async function saveLastSeq(dbname, seq) { + await sequelize.query( + `INSERT INTO sync_state (dbname, last_seq, synced_at) + VALUES (:dbname, :seq, NOW()) + ON CONFLICT (dbname) DO UPDATE + SET last_seq = :seq, synced_at = NOW()`, + { replacements: { dbname, seq: String(seq) } } + ); +} + +// Postgres jsonb rejects the null-byte escape with "unsupported Unicode +// escape sequence", so strip it from the serialized JSON before insert. +// Seen in openneuro README/TSV fields containing stray null bytes. +function safeStringify(obj) { + return JSON.stringify(obj).replace(/\\u0000/g, ""); +} + +// A valid file type is a dot-prefixed extension with no slashes and +// a reasonable length. Some CouchDB links view rows (e.g. openneuro) +// emit paths like ".0/libraries/FID-A/..." where the version number +// gets parsed as a fake extension — reject those. +function isValidFileType(ext) { + return ( + typeof ext === "string" && + ext.startsWith(".") && + !ext.includes("/") && + ext.length <= 20 + ); +} + +async function upsertIoview(dbname, dsname, subj, view, json, transaction) { + const payload = safeStringify(json); + await sequelize.query( + `INSERT INTO ioviews (dbname, dsname, subj, view, json, search_vector, updated_at) + VALUES (:dbname, :dsname, :subj, :view, :json, to_tsvector('english', :text), NOW()) + ON CONFLICT (dbname, dsname, subj, view) DO UPDATE + SET json = :json, + search_vector = to_tsvector('english', :text), + updated_at = NOW()`, + { + replacements: { + dbname, + dsname, + subj: String(subj), + view, + json: payload, + text: payload, + }, + transaction, + } + ); +} + +async function insertIolink(dbname, dsname, subj, view, json, transaction) { + await sequelize.query( + `INSERT INTO iolinks (dbname, dsname, subj, view, json) + VALUES (:dbname, :dsname, :subj, :view, :json)`, + { + replacements: { + dbname, + dsname, + subj: String(subj), + view, + json: safeStringify(json), + }, + transaction, + } + ); +} + +async function deleteDataset(dbname, dsname, transaction) { + await sequelize.query( + "DELETE FROM ioviews WHERE dbname = :dbname AND dsname = :dsname", + { replacements: { dbname, dsname }, transaction } + ); + await sequelize.query( + "DELETE FROM iolinks WHERE dbname = :dbname AND dsname = :dsname", + { replacements: { dbname, dsname }, transaction } + ); +} + +// === First-time sync (fetch all three views once) === + +// Fetch a view, treating 404 as "view doesn't exist on this DB" (returns []). +// Non-BIDS DBs (e.g. brainmeshlibrary) only have the dbinfo view. +async function fetchView(dbname, viewName) { + try { + const res = await axios.get( + `${COUCHDB_URL}/${dbname}/_design/qq/_view/${viewName}` + ); + return res.data.rows || []; + } catch (err) { + if (err.response?.status === 404) { + console.log(` ${dbname}: view '${viewName}' not present, skipping`); + return []; + } + throw err; + } +} + +async function firstSync(dbname) { + console.log(` ${dbname}: first sync, fetching all views...`); + + const dbinfoRows = await fetchView(dbname, "dbinfo"); + for (const row of dbinfoRows) { + const subj = String(row.value?.subj?.length || 0); + await upsertIoview(dbname, row.id, subj, "dbinfo", row.value); + } + console.log(` ${dbname}: dbinfo synced (${dbinfoRows.length} rows)`); + + const subjectRows = await fetchView(dbname, "subjects"); + for (const row of subjectRows) { + const subj = String(row.key?.[6] || ""); + await upsertIoview(dbname, row.id, subj, "subjects", { + key: row.key, + value: row.value, + }); + } + console.log(` ${dbname}: subjects synced (${subjectRows.length} rows)`); + + const linkRows = await fetchView(dbname, "links"); + let linkCount = 0; + for (const row of linkRows) { + const fileType = row.key?.[0]; + if (!isValidFileType(fileType)) continue; + const subjId = String(row.key?.[1] || ""); + await insertIolink(dbname, row.id, subjId, fileType, { + key: row.key, + value: row.value, + }); + linkCount++; + } + console.log(` ${dbname}: links synced (${linkCount}/${linkRows.length} rows)`); +} + +// === Process one changed dataset (Option A: 2 HTTP requests + local transforms) === + +async function processDatasetUpdate(dbname, dsname) { + // dbinfo view supports key filtering; raw doc carries everything else. + const keyParam = encodeURIComponent(JSON.stringify(dsname)); + const [dbinfoRes, rawDocRes] = await Promise.all([ + axios.get( + `${COUCHDB_URL}/${dbname}/_design/qq/_view/dbinfo?key=${keyParam}` + ), + axios.get(`${COUCHDB_URL}/${dbname}/${encodeURIComponent(dsname)}`), + ]); + + const dbinfoRow = (dbinfoRes.data.rows || [])[0]; + if (!dbinfoRow) { + console.warn(` ${dbname}/${dsname}: no dbinfo row, skipping`); + return; + } + const dbinfoValue = dbinfoRow.value; + const doc = rawDocRes.data; + + const subjectRows = transformSubjects(doc); + const linkRows = transformLinks(doc); + + // Rule 1: wrap all writes for this dataset in one transaction. + await sequelize.transaction(async (t) => { + const subjCount = String(dbinfoValue?.subj?.length || 0); + await upsertIoview(dbname, dsname, subjCount, "dbinfo", dbinfoValue, t); + + // Rule 2: empty-subjs guard. NOT IN (NULL) silently matches nothing. + const currentSubjs = Array.isArray(dbinfoValue?.subj) + ? dbinfoValue.subj + : []; + if (currentSubjs.length > 0) { + // subjects view stores subj without "sub-"/"Sub-" prefix + // (key[6] = subj.substring(4) in upstream map). + const currentSubjIds = currentSubjs.map((s) => s.substring(4)); + await sequelize.query( + `DELETE FROM ioviews + WHERE dbname = :dbname AND dsname = :dsname AND view = 'subjects' + AND subj NOT IN (:subjs)`, + { + replacements: { dbname, dsname, subjs: currentSubjIds }, + transaction: t, + } + ); + } + + for (const row of subjectRows) { + const subj = String(row.key?.[6] || ""); + await upsertIoview( + dbname, + dsname, + subj, + "subjects", + { key: row.key, value: row.value }, + t + ); + } + + // iolinks: no usable upsert key, so delete + reinsert per dataset. + await sequelize.query( + "DELETE FROM iolinks WHERE dbname = :dbname AND dsname = :dsname", + { replacements: { dbname, dsname }, transaction: t } + ); + for (const row of linkRows) { + const fileType = row.key?.[0]; + const subjId = String(row.key?.[1] || ""); + await insertIolink( + dbname, + dsname, + subjId, + fileType, + { key: row.key, value: row.value }, + t + ); + } + }); +} + +// === Incremental sync === + +async function incrementalSync(dbname, lastSeq) { + // No include_docs=true: we fetch the raw doc per dataset so the _changes + // payload stays small and per-dataset work runs in parallel. + const { data } = await axios.get( + `${COUCHDB_URL}/${dbname}/_changes?since=${encodeURIComponent(lastSeq)}` + ); + + if (!data.results || data.results.length === 0) { + console.log(` ${dbname}: no changes since last sync`); + return data.last_seq; + } + + const changes = data.results.filter( + (c) => c.id && !c.id.startsWith("_design/") + ); + console.log( + ` ${dbname}: ${changes.length} dataset changes (raw=${data.results.length})` + ); + + // Rule 3: bounded concurrency + per-dataset try/catch. + for (let i = 0; i < changes.length; i += CONCURRENCY) { + const chunk = changes.slice(i, i + CONCURRENCY); + await Promise.all( + chunk.map(async (change) => { + try { + if (change.deleted) { + await sequelize.transaction((t) => + deleteDataset(dbname, change.id, t) + ); + console.log(` ${dbname}/${change.id}: deleted`); + } else { + await processDatasetUpdate(dbname, change.id); + } + } catch (err) { + console.error(` ${dbname}/${change.id}: failed - ${err.message}`); + } + }) + ); + } + + // Rule 4: return last_seq from THIS response. Never re-fetch update_seq + // afterward (writes during sync would be silently skipped). + return data.last_seq; +} + +// === Sync a single database === + +async function syncDatabase(dbname) { + console.log(`\nSyncing ${dbname}...`); + const lastSeq = await getLastSeq(dbname); + + try { + let nextSeq; + if (lastSeq === "0") { + // Rule 5: capture update_seq BEFORE firstSync. Writes during firstSync + // get picked up by the next incremental run. + const { data: info } = await axios.get(`${COUCHDB_URL}/${dbname}`); + const seqAtStart = String(info.update_seq); + await firstSync(dbname); + nextSeq = seqAtStart; + } else { + nextSeq = await incrementalSync(dbname, lastSeq); + } + + await saveLastSeq(dbname, String(nextSeq)); + console.log(` ${dbname}: sync complete ✓`); + } catch (err) { + console.error(` ${dbname}: sync failed - ${err.message}`); + } +} + +// === Main === + +async function runSync() { + console.log("=== Starting NeuroJSON sync ==="); + console.log(new Date().toISOString()); + console.log(`CouchDB: ${COUCHDB_URL}`); + + const databases = await getDatabases(); + console.log(`Databases: ${databases.length}`); + + for (const db of databases) { + await syncDatabase(db); + } + + await sequelize.close(); + console.log("\n=== Sync complete ==="); + console.log(new Date().toISOString()); +} + +runSync().catch((err) => { + console.error("Sync failed:", err); + process.exit(1); +}); diff --git a/package.json b/package.json index 1f8144e..aac37d2 100644 --- a/package.json +++ b/package.json @@ -50,19 +50,20 @@ "react-redux": "^8.1.2", "react-router-dom": "^6.15.0", "react-scripts": "^5.0.1", - "react-syntax-highlighter": "^15.6.1", + "react-syntax-highlighter": "16.1.1", "sharp": "^0.33.5", "stats-js": "^1.0.1", "stats.js": "0.17.0", "three": "0.145.0", "typescript": "^5.1.6", - "uplot": "1.6.17", + "uplot": "1.6.32", "web-vitals": "^2.1.0", "xlsx": "^0.18.5" }, "devDependencies": { "@babel/plugin-proposal-private-property-in-object": "^7.21.11", "@trivago/prettier-plugin-sort-imports": "^4.2.0", + "@types/js-yaml": "^4.0.9", "@types/node": "^20.5.7", "@types/pako": "^2.0.3", "@typescript-eslint/eslint-plugin": "^5.31.0", diff --git a/src/components/NavBar/NavItems.tsx b/src/components/NavBar/NavItems.tsx index d2882e6..bfbb9b3 100644 --- a/src/components/NavBar/NavItems.tsx +++ b/src/components/NavBar/NavItems.tsx @@ -35,11 +35,13 @@ const NavItems: React.FC = () => { const [signupOpen, setSignupOpen] = useState(false); // Resources dropdown state - const [resourcesAnchor, setResourcesAnchor] = useState( - null - ); + const [resourcesAnchor, setResourcesAnchor] = useState(null); const resourcesOpen = Boolean(resourcesAnchor); + // AutoBIDSify dropdown state + const [autobidsifyAnchor, setAutobidsifyAnchor] = useState(null); + const autobidsifyOpen = Boolean(autobidsifyAnchor); + const handleLogout = () => { dispatch(logoutUser()); navigate("/"); @@ -53,6 +55,14 @@ const NavItems: React.FC = () => { setResourcesAnchor(null); }; + const handleAutobidsifyClick = (event: React.MouseEvent) => { + setAutobidsifyAnchor(event.currentTarget); + }; + + const handleAutobidsifyClose = () => { + setAutobidsifyAnchor(null); + }; + const resourcesMenu = [ { category: "Converter", @@ -378,6 +388,38 @@ const NavItems: React.FC = () => { ))} + {/* AutoBIDSify Dropdown */} + + + + AutoBIDSify + + + + + {/* Resources Dropdown */} { ))} + + {/* AutoBIDSify Dropdown Menu */} + + { + window.open("https://github.com/COTILab/autobidsify", "_blank"); + handleAutobidsifyClose(); + }} + sx={{ + fontSize: "0.9rem", + color: Colors.white, + "&:hover": { bgcolor: Colors.purpleGrey, color: Colors.darkPurple }, + }} + > + AutoBIDSify (GitHub) + + { + navigate(RoutesEnum.BIDS_CONVERTER); + handleAutobidsifyClose(); + }} + sx={{ + fontSize: "0.9rem", + color: Colors.white, + "&:hover": { bgcolor: Colors.purpleGrey, color: Colors.darkPurple }, + }} + > + AutoBIDSify Web + + + setLoginOpen(false)} diff --git a/src/components/PreviewModal.tsx b/src/components/PreviewModal.tsx index 4a3ce82..bf81318 100644 --- a/src/components/PreviewModal.tsx +++ b/src/components/PreviewModal.tsx @@ -28,30 +28,14 @@ const PreviewModal: React.FC<{ // fix end--------------------- useEffect(() => { - if (!isOpen) return; - //add spinner - // if (!isOpen || isLoading) return; - - // fix start-----------: Get the container element from the ref. - // const container = canvasContainerRef.current; - // if (!container) { - // // This can happen briefly on the first render, so we just wait for the next render. - // return; - // } - // // 3. Check for the required legacy functions on the window object. - // if ( - // typeof window.previewdata !== "function" || - // typeof window.initcanvas_with_container !== "function" - // ) { - // console.error( - // "❌ Legacy preview script functions are not available on the window object." - // ); - // return; - // } - - // window.previewdata(dataKey, previewIndex, isInternal, false); - // fix end--------------------------------- - // clear old canvas + // if (!isOpen) return; + if (!isOpen) { + // Modal just closed — clean up Three.js immediately + if (typeof window.destroyPreview === "function") { + window.destroyPreview(); + } + return; + } const canvasDiv = document.getElementById("canvas"); if (canvasDiv) while (canvasDiv.firstChild) canvasDiv.removeChild(canvasDiv.firstChild); @@ -69,6 +53,10 @@ const PreviewModal: React.FC<{ return () => { clearInterval(interval); + // Component unmounting — clean up Three.js + if (typeof window.destroyPreview === "function") { + window.destroyPreview(); + } }; }, [isOpen, dataKey, previewIndex, isInternal]); diff --git a/src/components/Routes.tsx b/src/components/Routes.tsx index 5983701..10b15e9 100644 --- a/src/components/Routes.tsx +++ b/src/components/Routes.tsx @@ -1,3 +1,4 @@ +import BidsConverterPage from "pages/BidsConverterPage"; import ScrollToTop from "./ScrollToTop"; import CompleteProfile from "./User/CompleteProfile"; import CollectionDetailPage from "./User/Dashboard/CollectionDetailPage"; @@ -68,6 +69,7 @@ const Routes = () => ( element={} /> } /> + } /> diff --git a/src/components/SearchPage/DatasetCard.tsx b/src/components/SearchPage/DatasetCard.tsx index 42f6646..7ecefae 100644 --- a/src/components/SearchPage/DatasetCard.tsx +++ b/src/components/SearchPage/DatasetCard.tsx @@ -1,10 +1,37 @@ -import { Typography, Card, CardContent, Stack, Chip } from "@mui/material"; +import DownloadIcon from "@mui/icons-material/Download"; +import KeyboardArrowDownIcon from "@mui/icons-material/KeyboardArrowDown"; +import { + Typography, + Card, + CardContent, + Stack, + Chip, + Button, + Link as MuiLink, + Menu, + MenuItem, + Box, + Snackbar, + Alert, +} from "@mui/material"; +import { baseURL } from "services/instance"; import { Colors } from "design/theme"; import React from "react"; -import { useMemo } from "react"; +import { useMemo, useState } from "react"; import { Link } from "react-router-dom"; import RoutesEnum from "types/routes.enum"; +interface MatchingFile { + key?: any; + value?: { + file?: string; + url?: string; + path?: string; + suffix?: string; + ref?: string; + }; +} + interface DatasetCardProps { dbname: string; dsname: string; @@ -26,20 +53,45 @@ interface DatasetCardProps { index: number; onChipClick: (key: string, value: string) => void; keyword?: string; // for keyword highlight + matchingFiles?: MatchingFile[]; // sample of iolinks rows matching file_type + matchingFilesTotal?: number; // total count across all matches + fileTypes?: string[]; // the active file_type filter, used to build manifest URL } /** ---------- utility helpers ---------- **/ +// Some iolinks records (older openneuro links view) store a relative path like +// "file=sub-01/anat/sub-01_T1w.nii&size=1" instead of a full stat.cgi URL. +// Reconstruct the full URL so the browser can follow it. +const resolveFileUrl = ( + dbname: string, + dsname: string, + url?: string +): string => { + if (!url) return ""; + if (url.startsWith("http")) return url; + return `https://neurojson.org/io/stat.cgi?action=get&db=${dbname}&doc=${dsname}&${url}`; +}; const normalize = (s: string) => s ?.replace(/[\u2018\u2019\u2032]/g, "'") // curly → straight ?.replace(/[\u201C\u201D\u2033]/g, '"') ?? // curly → straight ""; +// Multi-word keyword support: backend tsquery treats "head brain" as AND of +// independent tokens. Highlighting should match the same logic — split on +// whitespace and treat each word independently. +const splitKeyword = (kw?: string): string[] => { + if (!kw) return []; + return normalize(kw).trim().split(/\s+/).filter(Boolean); +}; + +const escapeRegex = (s: string) => s.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); + const containsKeyword = (text?: string, kw?: string) => { if (!text || !kw) return false; const t = normalize(text).toLowerCase(); - const k = normalize(kw).toLowerCase(); - return t.includes(k); + const words = splitKeyword(kw.toLowerCase()); + return words.some((w) => t.includes(w)); }; /** Find a short snippet in secondary fields if not already visible */ @@ -62,24 +114,41 @@ function findMatchSnippet( ["ReferencesAndLinks", (v) => v?.info?.ReferencesAndLinks], ]; - const k = normalize(kw).toLowerCase(); + const words = splitKeyword(kw.toLowerCase()); + if (words.length === 0) return null; for (const [label, getter] of CANDIDATE_FIELDS) { const raw = getter(v); // v = parsedJson.value if (!raw) continue; const text = normalize(String(raw)); - const i = text.toLowerCase().indexOf(k); // k is the lowercase version of keyword - if (i >= 0) { - const start = Math.max(0, i - 40); - const end = Math.min(text.length, i + k.length + 40); - const before = text.slice(start, i); - const hit = text.slice(i, i + k.length); - const after = text.slice(i + k.length, end); - const html = `${ - start > 0 ? "…" : "" - }${before}${hit}${after}${end < text.length ? "…" : ""}`; - return { label, html }; + const lower = text.toLowerCase(); + + // Find the earliest occurrence of ANY matching word — that's the snippet anchor. + let anchor = -1; + let anchorLen = 0; + for (const w of words) { + const i = lower.indexOf(w); + if (i >= 0 && (anchor < 0 || i < anchor)) { + anchor = i; + anchorLen = w.length; + } } + if (anchor < 0) continue; + + const start = Math.max(0, anchor - 40); + const end = Math.min(text.length, anchor + anchorLen + 40); + const slice = text.slice(start, end); + + // Highlight every matching word inside the snippet, not just the first. + const regex = new RegExp( + `(${words.map(escapeRegex).join("|")})`, + "gi" + ); + const highlighted = slice.replace(regex, "$1"); + const html = `${start > 0 ? "…" : ""}${highlighted}${ + end < text.length ? "…" : "" + }`; + return { label, html }; } return null; } @@ -92,10 +161,69 @@ const DatasetCard: React.FC = ({ index, onChipClick, keyword, + matchingFiles, + matchingFilesTotal, + fileTypes, }) => { const { name, readme, modality, subj, info } = parsedJson.value; const datasetLink = `${RoutesEnum.DATABASES}/${dbname}/${dsname}`; + // Build manifest URL for any of the three formats. Backend serves + // text/plain for .txt, application/x-sh for .sh, text/plain for .bat — + // each with a Content-Disposition header so the browser saves them. + const buildManifestUrl = (format: "txt" | "sh" | "bat") => { + if (!fileTypes || fileTypes.length === 0) return null; + const ext = fileTypes.map((e) => encodeURIComponent(e)).join(","); + return `${baseURL}/dbs/${encodeURIComponent( + dbname + )}/${encodeURIComponent( + dsname + )}/files/manifest?ext=${ext}&format=${format}`; + }; + + const hasManifest = Array.isArray(fileTypes) && fileTypes.length > 0; + + // Dropdown state for the download format menu. + const [downloadMenuEl, setDownloadMenuEl] = useState( + null + ); + // Post-download instruction snackbar. Stays open until user dismisses it + // (no autoHideDuration) so researchers have time to read multi-step + // instructions. + const [downloadHint, setDownloadHint] = useState< + "sh" | "bat" | "txt" | null + >(null); + const handleDownload = (format: "txt" | "sh" | "bat") => { + const url = buildManifestUrl(format); + setDownloadMenuEl(null); + if (!url) return; + // Programmatic anchor click triggers the browser's normal download flow + // without leaving the current page (window.location would navigate away). + const a = document.createElement("a"); + a.href = url; + document.body.appendChild(a); + a.click(); + document.body.removeChild(a); + setDownloadHint(format); + }; + + // Extract a short "sub-XXX" tag from a BIDS path like + // "$.sub-019.ses-1.nirs.sub-019_ses-1_task-MA_run-01_nirs.snirf.SNIRFData..." + const subjectFromPath = (p?: string): string => { + if (!p) return ""; + const m = p.match(/sub-[^.]+/); + return m ? m[0] : ""; + }; + + // File size stored in key[1] of each iolinks row (bytes). Format for humans. + const formatBytes = (n?: number): string => { + if (typeof n !== "number" || !Number.isFinite(n) || n < 0) return ""; + if (n < 1024) return `${n} B`; + if (n < 1024 * 1024) return `${(n / 1024).toFixed(1)} KB`; + if (n < 1024 * 1024 * 1024) return `${(n / 1024 / 1024).toFixed(1)} MB`; + return `${(n / 1024 / 1024 / 1024).toFixed(2)} GB`; + }; + // prepare DOI URL const rawDOI = info?.DatasetDOI?.replace(/^doi:/, ""); const doiLink = rawDOI ? `https://doi.org/${rawDOI}` : null; @@ -122,19 +250,25 @@ const DatasetCard: React.FC = ({ [parsedJson.value, keyword, visibleHasKeyword] ); - // keyword highlight functional component (only for visible fields) + // keyword highlight functional component (only for visible fields). + // Splits the keyword on whitespace and highlights each word independently + // so "head brain" highlights both words wherever they appear. const highlightKeyword = (text: string, keyword?: string) => { - if (!keyword || !text?.toLowerCase().includes(keyword.toLowerCase())) { - return text; - } - - const regex = new RegExp(`(${keyword})`, "gi"); // for case-insensitive and global + const words = splitKeyword(keyword); + if (words.length === 0 || !text) return text; + const lowerWordSet = new Set(words.map((w) => w.toLowerCase())); + const regex = new RegExp( + `(${words.map(escapeRegex).join("|")})`, + "gi" + ); + if (!regex.test(text)) return text; + // Reset lastIndex because test() advances on /g regexes; safer to use split. const parts = text.split(regex); return ( <> {parts.map((part, i) => - part.toLowerCase() === keyword.toLowerCase() ? ( + lowerWordSet.has(part.toLowerCase()) ? ( = ({ )} + + {/* Matching files section — only shown when file_type filter is active */} + {Array.isArray(matchingFiles) && matchingFiles.length > 0 && ( + + + + Matching files + {typeof matchingFilesTotal === "number" && + ` (${ + matchingFiles.length < matchingFilesTotal + ? `${matchingFiles.length} of ${matchingFilesTotal}` + : matchingFilesTotal + })`} + + {hasManifest && ( + <> + + setDownloadMenuEl(null)} + > + handleDownload("sh")}> + For Mac / Linux (.sh) + + handleDownload("bat")}> + For Windows (.bat) + + handleDownload("txt")}> + URL list (.txt, advanced) + + + + )} + + + {matchingFiles.slice(0, 10).map((f, i) => { + const v = f.value || {}; + const subjTag = subjectFromPath(v.path); + const sizeBytes = + Array.isArray(f.key) && typeof f.key[1] === "number" + ? f.key[1] + : undefined; + const sizeTag = formatBytes(sizeBytes); + const meta = [subjTag, sizeTag].filter(Boolean).join(" · "); + return ( +
  • + + {v.file || v.url} + + {meta && ( + + ({meta}) + + )} +
    • + ); + })} +     +     + )} + + {/* Post-download instructions. No auto-hide so users can read at their + * own pace; dismiss with the ✕ when finished. */} + setDownloadHint(null)} + anchorOrigin={{ vertical: "bottom", horizontal: "center" }} + > + setDownloadHint(null)} + sx={{ maxWidth: 520 }} + > + {downloadHint === "sh" && ( + + + Downloaded the Mac / Linux script + + + To fetch your data files: + + +
  • Open Terminal
    • +
  • Go to the folder where the script was saved
    • +
  • + Run:{" "} + + bash <script-name>.sh + +
    • +     +     + )} + {downloadHint === "bat" && ( + + + Downloaded the Windows script + + + Open the folder where the script was saved and{" "} + double-click the .bat file. A command window + opens and the files download next to it. + + + )} + {downloadHint === "txt" && ( + + + Downloaded the URL list + + + In Terminal (Mac/Linux) or PowerShell (Windows), run:{" "} + + wget -i <file-name>.txt + + + + )} +     +     ); }; diff --git a/src/components/SearchPage/SubjectCard.tsx b/src/components/SearchPage/SubjectCard.tsx index c66ebce..f805b48 100644 --- a/src/components/SearchPage/SubjectCard.tsx +++ b/src/components/SearchPage/SubjectCard.tsx @@ -16,6 +16,7 @@ interface SubjectCardProps { modalities?: string[]; tasks?: string[]; sessions?: string[]; + runs?: string[]; types?: string[]; }; }; @@ -32,7 +33,8 @@ const SubjectCard: React.FC = ({ index, onChipClick, }) => { - const { modalities, tasks, sessions, types } = parsedJson.value; + const { modalities, tasks, sessions, runs, types } = parsedJson.value; + const runCount = Array.isArray(runs) ? runs.length : 0; const subjectLink = `${RoutesEnum.DATABASES}/${dbname}/${dsname}`; const formattedSubj = /^sub-/i.test(subj) ? subj : `sub-${String(subj)}`; @@ -229,6 +231,11 @@ const SubjectCard: React.FC = ({ {sessions?.length === 0 ? 1 : sessions?.length} + + + Runs: {runCount} + + diff --git a/src/components/SearchPage/widgets/FileTypeAutocompleteWidget.tsx b/src/components/SearchPage/widgets/FileTypeAutocompleteWidget.tsx new file mode 100644 index 0000000..eb941ad --- /dev/null +++ b/src/components/SearchPage/widgets/FileTypeAutocompleteWidget.tsx @@ -0,0 +1,40 @@ +import { Autocomplete, Chip, TextField } from "@mui/material"; +import { WidgetProps } from "@rjsf/utils"; + +// Multi-select combobox for file extensions (e.g. ".jdb", ".snirf"). +// Options come from uiSchema's ui:options.fileTypes, fetched once by the +// parent SearchPage from /api/v1/dbs/file-types. +export const FileTypeAutocompleteWidget = (props: WidgetProps) => { + const { value, onChange, options, label } = props; + const fileTypes = (options.fileTypes as string[]) || []; + const current: string[] = Array.isArray(value) ? value : []; + + return ( + onChange(v as string[])} + renderTags={(items, getTagProps) => + items.map((item, index) => ( + + )) + } + renderInput={(params) => ( + + )} + /> + ); +}; diff --git a/src/components/SearchPage/widgets/TypeAutocompleteWidget.tsx b/src/components/SearchPage/widgets/TypeAutocompleteWidget.tsx new file mode 100644 index 0000000..b0573fc --- /dev/null +++ b/src/components/SearchPage/widgets/TypeAutocompleteWidget.tsx @@ -0,0 +1,27 @@ +import { Autocomplete, TextField } from "@mui/material"; +import { WidgetProps } from "@rjsf/utils"; + +// Combobox: type freely OR pick from a modality-specific suggestion list. +export const TypeAutocompleteWidget = (props: WidgetProps) => { + const { value, onChange, options, label, placeholder } = props; + const suggestions = (options.suggestions as string[]) || []; + + return ( + onChange(typeof v === "string" ? v : "")} + onInputChange={(_, v) => onChange(v || "")} + renderInput={(params) => ( + + )} + /> + ); +}; diff --git a/src/components/User/Dashboard/DatasetOrganizer/FileTree.tsx b/src/components/User/Dashboard/DatasetOrganizer/FileTree.tsx index 29c0bd5..0dbcf90 100644 --- a/src/components/User/Dashboard/DatasetOrganizer/FileTree.tsx +++ b/src/components/User/Dashboard/DatasetOrganizer/FileTree.tsx @@ -522,7 +522,7 @@ const FileTree: React.FC = ({ }} > - + Virtual File System @@ -631,12 +631,11 @@ const FileTree: React.FC = ({ borderColor: "divider", }} > - - Saved Outputs + BIDS Conversion Package Preview {outputFiles diff --git a/src/components/User/Dashboard/DatasetOrganizer/LLMPanel.tsx b/src/components/User/Dashboard/DatasetOrganizer/LLMPanel.tsx index d9d6366..c574bf5 100644 --- a/src/components/User/Dashboard/DatasetOrganizer/LLMPanel.tsx +++ b/src/components/User/Dashboard/DatasetOrganizer/LLMPanel.tsx @@ -1,29 +1,19 @@ import { generateId } from "./utils/fileProcessors"; -import { extractSubjectAnalysis } from "./utils/filenameTokenizer"; -//add +import { LLMConfig } from "./utils/llm"; import { - buildFileSummary, - analyzeFilePatterns, - getUserContext, - getFileAnnotations, - downloadJSON, buildEvidenceBundle, - extractSubjectsFromFiles, buildIngestInfo, + downloadJSON, } from "./utils/llmHelpers"; -import { - getDatasetDescriptionPrompt, - getReadmePrompt, - getParticipantsPrompt, - getConversionScriptPrompt, - getBIDSPlanPrompt, -} from "./utils/llmPrompts"; +import { buildBidsPlan } from "./utils/plannerHelpers"; +import { generateTrioFiles } from "./utils/trioHelpers"; import { Close, ContentCopy, Download, AutoAwesome, DriveFileMove, + InfoOutlined, } from "@mui/icons-material"; import { Box, @@ -38,12 +28,15 @@ import { CircularProgress, IconButton, Alert, + Tooltip, } from "@mui/material"; import { Colors } from "design/theme"; +import { dump as yamlDump } from "js-yaml"; import JSZip from "jszip"; import React, { useState, useEffect } from "react"; import { FileItem } from "redux/projects/types/projects.interface"; -import { OllamaService } from "services/ollama.service"; + +// import { OllamaService } from "services/ollama.service"; interface LLMPanelProps { files: FileItem[]; @@ -55,6 +48,7 @@ interface LLMPanelProps { setTrioGenerated: (value: boolean) => void; // ✅ Add updateFiles: (updater: React.SetStateAction) => void; // ✅ Add onClose: () => void; + isPrivateMode?: boolean; } interface LLMProvider { @@ -68,46 +62,61 @@ interface LLMProvider { const llmProviders: Record = { ollama: { - name: "Ollama (Local Server)", - // baseUrl: "http://localhost:11434/v1/chat/completions", + name: "Ollama (NeuroJSON Server)", baseUrl: "", models: [ - { id: "qwen3-coder-next:latest", name: "Qwen 3 Coder Next" }, - { id: "qwen3-coder-careful:latest", name: "Qwen 3 Coder Careful" }, - { id: "qwen3.5:9b", name: "Qwen 3.5 9B" }, - { id: "qwen2.5-coder:latest", name: "Qwen 2.5 Coder (7.6B)" }, - { id: "qwen2.5-coder:7b", name: "Qwen 2.5 Coder 7B" }, + { id: "qwen3.6:27b", name: "Qwen 3.6 27B" }, + // { id: "qwen3-coder-next:latest", name: "Qwen 3 Coder Next" }, + // { id: "qwen3-coder-careful:latest", name: "Qwen 3 Coder Careful" }, + // { id: "qwen3.5:9b", name: "Qwen 3.5 9B" }, + // { id: "qwen2.5-coder:latest", name: "Qwen 2.5 Coder (7.6B)" }, + // { id: "qwen2.5-coder:7b", name: "Qwen 2.5 Coder 7B" }, + ], + noApiKey: true, + }, + "local-ollama": { + name: "Local AI (Ollama / LM Studio / Jan)", + baseUrl: "http://localhost:11434/v1/chat/completions", + models: [ + { id: "llama3.2:latest", name: "Llama 3.2 (Ollama)" }, + { id: "llama3.1:latest", name: "Llama 3.1 (Ollama)" }, + { id: "qwen2.5-coder:latest", name: "Qwen 2.5 Coder (Ollama)" }, + { id: "mistral:latest", name: "Mistral (Ollama)" }, + { id: "gemma3:latest", name: "Gemma 3 (Ollama)" }, + { id: "llama-3.2-3b-instruct", name: "Llama 3.2 3B (LM Studio)" }, + { id: "llama-3.1-8b-instruct", name: "Llama 3.1 8B (LM Studio)" }, + { id: "mistral-7b-instruct-v0.3", name: "Mistral 7B (LM Studio)" }, + { id: "llama3.2:3b", name: "Llama 3.2 3B (Jan)" }, + { id: "mistral:7b", name: "Mistral 7B (Jan)" }, ], noApiKey: true, - // customUrl: true, }, groq: { name: "Groq (Free API Key - 14,400 req/day)", baseUrl: "https://api.groq.com/openai/v1/chat/completions", models: [ { id: "llama-3.3-70b-versatile", name: "Llama 3.3 70B" }, + { id: "llama-3.1-70b-versatile", name: "Llama 3.1 70B" }, { id: "llama-3.1-8b-instant", name: "Llama 3.1 8B (Fast)" }, - { id: "mixtral-8x7b-32768", name: "Mixtral 8x7B" }, + { id: "gemma2-9b-it", name: "Gemma 2 9B" }, ], }, openrouter: { name: "OpenRouter (Free models available)", baseUrl: "https://openrouter.ai/api/v1/chat/completions", models: [ - { - id: "meta-llama/llama-3.1-8b-instruct:free", - name: "Llama 3.1 8B (Free)", - }, - { id: "google/gemma-2-9b-it:free", name: "Gemma 2 9B (Free)" }, - { id: "mistralai/mistral-7b-instruct:free", name: "Mistral 7B (Free)" }, + { id: "meta-llama/llama-3.3-70b-instruct:free", name: "Llama 3.3 70B (Free)" }, + { id: "google/gemma-3n-e4b-it:free", name: "Gemma 3n 4B (Free)" }, + { id: "mistralai/mistral-small-3.2-24b-instruct:free", name: "Mistral Small 3.2 24B (Free)" }, ], }, anthropic: { name: "Anthropic Claude (Paid)", baseUrl: "https://api.anthropic.com/v1/messages", models: [ - { id: "claude-sonnet-4-20250514", name: "Claude Sonnet 4" }, - { id: "claude-3-5-haiku-20241022", name: "Claude 3.5 Haiku" }, + { id: "claude-opus-4-7", name: "Claude Opus 4.7" }, + { id: "claude-sonnet-4-6", name: "Claude Sonnet 4.6" }, + { id: "claude-haiku-4-5-20251001", name: "Claude Haiku 4.5" }, ], isAnthropic: true, }, @@ -115,8 +124,11 @@ const llmProviders: Record = { name: "OpenAI (Paid)", baseUrl: "https://api.openai.com/v1/chat/completions", models: [ - { id: "gpt-4o-mini", name: "GPT-4o Mini" }, + { id: "gpt-5.5", name: "GPT-5.5" }, + { id: "gpt-5.4", name: "GPT-5.4" }, + { id: "gpt-5.4-mini", name: "GPT-5.4 Mini" }, { id: "gpt-4o", name: "GPT-4o" }, + { id: "gpt-4o-mini", name: "GPT-4o Mini" }, ], }, }; @@ -131,9 +143,11 @@ const LLMPanel: React.FC = ({ setTrioGenerated, // ✅ Add updateFiles, // ✅ Add onClose, + isPrivateMode = false, }) => { - const [provider, setProvider] = useState("ollama"); - const [model, setModel] = useState("qwen3-coder-next:latest"); + const [provider, setProvider] = useState(isPrivateMode ? "local-ollama" : "ollama"); + const [model, setModel] = useState(isPrivateMode ? "llama3.2:latest" : "qwen3-coder-next:latest"); + const [localOllamaUrl, setLocalOllamaUrl] = useState("http://localhost:11434"); // const [ollamaUrl, setOllamaUrl] = useState( // "http://jin.neu.edu:11434" // ); @@ -158,6 +172,18 @@ const LLMPanel: React.FC = ({ const [panelHeight, setPanelHeight] = useState(450); const [isResizing, setIsResizing] = useState(false); + // Build LLMConfig for all helper calls — mirrors autobidsify CLI arg assembly + const buildLLMConfig = (): LLMConfig => ({ + provider, + model, + apiKey, + baseUrl: provider === "local-ollama" + ? `${localOllamaUrl}/v1/chat/completions` + : currentProvider.baseUrl, + isAnthropic: currentProvider.isAnthropic, + noApiKey: currentProvider.noApiKey, + }); + // ======================================================================== // BUTTON 1: GENERATE EVIDENCE BUNDLE // ======================================================================== @@ -184,6 +210,7 @@ const LLMPanel: React.FC = ({ }); setEvidenceBundle(bundle); + setSubjectAnalysis(null); // ← add this line downloadJSON(bundle, "evidence_bundle.json"); setStatus("✓ Evidence bundle generated and downloaded!"); } catch (err: any) { @@ -201,291 +228,28 @@ const LLMPanel: React.FC = ({ setError("Please generate evidence bundle first"); return; } - if (!currentProvider.noApiKey && !apiKey.trim()) { setError("Please enter an API key"); return; } - // Create abort controller const controller = new AbortController(); setAbortController(controller); - setGeneratingTrio(true); setError(null); + setBidsPlan(""); setStatus("Generating BIDS trio files..."); try { - const userText = evidenceBundle.user_hints.user_text || ""; - - // ========================================== - // Call 1: Generate dataset_description.json - // ========================================== - let datasetDesc: any; - if (evidenceBundle.trio_found?.["dataset_description.json"]) { - setStatus("1/3 dataset_description.json already exists, skipping..."); - const existing = files.find( - (f) => f.source === "user" && f.name === "dataset_description.json" - ); - datasetDesc = existing?.content ? JSON.parse(existing.content) : {}; - } else { - setStatus("1/3 Generating dataset_description.json..."); - const ddPrompt = getDatasetDescriptionPrompt(userText, evidenceBundle); - - let ddResponse; - if (currentProvider.isAnthropic) { - ddResponse = await fetch(currentProvider.baseUrl, { - method: "POST", - signal: controller.signal, - headers: { - "Content-Type": "application/json", - "x-api-key": apiKey, - "anthropic-version": "2023-06-01", - }, - body: JSON.stringify({ - model, - max_tokens: 2048, - messages: [{ role: "user", content: ddPrompt }], - }), - }); - } else if (provider === "ollama") { - // const ollamaBaseUrl = ollamaUrl || "http://localhost:11434"; - // ddResponse = await fetch(`${ollamaBaseUrl}/v1/chat/completions`, { - // method: "POST", - // signal: controller.signal, - // headers: { "Content-Type": "application/json" }, - // body: JSON.stringify({ - // model, - // messages: [{ role: "user", content: ddPrompt }], - // stream: false, - // }), - // }); - ddResponse = await OllamaService.chat(model, [ - { role: "user", content: ddPrompt }, - ]); - } else { - ddResponse = await fetch(currentProvider.baseUrl, { - method: "POST", - signal: controller.signal, - headers: { - "Content-Type": "application/json", - Authorization: `Bearer ${apiKey}`, - }, - body: JSON.stringify({ - model, - messages: [{ role: "user", content: ddPrompt }], - max_tokens: 2048, - }), - }); - } - - // const ddData = await ddResponse.json(); - const ddData = - provider === "ollama" ? ddResponse : await ddResponse.json(); - let ddText = currentProvider.isAnthropic - ? ddData.content[0].text - : ddData.choices[0].message.content; - - // Clean up markdown fences - ddText = ddText - .replace(/^```json\n?/g, "") - .replace(/\n?```$/g, "") - .trim(); - datasetDesc = JSON.parse(ddText); - } + const { datasetDesc, readmeContent, participantsTsv, skipped } = + await generateTrioFiles({ + evidenceBundle, + files, + llmConfig: buildLLMConfig(), + signal: controller.signal, + onStatus: setStatus, + }); - // ========================================== - // Call 2: Generate README.md - // ========================================== - let readmeContent: string; - if (evidenceBundle.trio_found?.["README.md"]) { - setStatus("2/3 README.md already exists, skipping..."); - const existing = files.find( - (f) => - f.source === "user" && - ["README.md", "README.txt", "README.rst", "readme.md"].includes( - f.name - ) - ); - readmeContent = existing?.content || ""; - } else { - setStatus("2/3 Generating README.md..."); - const readmePrompt = getReadmePrompt(userText); - - let readmeResponse; - if (currentProvider.isAnthropic) { - readmeResponse = await fetch(currentProvider.baseUrl, { - method: "POST", - signal: controller.signal, - headers: { - "Content-Type": "application/json", - "x-api-key": apiKey, - "anthropic-version": "2023-06-01", - }, - body: JSON.stringify({ - model, - max_tokens: 2048, - messages: [{ role: "user", content: readmePrompt }], - }), - }); - } else if (provider === "ollama") { - // const ollamaBaseUrl = ollamaUrl || "http://localhost:11434"; - // readmeResponse = await fetch(`${ollamaBaseUrl}/v1/chat/completions`, { - // method: "POST", - // signal: controller.signal, - // headers: { "Content-Type": "application/json" }, - // body: JSON.stringify({ - // model, - // messages: [{ role: "user", content: readmePrompt }], - // stream: false, - // }), - // }); - readmeResponse = await OllamaService.chat(model, [ - { role: "user", content: readmePrompt }, - ]); - } else { - readmeResponse = await fetch(currentProvider.baseUrl, { - method: "POST", - signal: controller.signal, - headers: { - "Content-Type": "application/json", - Authorization: `Bearer ${apiKey}`, - }, - body: JSON.stringify({ - model, - messages: [{ role: "user", content: readmePrompt }], - max_tokens: 2048, - }), - }); - } - - // const readmeData = await readmeResponse.json(); - const readmeData = - provider === "ollama" ? readmeResponse : await readmeResponse.json(); - readmeContent = currentProvider.isAnthropic - ? readmeData.content[0].text - : readmeData.choices[0].message.content; - } - // ========================================== - // Call 3: Generate participants.tsv - // ========================================== - let participantsContent: string; - if (evidenceBundle.trio_found?.["participants.tsv"]) { - setStatus("3/3 participants.tsv already exists, skipping..."); - const existing = files.find( - (f) => f.source === "user" && f.name === "participants.tsv" - ); - participantsContent = existing?.content || ""; - } else { - setStatus("3/3 Generating participants.tsv..."); - const partsPrompt = getParticipantsPrompt(userText); - - let partsResponse; - if (currentProvider.isAnthropic) { - partsResponse = await fetch(currentProvider.baseUrl, { - method: "POST", - signal: controller.signal, - headers: { - "Content-Type": "application/json", - "x-api-key": apiKey, - "anthropic-version": "2023-06-01", - }, - body: JSON.stringify({ - model, - max_tokens: 1024, - messages: [{ role: "user", content: partsPrompt }], - }), - }); - } else if (provider === "ollama") { - // const ollamaBaseUrl = ollamaUrl || "http://localhost:11434"; - // partsResponse = await fetch(`${ollamaBaseUrl}/v1/chat/completions`, { - // method: "POST", - // signal: controller.signal, - // headers: { "Content-Type": "application/json" }, - // body: JSON.stringify({ - // model, - // messages: [{ role: "user", content: partsPrompt }], - // stream: false, - // }), - // }); - partsResponse = await OllamaService.chat(model, [ - { role: "user", content: partsPrompt }, - ]); - } else { - partsResponse = await fetch(currentProvider.baseUrl, { - method: "POST", - signal: controller.signal, - headers: { - "Content-Type": "application/json", - Authorization: `Bearer ${apiKey}`, - }, - body: JSON.stringify({ - model, - messages: [{ role: "user", content: partsPrompt }], - max_tokens: 1024, - }), - }); - } - - // const partsData = await partsResponse.json(); - const partsData = - provider === "ollama" ? partsResponse : await partsResponse.json(); - const participantsRaw = currentProvider.isAnthropic - ? partsData.content[0].text - : partsData.choices[0].message.content; - - // Build TSV from schema - try { - const schemaText = participantsRaw - .replace(/^```json\n?/g, "") - .replace(/\n?```$/g, "") - .trim(); - const schema = JSON.parse(schemaText); - const columns: string[] = schema.columns.map((c: any) => c.name); - - // Get subject IDs from evidence bundle (extracted by Python-style analysis) - // const idMapping = - // evidenceBundle?.subject_analysis?.id_mapping?.id_mapping; - // const subjectLabels: string[] = idMapping - // ? Object.values(idMapping).map((id) => `sub-${id}`) - // : ["sub-01"]; // fallback if no subject analysis - // Get subject IDs from subjectAnalysis state (computed at plan stage) - // Fall back to computing fresh if plan hasn't been run yet - const currentSubjectAnalysis = - subjectAnalysis || - extractSubjectAnalysis( - evidenceBundle?.all_files || [], - evidenceBundle?.user_hints?.n_subjects, - evidenceBundle?.filename_analysis?.python_statistics - ?.dominant_prefixes - ); - const idMap = currentSubjectAnalysis?.id_mapping?.id_mapping; - const subjectLabels: string[] = - idMap && Object.keys(idMap).length > 0 - ? Object.values(idMap).map((id) => `sub-${id}`) - : Array.from( - { length: evidenceBundle?.user_hints?.n_subjects || 1 }, - (_, i) => `sub-${String(i + 1).padStart(2, "0")}` - ); - - const header = columns.join("\t"); - const rows = subjectLabels.map((subId) => - columns - .map((col: string) => (col === "participant_id" ? subId : "n/a")) - .join("\t") - ); - participantsContent = [header, ...rows].join("\n"); - } catch (e) { - // Fallback: LLM didn't return valid JSON schema, use raw content - participantsContent = participantsRaw - .replace(/^```\n?/g, "") - .replace(/\n?```$/g, "") - .trim(); - } - } - // ========================================== - // Add trio files to Virtual File System - // ========================================== const timestamp = new Date().toLocaleString(); const trioFiles: FileItem[] = [ { @@ -505,10 +269,7 @@ const LLMPanel: React.FC = ({ name: "README.md", type: "file", fileType: "meta", - content: readmeContent - .replace(/^```markdown\n?/g, "") - .replace(/\n?```$/g, "") - .trim(), + content: readmeContent, contentType: "text", isUserMeta: true, parentId: null, @@ -520,10 +281,7 @@ const LLMPanel: React.FC = ({ name: "participants.tsv", type: "file", fileType: "meta", - content: participantsContent - .replace(/^```\n?/g, "") - .replace(/\n?```$/g, "") - .trim(), + content: participantsTsv, contentType: "text", isUserMeta: true, parentId: null, @@ -531,35 +289,30 @@ const LLMPanel: React.FC = ({ generatedAt: timestamp, }, ]; - // replace existing trio files, add if not exist + updateFiles((prev) => { const trioNames = [ "dataset_description.json", "README.md", "participants.tsv", ]; - - // Remove old AI generated trio files const withoutOldTrio = prev.filter( (f) => !(f.source === "ai" && trioNames.includes(f.name)) ); - - // Add new trio files - // return [...withoutOldTrio, ...trioFiles]; - - // Only add AI-generated files for ones that weren't user-uploaded - const newTrioFiles = trioFiles.filter( - (tf) => - !evidenceBundle.trio_found?.[ - tf.name as keyof typeof evidenceBundle.trio_found - ] - ); - + // Only add AI files for ones that weren't user-uploaded (skipped=true means user-uploaded) + const newTrioFiles = trioFiles.filter((tf) => { + if (tf.name === "dataset_description.json") + return !skipped.datasetDesc; + if (tf.name === "README.md") return !skipped.readme; + if (tf.name === "participants.tsv") return !skipped.participants; + return true; + }); return [...withoutOldTrio, ...newTrioFiles]; }); + setTrioGenerated(true); setStatus( - "✓ BIDS trio files generated and added to Virtual File System!" + "✓ BIDS metadata files generated and added to file tree!" ); } catch (err: any) { if (err.name === "AbortError") { @@ -570,9 +323,469 @@ const LLMPanel: React.FC = ({ } } finally { setGeneratingTrio(false); - setAbortController(null); // Clear controller + setAbortController(null); } }; + // const handleGenerateTrio = async () => { + // if (!evidenceBundle) { + // setError("Please generate evidence bundle first"); + // return; + // } + + // if (!currentProvider.noApiKey && !apiKey.trim()) { + // setError("Please enter an API key"); + // return; + // } + + // // Create abort controller + // const controller = new AbortController(); + // setAbortController(controller); + + // setGeneratingTrio(true); + // setError(null); + // setStatus("Generating BIDS trio files..."); + + // try { + // const userText = evidenceBundle.user_hints.user_text || ""; + + // // ========================================== + // // Call 1: Generate dataset_description.json + // // ========================================== + // let datasetDesc: any; + // if (evidenceBundle.trio_found?.["dataset_description.json"]) { + // setStatus("1/3 dataset_description.json already exists, skipping..."); + // const existing = files.find( + // (f) => f.source === "user" && f.name === "dataset_description.json" + // ); + // datasetDesc = existing?.content ? JSON.parse(existing.content) : {}; + // } else { + // setStatus("1/3 Generating dataset_description.json..."); + // const ddPrompt = getDatasetDescriptionPrompt(userText, evidenceBundle); + + // let ddResponse; + // if (currentProvider.isAnthropic) { + // ddResponse = await fetch(currentProvider.baseUrl, { + // method: "POST", + // signal: controller.signal, + // headers: { + // "Content-Type": "application/json", + // "x-api-key": apiKey, + // "anthropic-version": "2023-06-01", + // }, + // body: JSON.stringify({ + // model, + // max_tokens: 2048, + // messages: [{ role: "user", content: ddPrompt }], + // }), + // }); + // } else if (provider === "ollama") { + + // ddResponse = await OllamaService.chat(model, [ + // { role: "user", content: ddPrompt }, + // ]); + // } else { + // ddResponse = await fetch(currentProvider.baseUrl, { + // method: "POST", + // signal: controller.signal, + // headers: { + // "Content-Type": "application/json", + // Authorization: `Bearer ${apiKey}`, + // }, + // body: JSON.stringify({ + // model, + // messages: [{ role: "user", content: ddPrompt }], + // max_tokens: 2048, + // }), + // }); + // } + + // // const ddData = await ddResponse.json(); + // const ddData = + // provider === "ollama" ? ddResponse : await ddResponse.json(); + // let ddText = currentProvider.isAnthropic + // ? ddData.content[0].text + // : ddData.choices[0].message.content; + + // // Clean up markdown fences + // ddText = ddText + // .replace(/^```json\n?/g, "") + // .replace(/\n?```$/g, "") + // .trim(); + // datasetDesc = JSON.parse(ddText); + // } + + // // ========================================== + // // Call 2: Generate README.md + // // ========================================== + // let readmeContent: string; + // if (evidenceBundle.trio_found?.["README.md"]) { + // setStatus("2/3 README.md already exists, skipping..."); + // const existing = files.find( + // (f) => + // f.source === "user" && + // ["README.md", "README.txt", "README.rst", "readme.md"].includes( + // f.name + // ) + // ); + // readmeContent = existing?.content || ""; + // } else { + // setStatus("2/3 Generating README.md..."); + // const readmePrompt = getReadmePrompt(userText); + + // let readmeResponse; + // if (currentProvider.isAnthropic) { + // readmeResponse = await fetch(currentProvider.baseUrl, { + // method: "POST", + // signal: controller.signal, + // headers: { + // "Content-Type": "application/json", + // "x-api-key": apiKey, + // "anthropic-version": "2023-06-01", + // }, + // body: JSON.stringify({ + // model, + // max_tokens: 2048, + // messages: [{ role: "user", content: readmePrompt }], + // }), + // }); + // } else if (provider === "ollama") { + + // readmeResponse = await OllamaService.chat(model, [ + // { role: "user", content: readmePrompt }, + // ]); + // } else { + // readmeResponse = await fetch(currentProvider.baseUrl, { + // method: "POST", + // signal: controller.signal, + // headers: { + // "Content-Type": "application/json", + // Authorization: `Bearer ${apiKey}`, + // }, + // body: JSON.stringify({ + // model, + // messages: [{ role: "user", content: readmePrompt }], + // max_tokens: 2048, + // }), + // }); + // } + + // const readmeData = + // provider === "ollama" ? readmeResponse : await readmeResponse.json(); + // readmeContent = currentProvider.isAnthropic + // ? readmeData.content[0].text + // : readmeData.choices[0].message.content; + // } + // // ========================================== + // // Call 3: Generate participants.tsv + // // ========================================== + // let participantsContent: string; + // if (evidenceBundle.trio_found?.["participants.tsv"]) { + // setStatus("3/3 participants.tsv already exists, skipping..."); + // const existing = files.find( + // (f) => f.source === "user" && f.name === "participants.tsv" + // ); + // participantsContent = existing?.content || ""; + // } else { + // setStatus("3/3 Generating participants.tsv..."); + // const partsPrompt = getParticipantsPrompt(userText); + + // const currentSubjectAnalysis = extractSubjectAnalysis( + // evidenceBundle?.all_files || [], + // evidenceBundle?.user_hints?.n_subjects, + // evidenceBundle?.filename_analysis?.python_statistics + // ?.dominant_prefixes + // ); + + // console.log("=== PARTICIPANTS DEBUG ==="); + // console.log("method:", currentSubjectAnalysis?.method); + // console.log("subject_count:", currentSubjectAnalysis?.subject_count); + // console.log( + // "id_mapping:", + // currentSubjectAnalysis?.id_mapping?.id_mapping + // ); + // console.log( + // "reverse_mapping:", + // currentSubjectAnalysis?.id_mapping?.reverse_mapping + // ); + // console.log( + // "subject_records sample:", + // currentSubjectAnalysis?.subject_records?.slice(0, 3) + // ); + // const idMap = currentSubjectAnalysis?.id_mapping?.id_mapping; + // const expectedCount = evidenceBundle?.user_hints?.n_subjects; + // const subjectLabels: string[] = + // idMap && + // Object.keys(idMap).length > 0 && + // (!expectedCount || Object.keys(idMap).length === expectedCount) + // ? Object.values(idMap).map((id: string) => `sub-${id}`) + // : Array.from( + // { + // length: expectedCount || Object.keys(idMap || {}).length || 1, + // }, + // (_, i) => `sub-${String(i + 1).padStart(2, "0")}` + // ); + + // let partsResponse; + // if (currentProvider.isAnthropic) { + // partsResponse = await fetch(currentProvider.baseUrl, { + // method: "POST", + // signal: controller.signal, + // headers: { + // "Content-Type": "application/json", + // "x-api-key": apiKey, + // "anthropic-version": "2023-06-01", + // }, + // body: JSON.stringify({ + // model, + // max_tokens: 1024, + // messages: [{ role: "user", content: partsPrompt }], + // }), + // }); + // } else if (provider === "ollama") { + + // partsResponse = await OllamaService.chat(model, [ + // { role: "user", content: partsPrompt }, + // ]); + // } else { + // partsResponse = await fetch(currentProvider.baseUrl, { + // method: "POST", + // signal: controller.signal, + // headers: { + // "Content-Type": "application/json", + // Authorization: `Bearer ${apiKey}`, + // }, + // body: JSON.stringify({ + // model, + // messages: [{ role: "user", content: partsPrompt }], + // max_tokens: 1024, + // }), + // }); + // } + + // // const partsData = await partsResponse.json(); + // const partsData = + // provider === "ollama" ? partsResponse : await partsResponse.json(); + // const participantsRaw = currentProvider.isAnthropic + // ? partsData.content[0].text + // : partsData.choices[0].message.content; + + // // Build TSV from schema + // // try { + // // const schemaText = participantsRaw + // // .replace(/^```json\n?/g, "") + // // .replace(/\n?```$/g, "") + // // .trim(); + // // const schema = JSON.parse(schemaText); + // // const columns: string[] = schema.columns.map((c: any) => c.name); + + // // // Get subject IDs from evidence bundle (extracted by Python-style analysis) + // // // const idMapping = + // // // evidenceBundle?.subject_analysis?.id_mapping?.id_mapping; + // // // const subjectLabels: string[] = idMapping + // // // ? Object.values(idMapping).map((id) => `sub-${id}`) + // // // : ["sub-01"]; // fallback if no subject analysis + // // // Get subject IDs from subjectAnalysis state (computed at plan stage) + // // // Fall back to computing fresh if plan hasn't been run yet + // // const currentSubjectAnalysis = + // // subjectAnalysis || + // // extractSubjectAnalysis( + // // evidenceBundle?.all_files || [], + // // evidenceBundle?.user_hints?.n_subjects, + // // evidenceBundle?.filename_analysis?.python_statistics + // // ?.dominant_prefixes + // // ); + // // const idMap = currentSubjectAnalysis?.id_mapping?.id_mapping; + // // const subjectLabels: string[] = + // // idMap && Object.keys(idMap).length > 0 + // // ? Object.values(idMap).map((id) => `sub-${id}`) + // // : Array.from( + // // { length: evidenceBundle?.user_hints?.n_subjects || 1 }, + // // (_, i) => `sub-${String(i + 1).padStart(2, "0")}` + // // ); + + // // const header = columns.join("\t"); + // // // ====origin==== + // // // const rows = subjectLabels.map((subId) => + // // // columns + // // // .map((col: string) => (col === "participant_id" ? subId : "n/a")) + // // // .join("\t") + // // // ); + // // //====== end ====== + // // // =====update start===== + // // const reverseMap = + // // currentSubjectAnalysis?.id_mapping?.reverse_mapping || {}; + // // const subjectRecords = currentSubjectAnalysis?.subject_records || []; + + // // const rows = subjectLabels.map((subId) => { + // // const bareId = subId.replace(/^sub-/, ""); + // // const originalId = reverseMap[bareId]; + // // const record = subjectRecords.find( + // // (r: any) => r.original_id === originalId + // // ); + // // return columns + // // .map((col: string) => { + // // if (col === "participant_id") return subId; + // // if (col === "original_id") return originalId || "n/a"; + // // if (col === "group") return (record as any)?.group || "n/a"; + // // return "n/a"; + // // }) + // // .join("\t"); + // // }); + // // //====update end====== + // // participantsContent = [header, ...rows].join("\n"); + // // } catch (e) { + // // // Fallback: LLM didn't return valid JSON schema, use raw content + // // participantsContent = participantsRaw + // // .replace(/^```\n?/g, "") + // // .replace(/\n?```$/g, "") + // // .trim(); + // // } + // // Build TSV from schema + subject analysis + // // Mirrors _generate_participants_tsv_from_python() in planner.py + // try { + // const schemaText = participantsRaw + // .replace(/^```json\n?/g, "") + // .replace(/\n?```$/g, "") + // .trim(); + // const schema = JSON.parse(schemaText); + + // // LLM decides extra demographic columns (sex, age, group etc.) + // // but we always add participant_id and original_id ourselves + // const extraColumns: string[] = schema.columns + // .map((c: any) => c.name) + // .filter( + // (name: string) => + // name !== "participant_id" && name !== "original_id" + // ); + + // // Always start with participant_id and original_id + // const columns = ["participant_id", "original_id", ...extraColumns]; + + // const reverseMap = + // currentSubjectAnalysis?.id_mapping?.reverse_mapping || {}; + // const subjectRecords = currentSubjectAnalysis?.subject_records || []; + + // const header = columns.join("\t"); + // const rows = subjectLabels.map((subId) => { + // const bareId = subId.replace(/^sub-/, ""); + // const originalId = reverseMap[bareId] || "n/a"; + // const record = subjectRecords.find( + // (r: any) => r.original_id === originalId + // ); + // return columns + // .map((col: string) => { + // if (col === "participant_id") return subId; + // if (col === "original_id") return originalId; + // if (col === "group") return (record as any)?.group || "n/a"; + // return "n/a"; + // }) + // .join("\t"); + // }); + + // participantsContent = [header, ...rows].join("\n"); + // } catch (e) { + // // Fallback: generate minimal TSV directly from subject analysis + // const reverseMap = + // currentSubjectAnalysis?.id_mapping?.reverse_mapping || {}; + // const header = "participant_id\toriginal_id"; + // const rows = subjectLabels.map((subId) => { + // const bareId = subId.replace(/^sub-/, ""); + // const originalId = reverseMap[bareId] || "n/a"; + // return `${subId}\t${originalId}`; + // }); + // participantsContent = [header, ...rows].join("\n"); + // } + // } + // // ========================================== + // // Add trio files to Virtual File System + // // ========================================== + // const timestamp = new Date().toLocaleString(); + // const trioFiles: FileItem[] = [ + // { + // id: generateId(), + // name: "dataset_description.json", + // type: "file", + // fileType: "meta", + // content: JSON.stringify(datasetDesc, null, 2), + // contentType: "text", + // isUserMeta: true, + // parentId: null, + // source: "ai", + // generatedAt: timestamp, + // }, + // { + // id: generateId(), + // name: "README.md", + // type: "file", + // fileType: "meta", + // content: readmeContent + // .replace(/^```markdown\n?/g, "") + // .replace(/\n?```$/g, "") + // .trim(), + // contentType: "text", + // isUserMeta: true, + // parentId: null, + // source: "ai", + // generatedAt: timestamp, + // }, + // { + // id: generateId(), + // name: "participants.tsv", + // type: "file", + // fileType: "meta", + // content: participantsContent + // .replace(/^```\n?/g, "") + // .replace(/\n?```$/g, "") + // .trim(), + // contentType: "text", + // isUserMeta: true, + // parentId: null, + // source: "ai", + // generatedAt: timestamp, + // }, + // ]; + // // replace existing trio files, add if not exist + // updateFiles((prev) => { + // const trioNames = [ + // "dataset_description.json", + // "README.md", + // "participants.tsv", + // ]; + + // // Remove old AI generated trio files + // const withoutOldTrio = prev.filter( + // (f) => !(f.source === "ai" && trioNames.includes(f.name)) + // ); + + // // Add new trio files + // // return [...withoutOldTrio, ...trioFiles]; + + // // Only add AI-generated files for ones that weren't user-uploaded + // const newTrioFiles = trioFiles.filter( + // (tf) => + // !evidenceBundle.trio_found?.[ + // tf.name as keyof typeof evidenceBundle.trio_found + // ] + // ); + + // return [...withoutOldTrio, ...newTrioFiles]; + // }); + // setTrioGenerated(true); + // setStatus( + // "✓ BIDS trio files generated and added to Virtual File System!" + // ); + // } catch (err: any) { + // if (err.name === "AbortError") { + // setStatus("❌ Generation cancelled"); + // } else { + // setError(err.message || "Failed to generate trio files"); + // setStatus("❌ Error generating trio files"); + // } + // } finally { + // setGeneratingTrio(false); + // setAbortController(null); // Clear controller + // } + // }; const handleMouseDown = (e: React.MouseEvent) => { setIsResizing(true); @@ -609,321 +822,222 @@ const LLMPanel: React.FC = ({ const currentProvider = llmProviders[provider]; - const handleGenerate = async () => { + const handleGeneratePlan = async () => { if (!currentProvider.noApiKey && !apiKey.trim()) { setError("Please enter an API key"); return; } - if (!baseDirectoryPath.trim()) { setError("Please enter a base directory path"); return; } - // Create abort controller const controller = new AbortController(); setAbortController(controller); - setLoading(true); setError(null); - setStatus(`Generating script using ${currentProvider.name}...`); - - const fileSummary = buildFileSummary(files); - const filePatterns = analyzeFilePatterns(files); - const userContext = getUserContext(files); - const annotations = getFileAnnotations(files); - // console.log("=== PROMPT BEING SENT TO LLM ==="); - // console.log(fileSummary); - // console.log(filePatterns); - // console.log(userContext); - // console.log("================================="); - - // UPDATED: Improved prompt that uses trio files - const prompt = getConversionScriptPrompt( - baseDirectoryPath, - fileSummary, - filePatterns, - userContext, - annotations - ); + setStatus(`Generating BIDSPlan.yaml using ${currentProvider.name}...`); try { - let response; - - if (provider === "ollama") { - // const ollamaBaseUrl = ollamaUrl || "http://localhost:11434"; - // response = await fetch(`${ollamaBaseUrl}/v1/chat/completions`, { - // method: "POST", - // signal: controller.signal, - // headers: { - // "Content-Type": "application/json", - // }, - // body: JSON.stringify({ - // model, - // messages: [ - // { - // role: "system", - // content: - // "You are a neuroimaging data expert specializing in BIDS format conversion. Output only Python code without markdown fences or explanations.", - // }, - // { role: "user", content: prompt }, - // ], - // stream: false, - // }), - // }); - response = await OllamaService.chat(model, [ - { - role: "system", - content: - "You are a neuroimaging data expert specializing in BIDS format conversion. Output only Python code without markdown fences or explanations.", - }, - { role: "user", content: prompt }, - ]); - } else if (currentProvider.isAnthropic) { - response = await fetch(currentProvider.baseUrl, { - method: "POST", - signal: controller.signal, - headers: { - "Content-Type": "application/json", - "x-api-key": apiKey, - "anthropic-version": "2023-06-01", - }, - body: JSON.stringify({ - model, - max_tokens: 4096, - messages: [{ role: "user", content: prompt }], - }), - }); - } else { - const headers: Record = { - "Content-Type": "application/json", - }; + const { + planYaml, + subjectAnalysis: sa, + participantsTsv, + coverageWarnings, + } = await buildBidsPlan({ + evidenceBundle, + llmConfig: buildLLMConfig(), + signal: controller.signal, + onStatus: setStatus, + }); - if (!currentProvider.noApiKey) { - headers["Authorization"] = `Bearer ${apiKey}`; - } + // Store subject analysis for ZIP packaging + setSubjectAnalysis(sa); - response = await fetch(currentProvider.baseUrl, { - method: "POST", - signal: controller.signal, - headers, - body: JSON.stringify({ - model, - messages: [ - { - role: "system", - content: - "You are a neuroimaging data expert specializing in BIDS format conversion. Output only Python code without markdown fences or explanations.", - }, - { role: "user", content: prompt }, - ], - max_tokens: 4096, - temperature: 0.7, - }), + // Dump final YAML string (planYaml is raw string from LLM, already cleaned) + setBidsPlan(planYaml); + + // Update participants.tsv in VFS with the full version from the plan stage + if (participantsTsv) { + const timestamp = new Date().toLocaleString(); + updateFiles((prev) => { + const withoutOld = prev.filter( + (f) => !(f.source === "ai" && f.name === "participants.tsv") + ); + return [ + ...withoutOld, + { + id: generateId(), + name: "participants.tsv", + type: "file" as const, + fileType: "meta", + content: participantsTsv, + contentType: "text", + isUserMeta: true, + parentId: null, + source: "ai" as const, + generatedAt: timestamp, + }, + ]; }); } - // const data = await response.json(); - const data = provider === "ollama" ? response : await response.json(); - - // if (!response.ok) { - // throw new Error(data.error?.message || "Failed to generate script"); - // } - if (!response.ok && provider !== "ollama") { - throw new Error(data.error?.message || "Failed to generate script"); + if (coverageWarnings.length > 0) { + setStatus( + `✓ BIDSPlan.yaml generated (${coverageWarnings.length} coverage warning(s) — check console)` + ); + } else { + setStatus(`✓ BIDSPlan.yaml generated using ${currentProvider.name}`); } - - // let script = ""; - // if (currentProvider.isAnthropic) { - // script = data.content[0].text; - // } else { - // script = data.choices[0].message.content; - // } - let script = currentProvider.isAnthropic - ? data.content[0].text - : data.choices[0].message.content; - - // Clean up markdown fences if AI included them anyway - script = script.replace(/^```python\n?/g, "").replace(/\n?```$/g, ""); - - setGeneratedScript(script); - setStatus(`✓ Script generated using ${currentProvider.name}`); } catch (err: any) { if (err.name === "AbortError") { setStatus("❌ Generation cancelled"); } else { - setError(err.message || "Failed to generate script"); - setStatus("❌ Error generating script"); + setError(err.message || "Failed to generate BIDSPlan"); + setStatus("❌ Error generating BIDSPlan"); } } finally { setLoading(false); - setAbortController(null); // Clear controller + setAbortController(null); } }; + // const handleGeneratePlan = async () => { + // if (!currentProvider.noApiKey && !apiKey.trim()) { + // setError("Please enter an API key"); + // return; + // } + // if (!baseDirectoryPath.trim()) { + // setError("Please enter a base directory path"); + // return; + // } - const handleGeneratePlan = async () => { - if (!currentProvider.noApiKey && !apiKey.trim()) { - setError("Please enter an API key"); - return; - } - if (!baseDirectoryPath.trim()) { - setError("Please enter a base directory path"); - return; - } - - const controller = new AbortController(); - setAbortController(controller); - setLoading(true); - setError(null); - setStatus(`Generating BIDSPlan.yaml using ${currentProvider.name}...`); - - // ── Compute subject analysis (mirrors planner.py Step 1) - const allFiles = evidenceBundle?.all_files || []; - const userNSubjects = evidenceBundle?.user_hints?.n_subjects; - const dominantPrefixes = - evidenceBundle?.filename_analysis?.python_statistics?.dominant_prefixes; - - const computedSubjectAnalysis = extractSubjectAnalysis( - allFiles, - userNSubjects, - dominantPrefixes - ); - setSubjectAnalysis(computedSubjectAnalysis); - - const fileSummary = buildFileSummary(files); - const filePatterns = analyzeFilePatterns(files); - const userContext = getUserContext(files); - // const subjectInfo = extractSubjectsFromFiles(files); - const subjectInfo = computedSubjectAnalysis; - const sampleFiles = - evidenceBundle?.samples - ?.slice(0, 10) - .map((s: any) => ` - ${s.relpath}`) - .join("\n") || ""; - - // console.log("=== SAMPLE FILES ==="); - // console.log(sampleFiles); - // console.log("=== COUNTS BY EXT ==="); - // console.log(evidenceBundle?.counts_by_ext); - - const prompt = getBIDSPlanPrompt( - fileSummary, - filePatterns, - userContext, - { - subjects: Object.entries( - computedSubjectAnalysis.id_mapping.id_mapping - ).map(([originalId, bidsId]) => ({ originalId, bidsId })), - strategy: computedSubjectAnalysis.id_mapping.strategy_used, - }, - evidenceBundle?.counts_by_ext || {}, - sampleFiles, - evidenceBundle - ); - - try { - let response; - - if (provider === "ollama") { - // const ollamaBaseUrl = ollamaUrl || "http://localhost:11434"; - // response = await fetch(`${ollamaBaseUrl}/v1/chat/completions`, { - // method: "POST", - // signal: controller.signal, - // headers: { "Content-Type": "application/json" }, - // body: JSON.stringify({ - // model, - // messages: [ - // { - // role: "system", - // content: - // "You are a BIDS dataset architect. Output only valid YAML without markdown fences or explanations.", - // }, - // { role: "user", content: prompt }, - // ], - // stream: false, - // }), - // }); - response = await OllamaService.chat(model, [ - { - role: "system", - content: - "You are a BIDS dataset architect. Output only valid YAML without markdown fences or explanations.", - }, - { role: "user", content: prompt }, - ]); - } else if (currentProvider.isAnthropic) { - response = await fetch(currentProvider.baseUrl, { - method: "POST", - signal: controller.signal, - headers: { - "Content-Type": "application/json", - "x-api-key": apiKey, - "anthropic-version": "2023-06-01", - }, - body: JSON.stringify({ - model, - max_tokens: 2048, - messages: [{ role: "user", content: prompt }], - }), - }); - } else { - response = await fetch(currentProvider.baseUrl, { - method: "POST", - signal: controller.signal, - headers: { - "Content-Type": "application/json", - Authorization: `Bearer ${apiKey}`, - }, - body: JSON.stringify({ - model, - messages: [ - { - role: "system", - content: - "You are a BIDS dataset architect. Output only valid YAML without markdown fences or explanations.", - }, - { role: "user", content: prompt }, - ], - max_tokens: 2048, - temperature: 0.15, - }), - }); - } - - // const data = await response.json(); + // const controller = new AbortController(); + // setAbortController(controller); + // setLoading(true); + // setError(null); + // setStatus(`Generating BIDSPlan.yaml using ${currentProvider.name}...`); + + // // ── Compute subject analysis (mirrors planner.py Step 1) + // const allFiles = evidenceBundle?.all_files || []; + // const userNSubjects = evidenceBundle?.user_hints?.n_subjects; + // const dominantPrefixes = + // evidenceBundle?.filename_analysis?.python_statistics?.dominant_prefixes; + + // const computedSubjectAnalysis = extractSubjectAnalysis( + // allFiles, + // userNSubjects, + // dominantPrefixes + // ); - // if (!response.ok) { - // throw new Error(data.error?.message || "Failed to generate BIDSPlan"); - // } - const data = provider === "ollama" ? response : await response.json(); - if (!response.ok && provider !== "ollama") { - throw new Error(data.error?.message || "Failed to generate BIDSPlan"); - } + // setSubjectAnalysis(computedSubjectAnalysis); + + // const fileSummary = buildFileSummary(files); + // const filePatterns = analyzeFilePatterns(files); + // const userContext = getUserContext(files); + // // const subjectInfo = extractSubjectsFromFiles(files); + // const subjectInfo = computedSubjectAnalysis; + // const sampleFiles = + // evidenceBundle?.samples + // ?.slice(0, 10) + // .map((s: any) => ` - ${s.relpath}`) + // .join("\n") || ""; + + // const prompt = getBIDSPlanPrompt( + // fileSummary, + // filePatterns, + // userContext, + // { + // subjects: Object.entries( + // computedSubjectAnalysis.id_mapping.id_mapping + // ).map(([originalId, bidsId]) => ({ originalId, bidsId })), + // strategy: computedSubjectAnalysis.id_mapping.strategy_used, + // }, + // evidenceBundle?.counts_by_ext || {}, + // sampleFiles, + // evidenceBundle + // ); - let plan = currentProvider.isAnthropic - ? data.content[0].text - : data.choices[0].message.content; + // try { + // let response; + + // if (provider === "ollama") { + + // response = await OllamaService.chat(model, [ + // { + // role: "system", + // content: + // "You are a BIDS dataset architect. Output only valid YAML without markdown fences or explanations.", + // }, + // { role: "user", content: prompt }, + // ]); + // } else if (currentProvider.isAnthropic) { + // response = await fetch(currentProvider.baseUrl, { + // method: "POST", + // signal: controller.signal, + // headers: { + // "Content-Type": "application/json", + // "x-api-key": apiKey, + // "anthropic-version": "2023-06-01", + // }, + // body: JSON.stringify({ + // model, + // max_tokens: 2048, + // messages: [{ role: "user", content: prompt }], + // }), + // }); + // } else { + // response = await fetch(currentProvider.baseUrl, { + // method: "POST", + // signal: controller.signal, + // headers: { + // "Content-Type": "application/json", + // Authorization: `Bearer ${apiKey}`, + // }, + // body: JSON.stringify({ + // model, + // messages: [ + // { + // role: "system", + // content: + // "You are a BIDS dataset architect. Output only valid YAML without markdown fences or explanations.", + // }, + // { role: "user", content: prompt }, + // ], + // max_tokens: 2048, + // temperature: 0.15, + // }), + // }); + // } - // Clean up markdown fences if present - plan = plan - .replace(/^```yaml\n?/g, "") - .replace(/\n?```$/g, "") - .trim(); + // const data = provider === "ollama" ? response : await response.json(); + // if (!response.ok && provider !== "ollama") { + // throw new Error(data.error?.message || "Failed to generate BIDSPlan"); + // } - setBidsPlan(plan); - setStatus(`✓ BIDSPlan.yaml generated using ${currentProvider.name}`); - } catch (err: any) { - if (err.name === "AbortError") { - setStatus("❌ Generation cancelled"); - } else { - setError(err.message || "Failed to generate BIDSPlan"); - setStatus("❌ Error generating BIDSPlan"); - } - } finally { - setLoading(false); - setAbortController(null); - } - }; + // let plan = currentProvider.isAnthropic + // ? data.content[0].text + // : data.choices[0].message.content; + + // // Clean up markdown fences if present + // plan = plan + // .replace(/^```yaml\n?/g, "") + // .replace(/\n?```$/g, "") + // .trim(); + + // setBidsPlan(plan); + // setStatus(`✓ BIDSPlan.yaml generated using ${currentProvider.name}`); + // } catch (err: any) { + // if (err.name === "AbortError") { + // setStatus("❌ Generation cancelled"); + // } else { + // setError(err.message || "Failed to generate BIDSPlan"); + // setStatus("❌ Error generating BIDSPlan"); + // } + // } finally { + // setLoading(false); + // setAbortController(null); + // } + // }; const handleDownloadPlan = () => { const blob = new Blob([bidsPlan], { type: "text/yaml" }); @@ -1012,7 +1126,10 @@ const LLMPanel: React.FC = ({ const handleSaveZip = async () => { // Add output files to VFS const timestamp = new Date().toLocaleString(); - const zipLabel = `bids_output_${new Date().toISOString().slice(0, 10)}`; + const now = new Date(); + const dateStr = now.toISOString().slice(0, 10); + const timeStr = now.toTimeString().slice(0, 8).replace(/:/g, "-"); + const zipLabel = `bids_output_${dateStr}_${timeStr}`; const outputFiles: FileItem[] = []; const folderId = generateId(); @@ -1120,7 +1237,7 @@ const LLMPanel: React.FC = ({ }); updateFiles((prev) => [...prev, ...outputFiles]); - setStatus("✓ Saved to VFS. Click 'Save Changes' to persist to database."); + setStatus(isPrivateMode ? "✓ Added to file tree." : "✓ Added to file tree. Click 'Save Changes' to persist to database."); }; // const handleSaveZip = async () => { // const zip = new JSZip(); @@ -1262,7 +1379,38 @@ const LLMPanel: React.FC = ({ sx={{ display: "flex", alignItems: "center", gap: 1 }} > - AI-Generated BIDS Conversion Script + AI Assistant + + + + + @@ -1291,14 +1439,23 @@ const LLMPanel: React.FC = ({ setModel(llmProviders[e.target.value].models[0].id); }} > - {Object.entries(llmProviders).map(([key, p]) => ( - - {p.name} - - ))} + {Object.entries(llmProviders) + .filter(([key]) => isPrivateMode ? key !== "ollama" : key !== "local-ollama") + .map(([key, p]) => ( + + {p.name} + + ))} + {provider === "ollama" && ( + + Using qwen3.6:27b on NeuroJSON server + + )} + + {provider !== "ollama" && ( Model + )} + + {provider === "local-ollama" && ( + { + if (e.target.value.trim()) setModel(e.target.value.trim()); + }} + /> + )} + + {isPrivateMode && provider !== "local-ollama" && ( + + Your file information will be sent to {currentProvider.name}, an external AI service. Switch to Local AI (Ollama / LM Studio / Jan) to keep everything local. + + )} - {/* Ollama Server URL field */} - {/* {provider === "ollama" && ( + {provider === "local-ollama" && ( setOllamaUrl(e.target.value)} + label="Ollama URL" + value={localOllamaUrl} + onChange={(e) => setLocalOllamaUrl(e.target.value)} placeholder="http://localhost:11434" + helperText="Ollama: port 11434 · LM Studio: port 1234 · Jan: port 1337" sx={{ mb: 2 }} /> - )} */} + )} {/* Base Directory Path field (shows for ALL providers) */} = ({ )} - {/* = ({ size="small" multiline rows={2} - /> */} + sx={{ mb: 1 }} + /> {/* = ({ "&.Mui-disabled": { background: "#e0e0e0", color: "#9e9e9e" }, }} > - {loading ? "Generating..." : "3. Generate BIDSPlan.yaml"} + {loading ? "Generating..." : "3. Generate Conversion Package"} {/* - {/* */} @@ -1627,11 +1801,13 @@ const LLMPanel: React.FC = ({ color: "#d4d4d4", }} > - {/* {generatedScript || - 'Configure your LLM provider and click "Generate Script"...'} */} - {bidsPlan || - generatedScript || - 'Configure your LLM provider and click "Generate BIDSPlan.yaml"...'} + {bidsPlan || generatedScript || ( + + {status && !error + ? status + : 'Fill in the fields on the left and follow the steps to generate your conversion package...'} + + )} diff --git a/src/components/User/Dashboard/DatasetOrganizer/index.tsx b/src/components/User/Dashboard/DatasetOrganizer/index.tsx index ec8d97f..796fbaa 100644 --- a/src/components/User/Dashboard/DatasetOrganizer/index.tsx +++ b/src/components/User/Dashboard/DatasetOrganizer/index.tsx @@ -14,6 +14,8 @@ import { DialogContent, DialogActions, DialogContentText, + Chip, + Tooltip, } from "@mui/material"; import { Colors } from "design/theme"; import { useAppDispatch } from "hooks/useAppDispatch"; @@ -263,6 +265,55 @@ const DatasetOrganizer: React.FC = () => { {currentProject.description} )} + + An LLM-powered tool for automatically converting neuroimaging datasets into BIDS-compliant format. + window.open("https://github.com/COTILab/autobidsify", "_blank")} + > + Learn more + + + } + placement="bottom-start" + arrow + componentsProps={{ + tooltip: { + sx: { + backgroundColor: "white", + color: Colors.darkPurple, + border: `1px solid ${Colors.lightGray}`, + boxShadow: 3, + fontSize: "0.875rem", + p: 1.5, + maxWidth: 320, + }, + }, + arrow: { + sx: { + color: "white", + "&::before": { border: `1px solid ${Colors.lightGray}` }, + }, + }, + }} + > + window.open("https://github.com/COTILab/autobidsify", "_blank")} + sx={{ + mt: 0.5, + backgroundColor: Colors.purple, + color: Colors.white, + fontSize: "0.7rem", + cursor: "pointer", + "&:hover": { backgroundColor: Colors.secondaryPurple }, + }} + /> + @@ -271,7 +322,6 @@ const DatasetOrganizer: React.FC = () => { variant="contained" startIcon={} onClick={() => setShowLLMPanel(!showLLMPanel)} - disabled={files.length === 0} sx={{ backgroundColor: Colors.purple, color: Colors.lightGray, @@ -281,9 +331,9 @@ const DatasetOrganizer: React.FC = () => { }, }} > - Generate BIDS Plan + AI Assistant - {/* */} + + + + AutoBIDSify + + How to use: + +
  • Drop your dataset files into the workspace.
    • +
  • Enter the number of subjects, modality, and base directory path.
    • +
  • The AI will analyze your files and generate a BIDS conversion plan.
    • +
  • Download and run the script locally to reorganize your data into BIDS format.
    • +     +     + } + placement="bottom-start" + arrow + componentsProps={{ + tooltip: { + sx: { + backgroundColor: "white", + color: Colors.darkPurple, + border: `1px solid ${Colors.lightGray}`, + boxShadow: 3, + fontSize: "0.875rem", + lineHeight: 1.5, + p: 1.5, + maxWidth: 320, + }, + }, + arrow: { + sx: { + color: "white", + "&::before": { border: `1px solid ${Colors.lightGray}` }, + }, + }, + }} + > + + + + +     + + An LLM-powered tool for automatically converting neuroimaging datasets into BIDS-compliant format.{" "} + window.open("https://github.com/COTILab/autobidsify", "_blank")} + sx={{ color: Colors.purple, cursor: "pointer", textDecoration: "underline" }} + > + Learn more + + + + + + + + + + + + {/* Mode indicator bar */} + + + + + Private Mode + + + + Save to Account + + + + + {mode === "private" + ? "Files are processed locally. Nothing is uploaded. All data is lost when you close this page." + : "Log in to save your work to a project on your account."} + + + + {error && ( + setError(null)} sx={{ m: 2 }}> + {error} + + )} + + {/* Main Content */} + + + + {showLLMPanel && ( + setShowLLMPanel(false)} + isPrivateMode={mode === "private"} + /> + )} + + + + + + {/* Welcome dialog — shown on first load before user starts working */} + + + How would you like to use AutoBIDSify? + + + + An LLM-powered tool for automatically converting neuroimaging datasets into BIDS-compliant format.{" "} + window.open("https://github.com/COTILab/autobidsify", "_blank")} + sx={{ color: Colors.purple, cursor: "pointer", textDecoration: "underline" }} + > + Learn more + + + + {/* Private Mode card */} + + + + Private Mode + + + Work entirely in your browser. No files are uploaded to any + server. All data will be lost when you close the page. + + + + {/* Save to Account card */} + + + + Save to Account + + + Log in to save your work to a project. You can resume it any + time from your dashboard. + + + + + + + { + setLoginOpen(false); + // If user closes login without logging in, fall back to private mode + if (!isLoggedIn) { + setMode("private"); + setModeChosen(true); + } + }} + onSwitchToSignup={() => { + setLoginOpen(false); + setSignupOpen(true); + }} + /> + { + setSignupOpen(false); + if (!isLoggedIn) { + setMode("private"); + setModeChosen(true); + } + }} + onSwitchToLogin={() => { + setSignupOpen(false); + setLoginOpen(true); + }} + /> + + ); +}; + +export default BidsConverterPage; diff --git a/src/pages/SearchPage.tsx b/src/pages/SearchPage.tsx index 8e35494..df49761 100644 --- a/src/pages/SearchPage.tsx +++ b/src/pages/SearchPage.tsx @@ -13,6 +13,12 @@ import { Tooltip, IconButton, Alert, + Slider, + Stack, + TextField, + ToggleButton, + ToggleButtonGroup, + Autocomplete, } from "@mui/material"; import { useTheme } from "@mui/material/styles"; import useMediaQuery from "@mui/material/useMediaQuery"; @@ -22,6 +28,8 @@ import ClickTooltip from "components/SearchPage/ClickTooltip"; import DatabaseCard from "components/SearchPage/DatabaseCard"; import DatasetCard from "components/SearchPage/DatasetCard"; import SubjectCard from "components/SearchPage/SubjectCard"; +import { FileTypeAutocompleteWidget } from "components/SearchPage/widgets/FileTypeAutocompleteWidget"; +import { TypeAutocompleteWidget } from "components/SearchPage/widgets/TypeAutocompleteWidget"; import { Colors } from "design/theme"; import { useAppDispatch } from "hooks/useAppDispatch"; import { useAppSelector } from "hooks/useAppSelector"; @@ -29,6 +37,7 @@ import pako from "pako"; import React from "react"; import { useState, useEffect, useMemo } from "react"; import { + fetchFileTypes, fetchMetadataSearchResults, fetchRegistry, } from "redux/neurojson/neurojson.action"; @@ -45,6 +54,194 @@ type RegistryItem = { logo?: string; }; +// Module-scope so the component identity is stable across SearchPage renders. +// An inline arrow function inside customFields was getting a new identity each +// render, which made RJSF remount the slider mid-drag. +const AGE_MIN_BOUND = 0; +const AGE_MAX_BOUND = 100; + +const DATASET_MODALITIES = [ + "anat", "func", "dwi", "fmap", "perf", + "meg", "eeg", "ieeg", "beh", "pet", + "micr", "nirs", "motion", "ephys", "atlas", + "JMesh", "JNIFTI", "JSNIRF", "JData", +]; + +const DatasetModalityFilterField = (props: any) => { + const ctx = props?.registry?.formContext as + | { formData: Record; setFormData: React.Dispatch>> } + | undefined; + if (!ctx) return null; + const { formData, setFormData } = ctx; + const selected: string[] = Array.isArray(formData.modalities) ? formData.modalities : []; + const mode: string = formData.modality_mode || "or"; + + const handleChange = (_: any, next: string[]) => { + setFormData((prev) => { + const updated = { ...prev }; + if (next.length === 0) { + delete updated.modalities; + delete updated.modality_mode; + } else { + updated.modalities = next; + if (!updated.modality_mode) updated.modality_mode = "or"; + } + return updated; + }); + }; + + const handleModeChange = (_: any, val: string | null) => { + if (!val) return; + setFormData((prev) => ({ ...prev, modality_mode: val })); + }; + + return ( + + + items.map((item, index) => ( + + )) + } + renderInput={(params) => ( + + )} + /> + {selected.length > 1 && ( + + + OR + AND + + + {mode === "and" ? "must have all selected" : "must have any selected"} + + + )} + + ); +}; + +const AgeRangeSliderField = (props: any) => { + const ctx = props?.registry?.formContext as + | { + formData: Record; + setFormData: React.Dispatch>>; + } + | undefined; + if (!ctx) return null; + const { formData, setFormData } = ctx; + const lo = + typeof formData.age_min === "number" ? formData.age_min : AGE_MIN_BOUND; + const hi = + typeof formData.age_max === "number" ? formData.age_max : AGE_MAX_BOUND; + const isAny = lo === AGE_MIN_BOUND && hi === AGE_MAX_BOUND; + return ( + + + Age: {isAny ? "Any" : `${lo} – ${hi}`} + + { + const [newLo, newHi] = v as number[]; + setFormData((prev) => { + const next = { ...prev }; + // Each handle is its own filter. A handle at the bound means + // "no constraint on that side", so we leave it out of formData + // (otherwise age_min=0 silently excludes unknown-age subjects + // whose stored key is "000-1", lexicographically below "00000"). + if (newLo === AGE_MIN_BOUND) delete next.age_min; + else next.age_min = newLo; + if (newHi === AGE_MAX_BOUND) delete next.age_max; + else next.age_max = newHi; + return next; + }); + }} + valueLabelDisplay="auto" + min={AGE_MIN_BOUND} + max={AGE_MAX_BOUND} + step={1} + disableSwap + sx={{ color: Colors.purple }} + /> + + ); +}; + +// Pairs a "_min" + "_max" into a single row of two number inputs. +// Reads target field names + label from uiSchema's ui:options: +// { minKey: "sess_min", maxKey: "sess_max", label: "sessions" } +const CountRangePairField = (props: any) => { + const ctx = props?.registry?.formContext as + | { + formData: Record; + setFormData: React.Dispatch>>; + } + | undefined; + const opts = props?.uiSchema?.["ui:options"] || {}; + const minKey = opts.minKey as string; + const maxKey = opts.maxKey as string; + const label = (opts.label as string) || ""; + if (!ctx || !minKey || !maxKey) return null; + const { formData, setFormData } = ctx; + const minVal = formData[minKey] ?? ""; + const maxVal = formData[maxKey] ?? ""; + + const update = (key: string, raw: string) => { + setFormData((prev) => { + const next = { ...prev }; + if (raw === "" || raw === undefined) { + delete next[key]; + } else { + const n = Number(raw); + if (Number.isNaN(n)) delete next[key]; + else next[key] = n; + } + return next; + }); + }; + + return ( + + update(minKey, e.target.value)} + fullWidth + inputProps={{ min: 0 }} + /> + update(maxKey, e.target.value)} + fullWidth + inputProps={{ min: 0 }} + /> + + ); +}; + const matchesKeyword = (item: RegistryItem, keyword: string) => { if (!keyword) return false; const needle = keyword.toLowerCase(); @@ -71,6 +268,9 @@ const SearchPage: React.FC = () => { const registry = useAppSelector( (state: RootState) => state.neurojson.registry ); + const fileTypes = useAppSelector( + (state: RootState) => state.neurojson.fileTypes + ); const loading = useAppSelector((state: RootState) => state.neurojson.loading); const [formData, setFormData] = useState>({}); @@ -91,9 +291,10 @@ const SearchPage: React.FC = () => { const placement = upMd ? "right" : "top"; - // for database card - const keywordInput = String(formData?.keyword ?? "").trim(); - const selectedDbId = String(formData?.database ?? "").trim(); + // inputs for the "Suggested databases" memo — read from appliedFilters so + // the suggestion list refreshes only on Search click, matching the results. + const keywordInput = String(appliedFilters?.keyword ?? "").trim(); + const selectedDbId = String(appliedFilters?.database ?? "").trim(); const registryMatches: RegistryItem[] = React.useMemo(() => { if (!Array.isArray(registry)) return []; @@ -119,10 +320,12 @@ const SearchPage: React.FC = () => { ([key, value]) => key !== "skip" && key !== "limit" && + key !== "modality_mode" && value !== undefined && value !== null && value !== "" && - value !== "any" + value !== "any" && + !(Array.isArray(value) && value.length === 0) ); useEffect(() => { @@ -207,14 +410,26 @@ const SearchPage: React.FC = () => { // form UI const uiSchema = useMemo( - () => generateUiSchema(formData, showSubjectFilters, showDatasetFilters), - [formData, showSubjectFilters, showDatasetFilters] + () => + generateUiSchema( + formData, + showSubjectFilters, + showDatasetFilters, + fileTypes || [] + ), + [formData, showSubjectFilters, showDatasetFilters, fileTypes] ); + // Custom RJSF widgets — comboboxes for the Data type and File types fields. + const customWidgets = { + typeAutocomplete: TypeAutocompleteWidget, + fileTypeAutocomplete: FileTypeAutocompleteWidget, + }; + // Create the "Subject-level Filters" button as a custom field const customFields = { subjectFiltersToggle: () => ( - + + + + + + ), datasetFiltersToggle: () => ( @@ -247,6 +478,9 @@ const SearchPage: React.FC = () => { ), + ageRangeSlider: AgeRangeSliderField, + countRangePair: CountRangePairField, + datasetModalityFilter: DatasetModalityFilterField, }; // determine the results are subject-level or dataset-level @@ -266,6 +500,11 @@ const SearchPage: React.FC = () => { dispatch(fetchRegistry()); }, [dispatch]); + // get the distinct file extensions for the "File types" multi-select. + useEffect(() => { + dispatch(fetchFileTypes()); + }, [dispatch]); + // dynamically add database enum to schema const schema = useMemo(() => { const dbList = registry?.length @@ -400,6 +639,8 @@ const SearchPage: React.FC = () => { onChange={({ formData }) => setFormData(formData)} uiSchema={uiSchema} fields={customFields} + widgets={customWidgets} + formContext={{ formData, setFormData }} /> ); @@ -418,6 +659,35 @@ const SearchPage: React.FC = () => { !loading && // !hasDbMatches && (!hasDatasetMatches || backendEmpty); + + // Tailored empty-state message: when the user combined a file_type filter + // with any subject-level filter and got nothing back, it's almost certainly + // because the file extension lives in non-BIDS datasets (which have no + // subject rows in ioviews). The generic "adjust filters" message hides this. + const SUBJECT_FILTER_KEYS = [ + "age_min", + "age_max", + "gender", + "task_min", + "task_max", + "task_name", + "run_min", + "run_max", + "run_name", + "sess_min", + "sess_max", + "session_name", + "type_name", + "modality", + "subject", + ]; + const isAppliedFilter = (v: any) => + v !== "" && v !== "any" && v !== undefined && v !== null; + const showFileTypeNonBidsHint = + showNoResults && + Array.isArray(appliedFilters.file_type) && + appliedFilters.file_type.length > 0 && + SUBJECT_FILTER_KEYS.some((k) => isAppliedFilter(appliedFilters[k])); return ( { mt: 1, }} > - {activeFilters.map(([key, value]) => ( + {activeFilters.map(([key, value]) => { + let label = `${String(key)}: ${String(value)}`; + if (key === "modalities" && Array.isArray(value)) { + const mode = appliedFilters.modality_mode || "or"; + label = `modalities (${mode}): ${value.join(", ")}`; + } + return ( { } }} /> - ))} + ); + })} )} @@ -635,10 +912,9 @@ const SearchPage: React.FC = () => { }} title={ - Live preview based on your keyword or selected database. - This list updates as you type or change the dropdown. - It’s separate from the results—you’ll - see datasets/subjects after you click Search. + Databases that match your keyword or selected database + filter. This list refreshes when you click{" "} + Search, alongside the datasets/subjects below. } > @@ -668,7 +944,7 @@ const SearchPage: React.FC = () => { datasets={db.datasets} modalities={db.datatype} logo={db.logo} - keyword={formData.keyword} // for keyword highlight + keyword={appliedFilters.keyword} // highlight the searched keyword, not the live input onChipClick={handleChipClick} /> ))} @@ -833,7 +1109,14 @@ const SearchPage: React.FC = () => { dsname={item.dsname} parsedJson={parsedJson} onChipClick={handleChipClick} - keyword={formData.keyword} // for keyword highlight + keyword={appliedFilters.keyword} // highlight what was searched, not the live form + matchingFiles={ + item.matching_files + ? JSON.parse(item.matching_files) + : undefined + } + matchingFilesTotal={item.matching_files_total} + fileTypes={appliedFilters.file_type} /> ) : ( { {...item} parsedJson={parsedJson} onChipClick={handleChipClick} + age={parsedJson?.key?.[0]} /> ); } catch (e) { @@ -869,10 +1153,20 @@ const SearchPage: React.FC = () => { Search Results - - No datasets or subjects found. Please adjust the - filters and try again. - + {showFileTypeNonBidsHint ? ( + + No matching subjects found. The selected file type + may only exist in non-BIDS datasets (e.g. mesh or + atlas libraries), which have no subject-level + records. Try removing subject-level filters + (modality, age, gender, etc.) and search again. + + ) : ( + + No datasets or subjects found. Please adjust the + filters and try again. + + )} )} diff --git a/src/pages/UpdatedDatasetDetailPage.tsx b/src/pages/UpdatedDatasetDetailPage.tsx index 52f87ca..835a6b5 100644 --- a/src/pages/UpdatedDatasetDetailPage.tsx +++ b/src/pages/UpdatedDatasetDetailPage.tsx @@ -7,6 +7,7 @@ import ExpandLess from "@mui/icons-material/ExpandLess"; import ExpandMore from "@mui/icons-material/ExpandMore"; import HomeIcon from "@mui/icons-material/Home"; import InfoOutlinedIcon from "@mui/icons-material/InfoOutlined"; +import KeyboardArrowDownIcon from "@mui/icons-material/KeyboardArrowDown"; import { Box, Typography, @@ -17,6 +18,9 @@ import { Collapse, Tooltip, IconButton, + Menu, + MenuItem, + Snackbar, } from "@mui/material"; import DatasetActions from "components/DatasetDetailPage/DatasetAction"; import FileTree from "components/DatasetDetailPage/FileTree/FileTree"; @@ -68,6 +72,7 @@ import { fetchDbInfoByDatasetId, } from "redux/neurojson/neurojson.action"; import { NeurojsonSelector } from "redux/neurojson/neurojson.selector"; +import { resetDocument } from "redux/neurojson/neurojson.slice"; // import { NeurojsonService } from "services/neurojson.service"; import RoutesEnum from "types/routes.enum"; @@ -262,12 +267,27 @@ const UpdatedDatasetDetailPage: React.FC = () => { const [searchParams, setSearchParams] = useSearchParams(); const focus = searchParams.get("focus") || undefined; // get highlight from url const rev = searchParams.get("rev") || undefined; // get revision from url - + const [chart2DPreviewPath, setChart2DPreviewPath] = useState(""); const [externalLinks, setExternalLinks] = useState([]); const [internalLinks, setInternalLinks] = useState([]); const [isInternalExpanded, setIsInternalExpanded] = useState(true); - const [downloadScript, setDownloadScript] = useState(""); + // Three script formats generated client-side: bash (Mac/Linux), batch + // (Windows), and a plain URL list. Same files in all three; only the + // wrapper syntax differs. + const [downloadScripts, setDownloadScripts] = useState<{ + sh: string; + bat: string; + txt: string; + }>({ sh: "", bat: "", txt: "" }); const [downloadScriptSize, setDownloadScriptSize] = useState(0); + // Dropdown state for the download format menu. + const [downloadMenuEl, setDownloadMenuEl] = useState( + null + ); + // Post-download instruction snackbar. Stays open until user dismisses. + const [downloadHint, setDownloadHint] = useState< + "sh" | "bat" | "txt" | null + >(null); const [totalFileSize, setTotalFileSize] = useState(0); const [previewIsInternal, setPreviewIsInternal] = useState(false); const [isExternalExpanded, setIsExternalExpanded] = useState(true); @@ -284,6 +304,14 @@ const UpdatedDatasetDetailPage: React.FC = () => { ? rawSummary : Object.values(rawSummary).filter(Boolean).join("\n\n"); const readme = datasetDocument?.["README"] ?? ""; + + useEffect(() => { + window.__clear2DPath = () => setChart2DPreviewPath(""); + return () => { + delete window.__clear2DPath; + }; + }, []); + const handleSelectRevision = (newRev?: string | null) => { setSearchParams((prev) => { const p = new URLSearchParams(prev); // copy of the query url @@ -335,7 +363,7 @@ const UpdatedDatasetDetailPage: React.FC = () => { : "Unknown Size"; const parts = currentPath.split("/"); - const subpath = parts.slice(-3).join("/"); + const subpath = parts.slice(-6).join("/"); const label = parentKey || "ExternalData"; links.push({ @@ -478,6 +506,12 @@ const UpdatedDatasetDetailPage: React.FC = () => { }; }, []); + // clean old dataset detail and metadata panel(include rev) + useEffect(() => { + dispatch(resetDocument()); // clear redux state + setRevsList([]); // clear local state + }, [dbName, docId, dispatch]); + useEffect(() => { if (!dbName || !docId) return; @@ -493,10 +527,10 @@ const UpdatedDatasetDetailPage: React.FC = () => { const fromDoc = Array.isArray(datasetDocument?._revs_info) ? (datasetDocument._revs_info as { rev: string }[]) : []; - if (fromDoc.length && revsList.length === 0) { - setRevsList(fromDoc); + if (fromDoc.length > 0) { + setRevsList(fromDoc); // only update when we have revisions } - }, [datasetDocument, revsList.length]); + }, [datasetDocument]); useEffect(() => { if (datasetDocument) { @@ -562,31 +596,62 @@ const UpdatedDatasetDetailPage: React.FC = () => { // }); // setJsonSize(blob.size); - // Construct download script dynamically - let script = `curl -L --create-dirs "https://neurojson.io:7777/${dbName}/${docId}" -o "${docId}.json"\n`; - - links.forEach((link) => { - const url = link.url; - const match = url.match(/file=([^&]+)/); - - const filename = match - ? (() => { - try { - return decodeURIComponent(match[1]); - } catch { - return match[1]; // fallback if decode fails - } - })() - : `file-${link.index}`; - - const outputPath = `$HOME/.neurojson/io/${dbName}/${docId}/${filename}`; - - script += `curl -L --create-dirs "${url}" -o "${outputPath}"\n`; - }); - setDownloadScript(script); - // Calculate and set script size - const scriptBlob = new Blob([script], { type: "text/plain" }); - setDownloadScriptSize(scriptBlob.size); + // Construct download scripts (three formats) dynamically — everything + // lands in a .// folder next to where the user runs the script. + // JSON metadata and data files stay together (was split between cwd + // and ~/.neurojson/io/... previously, hard to find). + const docUrl = `https://neurojson.io:7777/${dbName}/${docId}`; + type DlItem = { url: string; filename: string }; + const items: DlItem[] = [ + { url: docUrl, filename: `${docId}.json` }, + ...links.map((link) => { + const match = link.url.match(/file=([^&]+)/); + const filename = match + ? (() => { + try { + return decodeURIComponent(match[1]); + } catch { + return match[1]; + } + })() + : `file-${link.index}`; + return { url: link.url, filename }; + }), + ]; + + // Bash script (Mac/Linux) + const sh = + `#!/bin/bash\n` + + `# Downloads ${docId} from ${dbName}\n` + + `# Usage: bash ${docId}.sh\n` + + `set -e\n` + + `mkdir -p "${docId}"\n` + + `cd "${docId}" || exit 1\n` + + items + .map((it) => `curl -L -C - -o "${it.filename}" "${it.url}"`) + .join("\n") + + `\necho "Done. Files saved to $(pwd)"\n`; + + // Batch script (Windows) — curl ships with Windows 10+. CRLF endings. + const bat = + `@echo off\r\n` + + `REM Downloads ${docId} from ${dbName}\r\n` + + `REM Usage: double-click or run ${docId}.bat\r\n` + + `if not exist "${docId}" mkdir "${docId}"\r\n` + + `cd /d "${docId}"\r\n` + + items + .map((it) => `curl -L -C - -o "${it.filename}" "${it.url}"`) + .join("\r\n") + + `\r\necho Done. Files saved to %cd%\r\n` + + `pause\r\n`; + + // Plain URL list — for advanced users with wget. + const txt = items.map((it) => it.url).join("\n") + "\n"; + + setDownloadScripts({ sh, bat, txt }); + // Size shown on the button is the .sh script size (representative). + const shBlob = new Blob([sh], { type: "text/plain" }); + setDownloadScriptSize(shBlob.size); } }, [datasetDocument, docId]); @@ -612,20 +677,33 @@ const UpdatedDatasetDetailPage: React.FC = () => { document.body.removeChild(link); }; - const handleDownloadScript = () => { - const blob = new Blob([downloadScript], { type: "text/plain" }); + // Trigger download of the selected script format. Programmatic anchor + // click triggers the browser's normal download flow without navigating. + const handleDownloadScript = (format: "sh" | "bat" | "txt") => { + const content = downloadScripts[format]; + if (!content) return; + const mime = + format === "sh" ? "application/x-sh" : "text/plain"; + const filename = + format === "txt" ? `${docId}_manifest.txt` : `${docId}.${format}`; + const blob = new Blob([content], { type: `${mime}; charset=utf-8` }); const link = document.createElement("a"); link.href = URL.createObjectURL(blob); - link.download = `${docId}.sh`; + link.download = filename; document.body.appendChild(link); link.click(); document.body.removeChild(link); + URL.revokeObjectURL(link.href); + setDownloadMenuEl(null); + setDownloadHint(format); }; const handlePreview = ( dataOrUrl: string | any, idx: number, - isInternal: boolean = false + isInternal: boolean = false, + previewPath: string = "", + displayNumber?: number ) => { // console.log( // "🟢 Preview button clicked for:", @@ -635,6 +713,9 @@ const UpdatedDatasetDetailPage: React.FC = () => { // "Is Internal:", // isInternal // ); + setChart2DPreviewPath( + displayNumber ? `[${displayNumber}] ${previewPath}` : previewPath + ); // Clear any stale preview type from last run delete (window as any).__previewType; @@ -783,14 +864,14 @@ const UpdatedDatasetDetailPage: React.FC = () => { // Try internal data first const internal = internalMap.get(previewPath); if (internal) { - handlePreview(internal.data, internal.index, true); + handlePreview(internal.data, internal.index, true, previewPath); return; } // Then try external data by JSON path const external = linkMap.get(previewPath); if (external) { - handlePreview(external.url, external.index, false); + handlePreview(external.url, external.index, false, previewPath); } }, [ datasetDocument, @@ -1098,20 +1179,35 @@ const UpdatedDatasetDetailPage: React.FC = () => { + setDownloadMenuEl(null)} + > + handleDownloadScript("sh")}> + For Mac / Linux (.sh) + + handleDownloadScript("bat")}> + For Windows (.bat) + + handleDownloadScript("txt")}> + URL list (.txt, advanced) + + @@ -1324,7 +1420,12 @@ const UpdatedDatasetDetailPage: React.FC = () => { }, }} onClick={() => - handlePreview(link.data, link.index, true) + handlePreview( + link.data, + link.index, + true, + link.path + ) } > Preview @@ -1463,7 +1564,7 @@ const UpdatedDatasetDetailPage: React.FC = () => { }} title={link.name} > - {link.name} + {index + 1}. {link.name}