From 2974bd3c5e63369fdc54a2eef0854768c438401b Mon Sep 17 00:00:00 2001 From: 7bit <662932+sevenbitbyte@users.noreply.github.com> Date: Wed, 25 Mar 2026 12:55:38 -0700 Subject: [PATCH 01/49] Update match-maker-client.js --- src/party/peer/match-maker-client.js | 25 ++++++++++++++++++------- 1 file changed, 18 insertions(+), 7 deletions(-) diff --git a/src/party/peer/match-maker-client.js b/src/party/peer/match-maker-client.js index 3ebd7fa..c5c3791 100644 --- a/src/party/peer/match-maker-client.js +++ b/src/party/peer/match-maker-client.js @@ -12,7 +12,7 @@ const WebsocketComms = require('../../comms/websocket-comms') const PeerInvite = require('./peer-invite') class MatchMakerClient extends EventEmitter { - constructor(identity, contacts, urlOrParty = 'https://postquantum.one/api/', wsUrlOrParty = 'wss://postquantum.one/ws'){ + constructor(identity, contacts, urlOrParty = 'https://postquantum.one/api/', wsUrlOrParty = 'wss://postquantum.one/ws', billingIdentity=null){ super() @@ -22,6 +22,7 @@ class MatchMakerClient extends EventEmitter { this.identity = identity this.wsParty = null this.restParty = null + this.billingIdentity = null if(typeof urlOrParty == 'string'){ this.restUrl = urlOrParty @@ -157,15 +158,23 @@ class MatchMakerClient extends EventEmitter { debug('calling onInviteMsg') await pending.onInviteMsg(msg.invite) - + } } + async announceBillingKey({stripeCheckoutSession}={}){ + this.announcePublicKeys(true, { + stripe: stripeCheckoutSession + }) + } + async announcePublicKeys(useBillingKeyAsActor=false, billingMethodDetails=null){ - async announcePublicKeys(){ + let currentActor = useBillingKeyAsActor == true ? this.billingIdentity : this.identity + const announceData = { annoucement: { + type: useBillingKeyAsActor ? 'billing_identity' : 'user_identity' created: Date.now(), expiry: Date.now() + 24*60*60*1000, //! Set session expiry to 24hr from now sessionKey: { @@ -174,9 +183,9 @@ class MatchMakerClient extends EventEmitter { public: this.sessionKey.key.public }, actorKey: { - type: this.identity.key.type, - hash: this.identity.key.hash, - public: this.identity.key.public + type: currentActor.key.type, + hash: currentActor.key.hash, + public: currentActor.key.public } }, trust: { @@ -185,8 +194,10 @@ class MatchMakerClient extends EventEmitter { } } + if( + - const actorSigMsg = await this.identity.sign(announceData.annoucement, true) + const actorSigMsg = await currentActor.sign(announceData.annoucement, true) const sessionSigMsg = await this.sessionKey.sign(announceData.annoucement, true) debug('actorSigMsg', actorSigMsg) From 34d7484c894fa3cf34ee79e77a441ceb90088b78 Mon Sep 17 00:00:00 2001 From: sevenbitbyte Date: Mon, 18 May 2026 20:14:04 +0000 Subject: [PATCH 02/49] add runner to EndpointContext --- src/service/endpoint-context.js | 7 ++++++- src/service/service-runner-node.js | 1 + 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/src/service/endpoint-context.js b/src/service/endpoint-context.js index 80c725a..272a550 100644 --- a/src/service/endpoint-context.js +++ b/src/service/endpoint-context.js @@ -15,7 +15,7 @@ class EndpointContext { * @param {Debug} options.debug Debug constructor (defaults to npm:Debug) * @param {boolean} options.sendFullErrors Enables sending full stack traces to client (defaults to false) */ - constructor({party, endpoint, req, res, input, debug=Debug, sendFullErrors=false}){ + constructor({party, endpoint, runner, req, res, input, debug=Debug, sendFullErrors=false}){ /** * @member module:Service.EndpointContext.debug @@ -27,6 +27,11 @@ class EndpointContext { */ this.endpoint = endpoint + /** + * @member module:Service.EndpointContext.runner + */ + this.runner = runner + /** * @member module:Service.EndpointContext.MiddlewareConfig */ diff --git a/src/service/service-runner-node.js b/src/service/service-runner-node.js index 5df899c..b251c2e 100644 --- a/src/service/service-runner-node.js +++ b/src/service/service-runner-node.js @@ -516,6 +516,7 @@ class ServiceRunnerNode { req: event.request, res: event.response, endpoint, party: this.party, + runner: this, input: event.request.body, debug: Debug, sendFullErrors: this.sendFullErrors From 3b407d6981ef1049c64070cd03683bc360986ffd Mon Sep 17 00:00:00 2001 From: sevenbitbyte Date: Mon, 18 May 2026 20:54:54 +0000 Subject: [PATCH 03/49] key management and ephemeral sessions --- package.json | 1 + src/party/peer/match-maker-client.js | 8 +- src/venue/endpoints/key-announce.js | 204 +++++++++++++++++++ src/venue/middleware/pre/ephmeral-session.js | 101 +++++++++ src/venue/schema/public-key.js | 47 +++++ src/venue/schema/session-key.js | 69 +++++++ src/venue/venue-service.js | 5 + 7 files changed, 431 insertions(+), 4 deletions(-) create mode 100644 src/venue/endpoints/key-announce.js create mode 100644 src/venue/middleware/pre/ephmeral-session.js create mode 100644 src/venue/schema/public-key.js create mode 100644 src/venue/schema/session-key.js diff --git a/package.json b/package.json index 5caa37b..4687a73 100644 --- a/package.json +++ b/package.json @@ -120,6 +120,7 @@ "devDependencies": { "@dataparty/bouncer-model": "1.4.3", "@hapi/code": "^9.0.1", + "@hapi/joi": "^17.1.1", "@hapi/lab": "^25.0.1", "argon2": "^0.30.3", "argon2-browser": "^1.18.0", diff --git a/src/party/peer/match-maker-client.js b/src/party/peer/match-maker-client.js index c5c3791..65969a5 100644 --- a/src/party/peer/match-maker-client.js +++ b/src/party/peer/match-maker-client.js @@ -174,7 +174,7 @@ class MatchMakerClient extends EventEmitter { const announceData = { annoucement: { - type: useBillingKeyAsActor ? 'billing_identity' : 'user_identity' + //type: 'guest',//useBillingKeyAsActor ? 'billing_identity' : 'user_identity', created: Date.now(), expiry: Date.now() + 24*60*60*1000, //! Set session expiry to 24hr from now sessionKey: { @@ -194,8 +194,6 @@ class MatchMakerClient extends EventEmitter { } } - if( - const actorSigMsg = await currentActor.sign(announceData.annoucement, true) const sessionSigMsg = await this.sessionKey.sign(announceData.annoucement, true) @@ -208,7 +206,9 @@ class MatchMakerClient extends EventEmitter { debug('announcePublicKeys', announceData) - const announceResult = await this.restParty.comms.call('key/announce', announceData, { + let callPath = useBillingKeyAsActor ? 'billing/key/announce' : 'key/announce' + + const announceResult = await this.restParty.comms.call(callPath, announceData, { expectClearTextReply: false, sendClearTextRequest: false, useSessions: false diff --git a/src/venue/endpoints/key-announce.js b/src/venue/endpoints/key-announce.js new file mode 100644 index 0000000..fca2d26 --- /dev/null +++ b/src/venue/endpoints/key-announce.js @@ -0,0 +1,204 @@ +const Joi = require('@hapi/joi') +const Hoek = require('@hapi/hoek') +const debug = require('debug')('dataparty.endpoint.key-announce') + +const {Identity, Message, Routines} = require('@dataparty/crypto') + +//const IEndpoint = require('@dataparty/api/src/service/iendpoint') +const IEndpoint = require('../../service/iendpoint') + +const KeyVerifier = Joi.object().keys({ + id: Joi.string().max(100), + type: Joi.string().max(300).required(), + hash: Joi.string().max(200).required(), + public: { + box: Joi.string().max(60).required(), + sign: Joi.string().max(60).required(), + pqkem: Joi.string().max(8000).required(), + pqsign_ml: Joi.string().max(8000).required(), + pqsign_slh: Joi.string().max(800).required() + } +}) + +module.exports = class KeyAnnounceEndpoint extends IEndpoint { + + static get Name(){ + return 'key/announce' + } + + static get Description(){ + return 'announce a public key' + } + + static get MiddlewareConfig(){ + return { + pre: { + decrypt: true, + validate: Joi.object().keys({ + + annoucement: { + created: Joi.number().required(), + expiry: Joi.number().required(), + actorKey: KeyVerifier.required(), + sessionKey: KeyVerifier.required(), + }, + trust:{ + actorSig: Joi.string().required().description('actor signature of the annoucement in base64'), + sessionSig: Joi.string().required().description('session signature of the annoucement in base64') + } + + }).description('key to announce') + }, + post: { + encrypt: true, + validate: Joi.object().keys({ + done: Joi.boolean(), + }).description('public key') + } + } + } + + static async run(ctx, {Package}){ + + ctx.debug('hello key/announce') + + ctx.debug('ip', ctx.req.ip) + ctx.debug('input', ctx.input) + ctx.debug('sender', ctx.senderKey) + + const inputActorKey = { + type: ctx.input.annoucement.actorKey.type, + hash: ctx.input.annoucement.actorKey.hash, + public: ctx.input.annoucement.actorKey.public + } + + const inputSessionKey = { + type: ctx.input.annoucement.sessionKey.type, + hash: ctx.input.annoucement.sessionKey.hash, + public: ctx.input.annoucement.sessionKey.public + } + + + const computedActorHash = await Routines.hashKey( inputActorKey ) + const computedSessionHash = await Routines.hashKey( inputSessionKey ) + + ctx.debug('computed hash -', computedSessionHash) + + // verify keys are self consistent + if( + computedActorHash != inputActorKey.hash || + computedSessionHash != inputSessionKey.hash + ) { + ctx.debug('invalid actor or session key hash') + return {done: false} + } + + // ensure sender is connected using session key mentioned in annoucement + if(computedSessionHash == inputSessionKey.hash && + inputSessionKey.public.sign == ctx.senderKey.public.sign && + inputSessionKey.public.box == ctx.senderKey.public.box + ){ + + const actorSigBson = Routines.Utils.base64.decode( ctx.input.trust.actorSig ) + const sessionSigBson = Routines.Utils.base64.decode( ctx.input.trust.sessionSig ) + + const actorSigMsg = new Message({ msg: ctx.input.annoucement }) + const sessionSigMsg = new Message({ msg: ctx.input.annoucement }) + + actorSigMsg.sig = actorSigBson + sessionSigMsg.sig = sessionSigBson + + const actorIdentity = Identity.fromJSON({ + id: '', + key: inputActorKey + }) + + const sessionIdentity = Identity.fromJSON({ + id: '', + key: inputSessionKey + }) + + //verify actor & session signature. Require postquantum signing + await actorSigMsg.assertVerified( actorIdentity, true ) + await sessionSigMsg.assertVerified( sessionIdentity, true ) + + let sessionKeyDoc = (await ctx.party.find() + .type('session_key') + .where('annoucement.sessionKey.hash').equals(computedSessionHash) + .exec())[0] + + if(!sessionKeyDoc){ + // create session document + + const now = Date.now() + const tomorrow = now + 24*60*60*1000 + + const fiveMinAgo = now - (5*1000*60) + const fiveMinFromNow = now + (5*1000*60) + + // verify start time is valid + if( + ctx.input.annoucement.created < fiveMinAgo || + ctx.input.annoucement.created > fiveMinFromNow + ) { + ctx.debug('invalid start time') + return {done: false} + } + + // verify expiry time is valid + if( + ctx.input.annoucement.expiry < now || + ctx.input.annoucement.expiry > tomorrow + ) { + ctx.debug('invalid expiry time') + return {done: false} + } + + ctx.debug('opening session -', computedSessionHash) + + let sessionDoc = await ctx.party.createDocument('session_key', { + created: now, + expiry: ctx.input.annoucement.expiry, + annoucement: ctx.input.annoucement, + trust: ctx.input.trust + }) + + ctx.debug('session created - ', computedSessionHash) + + } else { + ctx.debug('session key already known - ', computedSessionHash) + return {done: false} + } + + let publicKey = (await ctx.party.find() + .type('public_key') + .where('annoucement.actorKey.hash').equals(computedActorHash) + .exec())[0] + + if(!publicKey){ + + ctx.debug('annoucing key -', computedActorHash) + + let keyDoc = await ctx.party.createDocument('public_key', { + created: Date.now(), + role: 'guest', + owner: computedActorHash, + ...inputActorKey + }) + + ctx.debug('actor annouced key -', computedActorHash) + } else { + ctx.debug('key already known -', computedActorHash) + } + + return {done: true} + + } else { + + ctx.debug('announce ERROR - BAD KEY HASH') + + return {done: false} + } + + } +} diff --git a/src/venue/middleware/pre/ephmeral-session.js b/src/venue/middleware/pre/ephmeral-session.js new file mode 100644 index 0000000..11f862e --- /dev/null +++ b/src/venue/middleware/pre/ephmeral-session.js @@ -0,0 +1,101 @@ +const Joi = require('joi') +const Hoek = require('@hapi/hoek') +const {Identity} = require('@dataparty/crypto') +const debug = require('debug')('dataparty.middleware.pre.ephemeral-session') + +const IMiddleware = require('../../service/imiddleware') + +module.exports = class Decrypt extends IMiddleware { + + static get Name(){ + return 'ephemeral_session' + } + + static get Type(){ + return 'pre' + } + + static get Description(){ + return 'Decrypt inbound data' + } + + static get ConfigSchema(){ + return Joi.boolean() + } + + static async start(party){ + + } + + static async run(ctx, {Config}){ + + if (!Config){ return } + + if(!ctx.input_session_id){ + throw new Error('no session id') + } + + ctx.debug('looking up session -', ctx.input_session_id) + + let sessionKeyDoc = (await ctx.party.find() + .type('session_key') + .where('annoucement.sessionKey.hash') + .equals(ctx.input_session_id) + .exec() + )[0] + + if(!sessionKeyDoc){ + throw new Error('invalid session') + } + + ctx.debug('located session - ', ctx.input_session_id) + + const sessionKey = sessionKeyDoc.data.annoucement.sessionKey + const actorKey = sessionKeyDoc.data.annoucement.actorKey + + // ensure sender is connected using session key mentioned in db + if(sessionKey.hash == ctx.input_session_id && + sessionKey.public.sign == ctx.senderKey.public.sign && + sessionKey.public.box == ctx.senderKey.public.box + ){ + + const now = Date.now() + + if( sessionKeyDoc.data.expiry < now ){ + throw new Error('session expired!') + } + + const actorIdentity = Identity.fromJSON({ + id: 'actor', + key: actorKey + }) + + const sessionIdentity = Identity.fromJSON({ + id: 'ephemeral-session', + key: sessionKey + }) + + ctx.debug('looking up actor - ', actorKey.hash) + + let actorDoc = (await ctx.party.find() + .type('public_key') + .where('hash') + .equals(actorKey.hash) + .exec() + )[0] + + if(!actorDoc){ + throw new Error('failed to find actor') + } + + ctx.setSession( sessionKeyDoc ) + ctx.setIdentity( actorIdentity ) + ctx.setActor( actorDoc ) + + ctx.debug('session ready') + } else { + throw new Error('invalid sender key for this session') + } + + } +} \ No newline at end of file diff --git a/src/venue/schema/public-key.js b/src/venue/schema/public-key.js new file mode 100644 index 0000000..597601b --- /dev/null +++ b/src/venue/schema/public-key.js @@ -0,0 +1,47 @@ +'use strict' + +const ISchema = require('../../bouncer/ischema') + +class PublicKey extends ISchema { + + static get Type () { return 'public_key' } + + static get Schema(){ + return { + created: { + type: Number, + required: true + }, + + role: String, // [ guest, billing, service, wallet ] + + owner: {required: true, index: true, type: String}, // public key hash + + type: String, + hash: {required: true, index: true, type: String, unique: true}, + public: { + box: String, + sign: String, + pqkem: String, + pqsign_ml: String, + pqsign_slh: String + } + } + } + + static setupSchema(schema){ + //schema.index({ 'hash': 1 }, {unique: true}) + return schema + } + + static permissions (context) { + return { + read: false, + new: false, + change: false + } + } +} + + +module.exports = PublicKey diff --git a/src/venue/schema/session-key.js b/src/venue/schema/session-key.js new file mode 100644 index 0000000..a0da93c --- /dev/null +++ b/src/venue/schema/session-key.js @@ -0,0 +1,69 @@ +'use strict' + +const ISchema = require('../../bouncer/ischema') + + +function PublicKeySchema(unique=true){ + return { + type: {required: true, type: String}, + hash: {required: true, index: true, type: String, unique}, + public: { + box: String, + sign: String, + pqkem: String, + pqsign_ml: String, + pqsign_slh: String + } + } +} + + +class SessionKey extends ISchema { + + static get Type () { return 'session_key' } + + static get Schema(){ + return { + created: { + type: Number, + required: true + }, + expiry: { + type: Number, + index: true, + required: true + }, + annoucement: { + created: { + type: Number, + required: true + }, + expiry: { + type: Number, + required: true + }, + sessionKey: PublicKeySchema(true), + actorKey: PublicKeySchema(false) + }, + trust: { + actorSig: {required: true, type: String}, //! base64 of BSON signature + sessionSig: {required: true, type: String} //! base64 of BSON signature + } + } + } + + static setupSchema(schema){ + return schema + } + + static permissions (context) { + return { + read: false, + new: false, + change: false + } + } +} + + +module.exports = SessionKey \ No newline at end of file diff --git a/src/venue/venue-service.js b/src/venue/venue-service.js index f5c7ad3..47eaad7 100644 --- a/src/venue/venue-service.js +++ b/src/venue/venue-service.js @@ -11,6 +11,9 @@ class VenueService extends DatapartySrv.IService { let builder = new DatapartySrv.ServiceBuilder(this) + + builder.addSchema(Path.join(__dirname, './schema/public-key.js')) + builder.addSchema(Path.join(__dirname, './schema/session-key.js')) builder.addSchema(Path.join(__dirname, './schema/venue_service.js')) builder.addMiddleware(DatapartySrv.middleware_paths.pre.decrypt) @@ -22,6 +25,8 @@ class VenueService extends DatapartySrv.IService { builder.addEndpoint(DatapartySrv.endpoint_paths.identity) builder.addEndpoint(DatapartySrv.endpoint_paths.version) + builder.addEndpoint(Path.join(__dirname, './endpoints/key-announce.js')) + builder.addEndpoint(Path.join(__dirname, './endpoints/create-service.js')) builder.addAuth(Path.join(__dirname, './auth.js')) } From 6df2d7079c14b0cea040b80840acf7bc8eab8033 Mon Sep 17 00:00:00 2001 From: sevenbitbyte Date: Mon, 18 May 2026 21:07:08 +0000 Subject: [PATCH 04/49] cleanup ephemeral sessions every 15 minutes --- src/venue/tasks/cleanup-ephemeral-sessions.js | 86 +++++++++++++++++++ src/venue/venue-service.js | 4 + 2 files changed, 90 insertions(+) create mode 100644 src/venue/tasks/cleanup-ephemeral-sessions.js diff --git a/src/venue/tasks/cleanup-ephemeral-sessions.js b/src/venue/tasks/cleanup-ephemeral-sessions.js new file mode 100644 index 0000000..435a48b --- /dev/null +++ b/src/venue/tasks/cleanup-ephemeral-sessions.js @@ -0,0 +1,86 @@ +const debug = require('debug')('dataparty.task.cleanup-ephemeral-sessions') + +//const ITask = require('@dataparty/api/src/service/itask') +const ITask = require('../../service/itask') + +class CleanupEphemeralSessionsTask extends ITask { + + constructor(options){ + super({ + name: CleanupEphemeralSessionsTask.name, + background: CleanupEphemeralSessionsTask.Config.background, + ...options + }) + + debug('new') + + this.duration = Math.round(1000*60*15) + this.timeout = null + } + + static get Config(){ + return { + background: true, + autostart: true + } + } + + async exec(){ + + this.setTimer() + + return this.detach() + } + + + async lookupSessions(){ + + let now = Date.now() + + return (await this.context.party.find() + .type('session_key') + .where('expiry').lt(now) + .exec()) + } + + setTimer(){ + this.timeout = setTimeout(this.onTimeout.bind(this), this.duration) + } + + async onTimeout(){ + this.timeout = null + + debug('cleanup ephemeral sessions task') + + try{ + let sessions = await this.lookupSessions() + + if(sessions && sessions.length > 0){ + debug('expired sessions ', sessions.length) + let list = sessions.map(i=>{return i.data}) + await this.context.party.remove(...list) + } + } catch (err){ + debug(err) + } + + this.setTimer() + } + + stop(){ + if(this.timeout !== null){ + clearTimeout(this.timeout) + this.timeout = null + } + } + + static get Name(){ + return 'cleanup-ephemeral-sessions' + } + + static get Description(){ + return 'Cleanup Ephemeral sessions' + } +} + +module.exports = CleanupEphemeralSessionsTask \ No newline at end of file diff --git a/src/venue/venue-service.js b/src/venue/venue-service.js index 47eaad7..dc56ea1 100644 --- a/src/venue/venue-service.js +++ b/src/venue/venue-service.js @@ -28,6 +28,10 @@ class VenueService extends DatapartySrv.IService { builder.addEndpoint(Path.join(__dirname, './endpoints/key-announce.js')) builder.addEndpoint(Path.join(__dirname, './endpoints/create-service.js')) + + + builder.addTask(Path.join(__dirname,'./tasks/cleanup-ephemeral-sessions.js')) + builder.addAuth(Path.join(__dirname, './auth.js')) } } From 8ddc422009b0ed69fa1f169887dce0e5e0232ef4 Mon Sep 17 00:00:00 2001 From: sevenbitbyte Date: Mon, 25 May 2026 17:13:08 +0000 Subject: [PATCH 05/49] admin only access to API --- src/comms/peer-comms.js | 7 ++-- src/config/json-file.js | 13 +++++++ src/party/peer/match-maker-client.js | 6 ++- src/service/service-host.js | 2 +- src/venue/auth.js | 32 ++++++++++++++-- src/venue/bin/add-admin.js | 40 ++++++++++++++++++++ src/venue/bin/chill.js | 0 src/venue/endpoints/key-announce.js | 16 +++++++- src/venue/public/p2p-test.html | 7 +++- src/venue/schema/venue_package.js | 56 ++++++++++++++++++++++++++++ src/venue/schema/venue_service.js | 1 - src/venue/venue-host.js | 12 +++--- 12 files changed, 176 insertions(+), 16 deletions(-) create mode 100755 src/venue/bin/add-admin.js create mode 100644 src/venue/bin/chill.js create mode 100644 src/venue/schema/venue_package.js diff --git a/src/comms/peer-comms.js b/src/comms/peer-comms.js index c19a07c..b64c98a 100644 --- a/src/comms/peer-comms.js +++ b/src/comms/peer-comms.js @@ -385,13 +385,13 @@ class PeerComms extends ISocketComms { if(this.party.hostRunner){ const actor = await this.party.hostRunner.auth.lookupIdentity(offer.sender) - const verified = await Routines.verifyDataPQ(actor, signature, offerBSON) + const verified = await Routines.verifyDataPQ(offer.sender, signature, offerBSON) if(!verified){ throw new Error('DENY(hostRunner) - auth op signature is not valid') } - if(this.discoverRemoteIdentity){ this.remoteIdentity = actor } + if(this.discoverRemoteIdentity){ this.remoteIdentity = offer.sender } const authorized = await this.party.hostRunner.auth.isSocketConnectionAllowed(actor) if(!authorized){ @@ -406,6 +406,7 @@ class PeerComms extends ISocketComms { await this.stop() debug('DENY - client not allowed - ', this.remoteIdentity) + throw new Error('DENY - client not allowed') } } else { const actor = offer.sender @@ -420,7 +421,7 @@ class PeerComms extends ISocketComms { } } - debug('clienr auth op offer -', offer) + debug('client auth op offer -', offer) debug('ALLOW - allowing client - ', this.remoteIdentity) this.aesStream = await AESStream.recoverStream( diff --git a/src/config/json-file.js b/src/config/json-file.js index b5280ab..f2f1bec 100644 --- a/src/config/json-file.js +++ b/src/config/json-file.js @@ -22,6 +22,7 @@ class JsonFileConfig extends IConfig { this.path = this.basePath +'/config.json' this.defaults = defaults || {} this.content = Object.assign({}, this.defaults) + this.writing = false } async load(){ @@ -49,6 +50,8 @@ class JsonFileConfig extends IConfig { async start () { await this.touchDir('') await this.load() + + fs.watchFile(this.path, this.handleFileChange.bind(this)) logger('started') } @@ -79,7 +82,9 @@ class JsonFileConfig extends IConfig { } async save(){ + this.writing = true fs.writeFileSync(this.path, JSON.stringify(this.content, null, 2)) + this.writing = false } async touchDir (path) { @@ -98,6 +103,14 @@ class JsonFileConfig extends IConfig { }) }) } + + async handleFileChange(current, previous){ + if(this.writing){ return } + + logger('config changed, reloading') + + await this.load() + } } module.exports = JsonFileConfig \ No newline at end of file diff --git a/src/party/peer/match-maker-client.js b/src/party/peer/match-maker-client.js index 65969a5..cf2f6bd 100644 --- a/src/party/peer/match-maker-client.js +++ b/src/party/peer/match-maker-client.js @@ -83,7 +83,7 @@ class MatchMakerClient extends EventEmitter { await this.announcePublicKeys() } - if(!this.wsParty){ + if(!this.wsParty && this.wsUrl){ this.wsParty = new PeerParty({ comms: new WebsocketComms({ uri: this.wsUrl, @@ -213,6 +213,10 @@ class MatchMakerClient extends EventEmitter { sendClearTextRequest: false, useSessions: false }) + + if(announceResult.done != true){ + throw new Error('annoucement request failed - '+callPath) + } } diff --git a/src/service/service-host.js b/src/service/service-host.js index 6fd1c68..a03a5dd 100644 --- a/src/service/service-host.js +++ b/src/service/service-host.js @@ -307,7 +307,7 @@ class ServiceHost { this.apiApp.use((err, req, res, _next) => { console.log('Error handler', err) if (err instanceof IpDeniedError) { - //res.status(401) + res.status(401) } else { res.status(err.status || 500) } diff --git a/src/venue/auth.js b/src/venue/auth.js index c2ff01c..7b8074c 100644 --- a/src/venue/auth.js +++ b/src/venue/auth.js @@ -1,6 +1,7 @@ const debug = require('debug')('dataparty.auth.venue-auth') const IAuth = require('../service/iauth') +const {Identity} = require('@dataparty/crypto') module.exports = class IAuth { @@ -36,12 +37,28 @@ module.exports = class IAuth { } async lookupIdentity(identity){ + + let sessionKeyDoc = (await this.context.party.find() + .type('session_key') + .where('annoucement.sessionKey.hash') + .equals(identity.key.hash) + .exec() + )[0] + + if(sessionKeyDoc){ + const actorIdentity = Identity.fromJSON({ + id: 'actor', + key: sessionKeyDoc.data.annoucement.actorKey + }) + + return actorIdentity + } + return identity } async isSocketConnectionAllowed(identity){ - //throw new Error('not implemented') - return true + return await this.isAdmin(identity) } async isInternal(identity){ @@ -49,7 +66,16 @@ module.exports = class IAuth { } async isAdmin(identity){ - return false + + // verify key-hash is an admin + const admins = (await this.context.party.config.read('admins')) || [] + + if(admins.indexOf(identity.key.hash) == -1){ + debug('non-admin user', identity.key.hash) + return false + } + + return true } async canReadDb(identity){ diff --git a/src/venue/bin/add-admin.js b/src/venue/bin/add-admin.js new file mode 100755 index 0000000..f09375e --- /dev/null +++ b/src/venue/bin/add-admin.js @@ -0,0 +1,40 @@ +#!/usr/bin/env node + +const Dataparty = require('../../index') + + +async function main(){ + + const path = '/data/dataparty/venue-service' + + let config = new Dataparty.Config.JsonFileConfig({ + basePath: path+'/config' + }) + + await config.start() + + console.log(process.argv) + + let admins = (await config.read('admins')) || [] + + const newAdmin = process.argv[2] + + console.log(await config.readAll()) + console.log(admins) + + if(admins.indexOf(newAdmin) != -1){ return } + + admins.push(newAdmin) + + await config.write('admins', admins) + + console.log('admin added -', newAdmin) + +} + + +main().catch(err=>{ + console.error(err) +}).finally(()=>{ + process.exit() +}) diff --git a/src/venue/bin/chill.js b/src/venue/bin/chill.js new file mode 100644 index 0000000..e69de29 diff --git a/src/venue/endpoints/key-announce.js b/src/venue/endpoints/key-announce.js index fca2d26..de1e7ec 100644 --- a/src/venue/endpoints/key-announce.js +++ b/src/venue/endpoints/key-announce.js @@ -93,6 +93,7 @@ module.exports = class KeyAnnounceEndpoint extends IEndpoint { return {done: false} } + // ensure sender is connected using session key mentioned in annoucement if(computedSessionHash == inputSessionKey.hash && inputSessionKey.public.sign == ctx.senderKey.public.sign && @@ -122,6 +123,19 @@ module.exports = class KeyAnnounceEndpoint extends IEndpoint { await actorSigMsg.assertVerified( actorIdentity, true ) await sessionSigMsg.assertVerified( sessionIdentity, true ) + // verify key-hash is an admin + //const admins = (await ctx.party.config.read('admins')) || [] + + //if(admins.indexOf(computedActorHash) == -1){ + const isAdmin = await ctx.runner.auth.isAdmin(actorIdentity) + if(!isAdmin){ + ctx.debug('non-admin user') + return {done: false} + + process.exit(1) + } + + let sessionKeyDoc = (await ctx.party.find() .type('session_key') .where('annoucement.sessionKey.hash').equals(computedSessionHash) @@ -172,7 +186,7 @@ module.exports = class KeyAnnounceEndpoint extends IEndpoint { let publicKey = (await ctx.party.find() .type('public_key') - .where('annoucement.actorKey.hash').equals(computedActorHash) + .where('hash').equals(computedActorHash) .exec())[0] if(!publicKey){ diff --git a/src/venue/public/p2p-test.html b/src/venue/public/p2p-test.html index 6c57aa8..a45f222 100644 --- a/src/venue/public/p2p-test.html +++ b/src/venue/public/p2p-test.html @@ -256,7 +256,12 @@

Offers recieved

await hostLocal.start() - matchMaker = new Dataparty.MatchMakerClient(hostLocal.privateIdentity, null) + matchMaker = new Dataparty.MatchMakerClient( + hostLocal.privateIdentity, + null, + 'https://api.dataparty.xyz/api', + 'wss://api.dataparty.xyz/ws' + ) await matchMaker.start() diff --git a/src/venue/schema/venue_package.js b/src/venue/schema/venue_package.js new file mode 100644 index 0000000..261f55e --- /dev/null +++ b/src/venue/schema/venue_package.js @@ -0,0 +1,56 @@ +'use strict' + +const debug = require('debug')('venue.venue_pkg') + +const ISchema = require('../../bouncer/ischema') + +const Utils = ISchema.Utils + + +class VenuePkg extends ISchema { + + static get Type () { return 'venue_pkg' } + + static get Schema(){ + return { + owner: {type: String, required: true, index: true}, //public_key.key.hash + created: {type: Number, required: true}, + changed: {type: Number}, + venue: {type: String}, + settings: { + enabled: {type: Boolean, required: true}, + sendFullErrors: {type: Boolean, required: true}, + useNative: {type: Boolean, required: true}, + defaultConfig: {type: Object} + }, + package: { + name: {type: String, required: true, index: true}, + version: {type: String, required: true}, + githash: {type: String, required: true}, + branch: {type: String, required: true} + }, + compressedBuild: {type: String, required: true}, //! zlib compressed + signature: { + timestamp: {type: Number, required: true}, + type: {type: String, required: true, maxlength: 10}, + value: {type: String, required: true} + } + } + } + + static setupSchema(schema){ + //schema.index({ 'package.name': 1 }, {unique: true}) + return schema + } + + static permissions (context) { + return { + read: false, + new: false, + change: false + } + } +} + + +module.exports = VenuePkg \ No newline at end of file diff --git a/src/venue/schema/venue_service.js b/src/venue/schema/venue_service.js index f5f8d08..f66ff9e 100644 --- a/src/venue/schema/venue_service.js +++ b/src/venue/schema/venue_service.js @@ -2,7 +2,6 @@ const debug = require('debug')('venue.venue_srv') - const ISchema = require('../../bouncer/ischema') const Utils = ISchema.Utils diff --git a/src/venue/venue-host.js b/src/venue/venue-host.js index fef4471..b3f8ab2 100644 --- a/src/venue/venue-host.js +++ b/src/venue/venue-host.js @@ -55,7 +55,8 @@ async function main(){ const CloudFlareIpFilter = { options: { - mode: 'allow' + mode: 'allow', + //trustProxy: true }, ips: [ '173.245.48.0/20', @@ -79,7 +80,8 @@ async function main(){ '2405:b500::/32', '2405:8100::/32', '2a06:98c0::/29', - '2c0f:f248::/32' + '2c0f:f248::/32', + '10.115.68.55/32' ] } @@ -114,7 +116,7 @@ async function main(){ trust_proxy: true, wsEnabled: true, ssl_key, ssl_cert, - listenUri: 'https://0.0.0.0:443', + listenUri: 'https://0.0.0.0:3000', staticPath: Path.join(__dirname,'public'), staticPrefix: '/venue/', ipFilter: CloudFlareIpFilter @@ -126,7 +128,7 @@ async function main(){ debug('started') console.log('partying') - +/* await loadService(runnerRouter, { enabled: true, domain: 'postquantum.one', @@ -135,7 +137,7 @@ async function main(){ sendFullErrors: true, useNative: false - }, '/home/ubuntu/match-maker/dataparty/@datapartyjs-match-maker.dataparty-service.json') + }, '/home/ubuntu/match-maker/dataparty/@datapartyjs-match-maker.dataparty-service.json')*/ } From 015a46ba8cf4d22d4ca8e046bc5941d3afd94058 Mon Sep 17 00:00:00 2001 From: sevenbitbyte Date: Mon, 25 May 2026 17:17:31 +0000 Subject: [PATCH 06/49] debounce start --- src/config/json-file.js | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/config/json-file.js b/src/config/json-file.js index f2f1bec..d52eb4f 100644 --- a/src/config/json-file.js +++ b/src/config/json-file.js @@ -23,6 +23,7 @@ class JsonFileConfig extends IConfig { this.defaults = defaults || {} this.content = Object.assign({}, this.defaults) this.writing = false + this.started = false } async load(){ @@ -48,11 +49,16 @@ class JsonFileConfig extends IConfig { } async start () { + + if(this.started){return} + await this.touchDir('') await this.load() fs.watchFile(this.path, this.handleFileChange.bind(this)) logger('started') + + this.started = true } async clear () { From 1bd2b5c4367bdca1190b000d2e574c67b6211d8a Mon Sep 17 00:00:00 2001 From: sevenbitbyte Date: Fri, 29 May 2026 08:32:28 +0000 Subject: [PATCH 07/49] build with tar of static files and add file hashes & signing to service builds --- package.json | 2 + src/service/iservice.js | 12 +- src/service/service-builder.js | 79 +++++++- src/venue/build.js | 114 ++++++++++- src/venue/endpoints/create-package.js | 189 ++++++++++++++++++ .../{create-service.js => file-create.js} | 16 +- src/venue/endpoints/file-download.js | 0 src/venue/endpoints/file-write.js | 0 ...hmeral-session.js => ephemeral-session.js} | 2 +- src/venue/schema/venue_package.js | 1 + src/venue/schema/venue_service.js | 6 - src/venue/venue-host.js | 2 +- src/venue/venue-service.js | 10 +- 13 files changed, 407 insertions(+), 26 deletions(-) create mode 100644 src/venue/endpoints/create-package.js rename src/venue/endpoints/{create-service.js => file-create.js} (89%) create mode 100644 src/venue/endpoints/file-download.js create mode 100644 src/venue/endpoints/file-write.js rename src/venue/middleware/pre/{ephmeral-session.js => ephemeral-session.js} (97%) diff --git a/package.json b/package.json index 4687a73..cf9c05e 100644 --- a/package.json +++ b/package.json @@ -85,6 +85,7 @@ "express-ipfilter": "^1.3.2", "express-list-routes": "^1.1.9", "git-repo-info": "^2.1.1", + "glob": "^13.0.6", "joi": "^17.13.3", "joi-objectid": "^4.0.2", "jshashes": "^1.0.8", @@ -108,6 +109,7 @@ "simple-peer": "9.11.1", "source-map": "^0.7.3", "store-js": "^2.0.4", + "tar": "^7.5.15", "tingodb": "^0.6.1", "touch": "^3.1.0", "url-parse": "^1.4.7", diff --git a/src/service/iservice.js b/src/service/iservice.js index b9f6c01..f8a8b8b 100644 --- a/src/service/iservice.js +++ b/src/service/iservice.js @@ -17,7 +17,7 @@ module.exports = class IService { * @param {*} build */ constructor({ - name, version, githash='', branch='' + name, version, githash='', branch='', owner=null }, build){ this.constructors = { @@ -48,11 +48,13 @@ module.exports = class IService { }, tasks: {}, topics: {}, - auth: null + auth: null, + files: [], + files_root: null } this.compiled = { - package: { name, version, githash, branch }, + package: {owner, name, version, githash, branch }, schemas: { IndexSettings: {}, JSONSchema: [], @@ -70,7 +72,9 @@ module.exports = class IService { }, tasks: {}, topics: {}, - auth: {} + auth: {}, + files: {}, + signatures: [] } this.compileSettings = { diff --git a/src/service/service-builder.js b/src/service/service-builder.js index 614afd9..0f449fd 100644 --- a/src/service/service-builder.js +++ b/src/service/service-builder.js @@ -8,6 +8,15 @@ const BouncerDb = require('@dataparty/bouncer-db') const mongoose = BouncerDb.mongoose() const debug = require('debug')('dataparty.service.ServiceBuilder') +const dataparty_crypto = require('@dataparty/crypto') + +const { + globSync +} = require('glob') +const { isArray } = require('lodash') + +const tar = require('tar') + //const IService = require('../iservice') @@ -27,7 +36,7 @@ module.exports = class ServiceBuilder { * @param {boolean} writeFile When true, files will be written. Defaults to `true` * @returns */ - async compile(outputPath, writeFile=true){ + async compile(outputPath, writeFile=true, owner){ if(!outputPath){ throw new Error('no output path') @@ -48,18 +57,34 @@ module.exports = class ServiceBuilder { this.compileList('tasks'), this.compileList('topics'), this.compileFile('auth'), - this.compileSchemas() + this.compileSchemas(), + this.compressFiles(outputPath, writeFile) ]) + + this.service.compiled.middleware_order = this.service.middleware_order this.service.compiled.compileSettings = this.service.compileSettings + if(owner){ + this.service.compiled.package.owner = owner.key.hash + const ownerSig = await owner.sign(this.service.compiled, true) + + console.log('sign keys', Object.keys(this.service.compiled)) + console.log(this.service.compiled.signature) + + this.service.compiled.signatures = { + [owner.key.hash]: dataparty_crypto.Routines.Utils.base64.encode(ownerSig.sig) + } + } + + if(writeFile){ - const buildOutput = outputPath+'/'+ this.service.compiled.package.name.replace('/', '-') +'.dataparty-service.json' + const buildOutput = outputPath+'/'+ this.service.compiled.package.name.replace('/', '-') +'.service.venue.json' fs.writeFileSync(buildOutput, JSON.stringify(this.service.compiled, null,2)) - const schemaOutput = outputPath+'/'+ this.service.compiled.package.name.replace('/', '-') +'.dataparty-schema.json' + const schemaOutput = outputPath+'/'+ this.service.compiled.package.name.replace('/', '-') +'.schema.venue.json' fs.writeFileSync(schemaOutput, JSON.stringify({ package: this.service.compiled.package, ...this.service.compiled.schemas @@ -315,4 +340,50 @@ module.exports = class ServiceBuilder { this.service.sources.auth = auth_path this.service.constructors.auth = TopicClass } + + addFiles(root, pattern, options){ + + let result = globSync(pattern, { + dotRelative: true, + cwd:root, + ...options + }) + + if(!this.service.files){ + this.service.sources.files = result + } else { + this.service.sources.files = this.service.sources.files.concat(result) + } + + this.service.sources.files_root = root + + debug('addFiles',result) + + } + + async compressFiles(outputPath, writeFile){ + + let fileMap={} + + let files = this.service.sources.files.map(file=>{ + // + const content = fs.readFileSync(file) + const hash = dataparty_crypto.Routines.Utils.base64.encode( + dataparty_crypto.Routines.Utils.hash(content) + ) + + fileMap[file] = { hash, size: content.length } + + return hash + }) + + this.service.compiled.files = fileMap + + await tar.create({ + cwd: this.service.sources.files_root, + gzip: true, + file: Path.join(outputPath, this.service.compiled.package.name.replace('/', '-')+'.files.venue.tgz') + }, this.service.sources.files) + + } } \ No newline at end of file diff --git a/src/venue/build.js b/src/venue/build.js index 756f098..6dcea3a 100644 --- a/src/venue/build.js +++ b/src/venue/build.js @@ -4,9 +4,119 @@ const debug = require('debug')('build') const Dataparty = require('../index.js') +const dataparty_crypto = require('@dataparty/crypto') + const Pkg = require('../../package.json') const VenueService = require('./venue-service') +/** + * build/ + * - service.venue.json + * - schema.venue.json + * - client.venue.json + * - staticFiles.venue.tgz + * - package.venue.json + * { author, venue, files[], signatures{} } + */ + + +/* + +venue: { + url, + publicKey, + +} + +.venue-admin/ + - config/config.json + - db/ + - packages/ + - AUTHOR/PACKAGE-NAME + - package.venue.json + - service.venue.json + - static-files.venue.tgz + - parties/ + - PARTY_PUBLIC/ + - config/config.json (optional) + - db/ (optional) + - static-files.venue.tgz + - projects/ + - AUTHOR/PROJECT_PUBLIC + - project.venue.json + - static-files.venue.tgz + - party/ + - PARTY_NAME/ + - static-files.venue.tgz + + +~/code/my_venue_project/ + - package.json + - build.js + - public/ + - party/NAME + - default-config.json + - public/ + - package/NAME + - venue.json + { + projects: { + NAME: { + name: string + domain, (optional) + venue, (optional) + parties: [NAME], + routes: { + PREFIX: [ { PACKAGE(owner,name,version), party } ] + } + } + }, + packages: { + NAME: { + name: String + version: String, (optional) + service: localPathToService.js OR built service.json + } + } + } + +*/ + +async function buildVenuePackage({authorIdentity, venueIdentity, outputPath, existingBuildPath, staticFilePaths}){ + + /* + + 0. create service.venue.json + 1. create schema.venue.json + 2. create client.venue.json + 3. create staticFiles.venue.tgz + 4. create package.venue.json + + 6. upload to venue + - create-package + * service.venue.json + * package.venue.json + + 7. upload files to venue + - create-files + * staticFiles.venue.tgz + * schema.venue.json + * client.venue.json + + 8. + + */ + +} + + +/** + * + * 1. generate admin/developer key + * 2. add venue identity (by url) + * 3. venue package build - build a package at some path + * 4. venue package push - c + */ async function main(){ const service = new VenueService({ @@ -14,8 +124,10 @@ async function main(){ version: Pkg.version }) + let a = await dataparty_crypto.Identity.fromRandomSeed(); + const builder = new Dataparty.ServiceBuilder(service) - const build = await builder.compile(Path.join(__dirname,'./dataparty'), true) + const build = await builder.compile(Path.join(__dirname,'./dataparty'), true, a) debug('compiled') } diff --git a/src/venue/endpoints/create-package.js b/src/venue/endpoints/create-package.js new file mode 100644 index 0000000..3e63cfa --- /dev/null +++ b/src/venue/endpoints/create-package.js @@ -0,0 +1,189 @@ +const Joi = require('joi') +const Hoek = require('@hapi/hoek') +const {Message, Routines} = require('@dataparty/crypto') +const debug = require('debug')('dataparty.endpoint.create-package') + +const IEndpoint = require('../../service/iendpoint') + +module.exports = class CreatePkgEndpoint extends IEndpoint { + + static get Name(){ + return 'create-package' + } + + + static get Description(){ + return 'Create venue package' + } + + { + venue_package: { + package:{ + owner: String, + info: { + name, version, githash, branch + }, + files: [ //package, service, static.tgz, + {hash: String, name: String, size: Number, signature} + ], + statics: { + PREFIX: [localPathGlob] + } + }, + trust: { + owner: signature + } + + } + } + + { + venue_project: { + + project:{ + + owner: String, + domain: String, + venue: String, + + parties: { + NAME: { + keys: {public, private: Secret(private)}, + defaultConfig: Object, + files: [ //static.tgz, + {hash: String, name: String, signature} + ], + statics: {prefix, [localPath]} + } + }, + + + + + routes: { + PREFIX: [{ + party: String, + package: { + owner: String, + info: {name, version, branch, githash}, + settings: { + sendFullErrors, + useNative + } + } + }] + } + }, + trust: { + owner: signature + } + } + } + + static get MiddlewareConfig(){ + return { + pre: { + decrypt: true, + validate: Joi.object().keys({ + /*settings: Joi.object().keys({ + enabled: Joi.boolean().default(true).required(), + //domain: Joi.string().required(), + staticPrefix: Joi.string().default('/'), + sendFullErrors: Joi.boolean().default(false).required(), + useNative: Joi.boolean().default(false).required() + }).required(),*/ + service: Joi.object().keys({ + package: Joi.object().keys({ + name: Joi.string().required(), + version: Joi.string().required(), + githash: Joi.string().required(), + branch: Joi.string().required() + }).required(), + schemas: Joi.object().keys(null), + documents: Joi.object().keys(null), + endpoints: Joi.object().keys(null), + middleware: Joi.object().keys(null), + middleware_order: Joi.object().keys(null), + tasks: Joi.object().keys(null), + topics: Joi.object().keys(null), + auth: Joi.object().keys(null), + compileSettings: Joi.object().keys(null) + }).required(), + signature: Joi.object().keys({ + timestamp: Joi.number().required(), + type: Joi.string().required(), + value: Joi.string().required() + }).required() + }) + }, + post: { + encrypt: true, + validate: Joi.object().keys(null).description('any output allowed') + } + } + } + + static async run(ctx){ + + //verify sender is admin + + ctx.debug('hello') + debug('echo') + ctx.debug('ctx.input', ctx.input) + + const compiledSrv = JSON.parse(ctx.input.service) + const serviceId = compiledSrv.package.name + '-' + compiledSrv.package.version + debug('addService', serviceId) + + let srvDoc = (await ctx.party.find() + .type('venue_srv') + .where('name').equals(compiledSrv.package.name) + .exec())[0] + + + + if(!srvDoc){ + debug('creating service') + srvDoc = await ctx.party.createDocument('venue_srv', { + name: compiledSrv.package.name, + 'created': (new Date()).toISOString(), + package: compiledSrv.package, + schemas: compiledSrv.schemas, + endpoints: compiledSrv.endpoints, + midddleware: compiledSrv.middleware, + middleware_order: compiledSrv.middleware_order + }) + + debug('service created') + } + else{ + + + try{ + + debug('updating service') + debug(srvDoc.data) + await srvDoc.mergeData({ + package: compiledSrv.package, + schemas: compiledSrv.schemas, + endpoints: compiledSrv.endpoints, + midddleware: compiledSrv.middleware, + middleware_order: compiledSrv.middleware_order + }) + + //debug(srvDoc.data) + + debug('saving doc') + + + await srvDoc.save() + } + catch(err){ + console.log(err) + } + debug('updated service') + } + + return {srv:srvDoc.data} + } +} \ No newline at end of file diff --git a/src/venue/endpoints/create-service.js b/src/venue/endpoints/file-create.js similarity index 89% rename from src/venue/endpoints/create-service.js rename to src/venue/endpoints/file-create.js index ac3e324..d2e20d6 100644 --- a/src/venue/endpoints/create-service.js +++ b/src/venue/endpoints/file-create.js @@ -1,19 +1,19 @@ const Joi = require('joi') const Hoek = require('@hapi/hoek') const {Message, Routines} = require('@dataparty/crypto') -const debug = require('debug')('dataparty.endpoint.create-service') +const debug = require('debug')('dataparty.endpoint.file-create') const IEndpoint = require('../../service/iendpoint') -module.exports = class CreateSrvEndpoint extends IEndpoint { +module.exports = class FileCreateEndpoint extends IEndpoint { static get Name(){ - return 'create-service' + return 'file-create' } static get Description(){ - return 'Create venue service' + return 'Create venue package file' } static get MiddlewareConfig(){ @@ -21,13 +21,13 @@ module.exports = class CreateSrvEndpoint extends IEndpoint { pre: { decrypt: true, validate: Joi.object().keys({ - settings: Joi.object().keys({ + /*settings: Joi.object().keys({ enabled: Joi.boolean().default(true).required(), - domain: Joi.string().required(), - prefix: Joi.string().default('').required(), + //domain: Joi.string().required(), + staticPrefix: Joi.string().default('/'), sendFullErrors: Joi.boolean().default(false).required(), useNative: Joi.boolean().default(false).required() - }).required(), + }).required(),*/ service: Joi.object().keys({ package: Joi.object().keys({ name: Joi.string().required(), diff --git a/src/venue/endpoints/file-download.js b/src/venue/endpoints/file-download.js new file mode 100644 index 0000000..e69de29 diff --git a/src/venue/endpoints/file-write.js b/src/venue/endpoints/file-write.js new file mode 100644 index 0000000..e69de29 diff --git a/src/venue/middleware/pre/ephmeral-session.js b/src/venue/middleware/pre/ephemeral-session.js similarity index 97% rename from src/venue/middleware/pre/ephmeral-session.js rename to src/venue/middleware/pre/ephemeral-session.js index 11f862e..de0fb2a 100644 --- a/src/venue/middleware/pre/ephmeral-session.js +++ b/src/venue/middleware/pre/ephemeral-session.js @@ -3,7 +3,7 @@ const Hoek = require('@hapi/hoek') const {Identity} = require('@dataparty/crypto') const debug = require('debug')('dataparty.middleware.pre.ephemeral-session') -const IMiddleware = require('../../service/imiddleware') +const IMiddleware = require('../../../service/imiddleware') module.exports = class Decrypt extends IMiddleware { diff --git a/src/venue/schema/venue_package.js b/src/venue/schema/venue_package.js index 261f55e..a56929e 100644 --- a/src/venue/schema/venue_package.js +++ b/src/venue/schema/venue_package.js @@ -19,6 +19,7 @@ class VenuePkg extends ISchema { venue: {type: String}, settings: { enabled: {type: Boolean, required: true}, + staticPrefix: String, sendFullErrors: {type: Boolean, required: true}, useNative: {type: Boolean, required: true}, defaultConfig: {type: Object} diff --git a/src/venue/schema/venue_service.js b/src/venue/schema/venue_service.js index f66ff9e..181a556 100644 --- a/src/venue/schema/venue_service.js +++ b/src/venue/schema/venue_service.js @@ -30,12 +30,6 @@ class VenueSrv extends ISchema { version: {type: String, required: true}, githash: {type: String, required: true}, branch: {type: String, required: true} - }, - compressedBuild: {type: String, required: true}, //! zlib compressed - signature: { - timestamp: {type: Number, required: true}, - type: {type: String, required: true, maxlength: 10}, - value: {type: String, required: true} } } } diff --git a/src/venue/venue-host.js b/src/venue/venue-host.js index b3f8ab2..c16f413 100644 --- a/src/venue/venue-host.js +++ b/src/venue/venue-host.js @@ -103,7 +103,7 @@ async function main(){ party, service, sendFullErrors: true, useNative: false, - prefix: 'api/' + prefix: 'venue/' }) let runnerRouter = new Dataparty.RunnerRouter(runner) diff --git a/src/venue/venue-service.js b/src/venue/venue-service.js index dc56ea1..d5eb35e 100644 --- a/src/venue/venue-service.js +++ b/src/venue/venue-service.js @@ -19,6 +19,8 @@ class VenueService extends DatapartySrv.IService { builder.addMiddleware(DatapartySrv.middleware_paths.pre.decrypt) builder.addMiddleware(DatapartySrv.middleware_paths.pre.validate) + builder.addMiddleware(Path.join(__dirname, './middleware/pre/ephemeral-session.js')) + builder.addMiddleware(DatapartySrv.middleware_paths.post.validate) builder.addMiddleware(DatapartySrv.middleware_paths.post.encrypt) @@ -27,12 +29,18 @@ class VenueService extends DatapartySrv.IService { builder.addEndpoint(Path.join(__dirname, './endpoints/key-announce.js')) - builder.addEndpoint(Path.join(__dirname, './endpoints/create-service.js')) + //builder.addEndpoint(Path.join(__dirname, './endpoints/create-service.js')) builder.addTask(Path.join(__dirname,'./tasks/cleanup-ephemeral-sessions.js')) builder.addAuth(Path.join(__dirname, './auth.js')) + + builder.addFiles(__dirname, [ + 'public/*', + 'public/dist/dataparty-browser.*', + 'public/node_modules/argon2-browser/dist/*' + ], { nodir: true, follow: true }) } } From aac16e10b886fadbd288efc14545fe486c6b9ee1 Mon Sep 17 00:00:00 2001 From: sevenbitbyte Date: Sat, 30 May 2026 15:37:53 +0000 Subject: [PATCH 08/49] packages uploading during build but not finished yet --- examples/secure-config-argon2.js | 1 - package.json | 2 +- src/comms/peer-comms.js | 13 ++++++- src/party/peer/match-maker-client.js | 7 ++-- src/service/endpoints/service-version.js | 3 +- src/service/middleware/post/encrypt.js | 6 ++- src/service/middleware/pre/decrypt.js | 10 ++++- src/service/service-host.js | 4 +- src/venue/build.js | 49 +++++++++++++++++++++++- src/venue/endpoints/create-package.js | 43 ++++++++++++++++----- src/venue/endpoints/key-announce.js | 7 +--- src/venue/middleware/pre/decrypt-nacl.js | 2 +- src/venue/venue-host.js | 4 +- src/venue/venue-service.js | 2 +- 14 files changed, 120 insertions(+), 33 deletions(-) diff --git a/examples/secure-config-argon2.js b/examples/secure-config-argon2.js index d4df83f..ce419d4 100644 --- a/examples/secure-config-argon2.js +++ b/examples/secure-config-argon2.js @@ -18,7 +18,6 @@ const HELP_INFO = ` ` async function main(){ - const memoryConfig = new Dataparty.Config.MemoryConfig({foo: 'bar'}) const jsonConfig = new Dataparty.Config.JsonFileConfig({ diff --git a/package.json b/package.json index cf9c05e..7c295b1 100644 --- a/package.json +++ b/package.json @@ -86,7 +86,7 @@ "express-list-routes": "^1.1.9", "git-repo-info": "^2.1.1", "glob": "^13.0.6", - "joi": "^17.13.3", + "joi": "^18.2.1", "joi-objectid": "^4.0.2", "jshashes": "^1.0.8", "jsonpath-plus": "^0.20.1", diff --git a/src/comms/peer-comms.js b/src/comms/peer-comms.js index b64c98a..b20fd16 100644 --- a/src/comms/peer-comms.js +++ b/src/comms/peer-comms.js @@ -458,10 +458,18 @@ class PeerComms extends ISocketComms { return } + debug('input type', typeof op.input.data, Object.keys(op.input.data)) + debug('op.msg type', typeof op.msg, Object.keys(op.msg), Buffer.isBuffer(op.msg)) + + let bodyValue = Buffer.isBuffer(op.msg) ? + op.input.data : + //Routines.BSON.parseObject(new Routines.BSON.BaseParser( op.msg )) : + JSON.parse(op.msg.toString()) + const req = HttpMocks.createRequest({ method: 'GET', url: '/'+op.input.endpoint, - body: (op.input.data) ? JSON.parse(op.msg.toString()) : undefined + body: bodyValue }) const res = HttpMocks.createResponse() @@ -474,6 +482,9 @@ class PeerComms extends ISocketComms { debug('route',route) + req.peer = this + req.source = 'PeerComms' + debug('call route', await route._events.route({ method: req.method, pathname: req.url, diff --git a/src/party/peer/match-maker-client.js b/src/party/peer/match-maker-client.js index cf2f6bd..7c9b9ff 100644 --- a/src/party/peer/match-maker-client.js +++ b/src/party/peer/match-maker-client.js @@ -2,20 +2,19 @@ const EventEmitter = require('eventemitter3') const debug = require('debug')('dataparty.match-maker-client') - const dataparty_crypto = require('@dataparty/crypto') const LokiParty = require('../local/loki-party') const PeerParty = require('./peer-party') const MemoryConfig = require('../../config/memory') +const RestComms = require('../../comms/rest-comms') const WebsocketComms = require('../../comms/websocket-comms') const PeerInvite = require('./peer-invite') class MatchMakerClient extends EventEmitter { - constructor(identity, contacts, urlOrParty = 'https://postquantum.one/api/', wsUrlOrParty = 'wss://postquantum.one/ws', billingIdentity=null){ + constructor(identity, contacts, urlOrParty = 'https://api.dataparty.xyz/api', wsUrlOrParty = 'wss://api.dataparty.xyz/ws', billingIdentity=null){ super() - this.contacts = contacts this.sessionKey = null @@ -72,7 +71,7 @@ class MatchMakerClient extends EventEmitter { await this.restParty.start() if(!this.restParty.comms){ - this.restParty.comms = new Dataparty.Comms.RestComms({ + this.restParty.comms = new RestComms({ party:this.restParty, config: this.restParty.config }) diff --git a/src/service/endpoints/service-version.js b/src/service/endpoints/service-version.js index b84c91e..ab5ed11 100644 --- a/src/service/endpoints/service-version.js +++ b/src/service/endpoints/service-version.js @@ -27,7 +27,8 @@ module.exports = class ServiceVersion extends IEndpoint { name: Joi.string(), branch: Joi.string(), version: Joi.string(), - githash: Joi.string() + githash: Joi.string(), + owner: Joi.string(), }) } } diff --git a/src/service/middleware/post/encrypt.js b/src/service/middleware/post/encrypt.js index a0912d8..8645087 100644 --- a/src/service/middleware/post/encrypt.js +++ b/src/service/middleware/post/encrypt.js @@ -30,7 +30,11 @@ module.exports = class Encrypt extends IMiddleware { static async run(ctx, {Config}){ if (!Config){ return } - + + if(ctx.req.source && ctx.req.source == 'PeerComms'){ + ctx.setOutput(ctx.output) + return + } const senderStr = JSON.stringify({key: ctx.senderKey}) diff --git a/src/service/middleware/pre/decrypt.js b/src/service/middleware/pre/decrypt.js index 20adf0c..048dd52 100644 --- a/src/service/middleware/pre/decrypt.js +++ b/src/service/middleware/pre/decrypt.js @@ -31,8 +31,16 @@ module.exports = class Decrypt extends IMiddleware { if (!Config){ return } + + if(!context.input || !context.input.enc){ - throw new Error('insecure message') + + if(!context.req.source || context.req.source != 'PeerComms'){ + throw new Error('insecure message -' + context.req.source) + } + + context.setInput( context.input ) + return } context.debug('input', context.input, typeof context.input) diff --git a/src/service/service-host.js b/src/service/service-host.js index a03a5dd..10033fa 100644 --- a/src/service/service-host.js +++ b/src/service/service-host.js @@ -113,8 +113,8 @@ class ServiceHost { if(debug.enabled){ this.apiApp.use(morgan('combined')) } this.apiApp.use(bodyParser.urlencoded({ extended: true })) - this.apiApp.use(bodyParser.json()) - this.apiApp.use(bodyParser.raw()) + this.apiApp.use(bodyParser.json({limit:'10MB'})) + this.apiApp.use(bodyParser.raw({limit:'10MB'})) this.apiApp.set('trust proxy', trust_proxy) diff --git a/src/venue/build.js b/src/venue/build.js index 6dcea3a..4ba65fc 100644 --- a/src/venue/build.js +++ b/src/venue/build.js @@ -1,3 +1,4 @@ +const fs = require('fs') const Path = require('path') const debug = require('debug')('build') @@ -118,18 +119,62 @@ async function buildVenuePackage({authorIdentity, venueIdentity, outputPath, exi * 4. venue package push - c */ +async function pushService(devId, build){ + + let client = new Dataparty.MatchMakerClient( + devId, + null, + 'https://api.dataparty.xyz/venue', + 'wss://api.dataparty.xyz/ws' + ) + + await client.start() + + const staticTar = fs.readFileSync('./dataparty/@dataparty-venue.files.venue.tgz') + + console.log('is staticTar a buffer? ', staticTar instanceof Buffer); // true + + let uploadResult = await client.restParty.comms.call('create-package', {build, staticTar: staticTar}, { + expectClearTextReply: false, + sendClearTextRequest: false, + useSessions: false + }) + + console.log('result', uploadResult) +} + async function main(){ const service = new VenueService({ name: '@dataparty/venue', version: Pkg.version }) - let a = await dataparty_crypto.Identity.fromRandomSeed(); + const path = Path.join(process.env.HOME, '.venue-admin') + let config = new Dataparty.Config.JsonFileConfig({ + basePath: path+'/config' + }) + + let party = new Dataparty.TingoParty({ + path: path+'/db', + config, + noCache: false + }) + + await party.start() + + console.log( 'identity - ', party.identity.key.hash ) + const builder = new Dataparty.ServiceBuilder(service) - const build = await builder.compile(Path.join(__dirname,'./dataparty'), true, a) + const build = await builder.compile(Path.join(__dirname,'./dataparty'), true, party.privateIdentity) debug('compiled') + + await pushService( party.privateIdentity, build ) + + + + } main().catch(err=>{ diff --git a/src/venue/endpoints/create-package.js b/src/venue/endpoints/create-package.js index 3e63cfa..517257c 100644 --- a/src/venue/endpoints/create-package.js +++ b/src/venue/endpoints/create-package.js @@ -16,6 +16,7 @@ module.exports = class CreatePkgEndpoint extends IEndpoint { return 'Create venue package' } + /* { venue_package: { package:{ @@ -80,10 +81,14 @@ module.exports = class CreatePkgEndpoint extends IEndpoint { } } + */ + static get MiddlewareConfig(){ return { pre: { decrypt: true, + ephemeral_session: true, + //validate: Joi.object().keys(null) validate: Joi.object().keys({ /*settings: Joi.object().keys({ enabled: Joi.boolean().default(true).required(), @@ -92,8 +97,9 @@ module.exports = class CreatePkgEndpoint extends IEndpoint { sendFullErrors: Joi.boolean().default(false).required(), useNative: Joi.boolean().default(false).required() }).required(),*/ - service: Joi.object().keys({ + build: Joi.object().keys({ package: Joi.object().keys({ + owner: Joi.string(), name: Joi.string().required(), version: Joi.string().required(), githash: Joi.string().required(), @@ -107,13 +113,11 @@ module.exports = class CreatePkgEndpoint extends IEndpoint { tasks: Joi.object().keys(null), topics: Joi.object().keys(null), auth: Joi.object().keys(null), + files: Joi.object().keys(null), + signatures: Joi.object().keys(null).required(), compileSettings: Joi.object().keys(null) }).required(), - signature: Joi.object().keys({ - timestamp: Joi.number().required(), - type: Joi.string().required(), - value: Joi.string().required() - }).required() + staticTar: Joi.binary() }) }, post: { @@ -125,12 +129,31 @@ module.exports = class CreatePkgEndpoint extends IEndpoint { static async run(ctx){ - //verify sender is admin + ctx.debug('hello') debug('echo') ctx.debug('ctx.input', ctx.input) + + //verify sender is admin + const isAdmin = await ctx.runner.auth.isAdmin(actorIdentity) + if(!isAdmin){ + ctx.debug('non-admin user') + return {done: false} + } + + // verify build signature + + + // untar listed files + + // verify static file checksums match verified signatures + + // create db entry + + /* + const compiledSrv = JSON.parse(ctx.input.service) const serviceId = compiledSrv.package.name + '-' + compiledSrv.package.version debug('addService', serviceId) @@ -182,8 +205,10 @@ module.exports = class CreatePkgEndpoint extends IEndpoint { console.log(err) } debug('updated service') - } + }*/ + + return {done: true} - return {srv:srvDoc.data} + //return {srv:srvDoc.data} } } \ No newline at end of file diff --git a/src/venue/endpoints/key-announce.js b/src/venue/endpoints/key-announce.js index de1e7ec..6b281b6 100644 --- a/src/venue/endpoints/key-announce.js +++ b/src/venue/endpoints/key-announce.js @@ -123,16 +123,11 @@ module.exports = class KeyAnnounceEndpoint extends IEndpoint { await actorSigMsg.assertVerified( actorIdentity, true ) await sessionSigMsg.assertVerified( sessionIdentity, true ) - // verify key-hash is an admin - //const admins = (await ctx.party.config.read('admins')) || [] - - //if(admins.indexOf(computedActorHash) == -1){ + // verify actor is an admin const isAdmin = await ctx.runner.auth.isAdmin(actorIdentity) if(!isAdmin){ ctx.debug('non-admin user') return {done: false} - - process.exit(1) } diff --git a/src/venue/middleware/pre/decrypt-nacl.js b/src/venue/middleware/pre/decrypt-nacl.js index 069812a..857f2e9 100644 --- a/src/venue/middleware/pre/decrypt-nacl.js +++ b/src/venue/middleware/pre/decrypt-nacl.js @@ -32,7 +32,7 @@ module.exports = class DecryptNaCl extends IMiddleware { if (!Config){ return } if(!context.input || !context.input.enc){ - throw new Error('insecure message') + throw new Error('insecure message - here') } context.debug('input', context.input, typeof context.input) diff --git a/src/venue/venue-host.js b/src/venue/venue-host.js index c16f413..7118e18 100644 --- a/src/venue/venue-host.js +++ b/src/venue/venue-host.js @@ -5,8 +5,8 @@ const Dataparty = require('../index') const VenueService = require('./venue-service') -const VenueServiceSchema = require('./dataparty/@dataparty-venue.dataparty-schema.json') -const VenueSrv = require('./dataparty/@dataparty-venue.dataparty-service.json') +const VenueServiceSchema = require('./dataparty/@dataparty-venue.schema.venue.json') +const VenueSrv = require('./dataparty/@dataparty-venue.service.venue.json') async function loadService(runnerRouter, settings, serviceFilePath){ diff --git a/src/venue/venue-service.js b/src/venue/venue-service.js index d5eb35e..d100d12 100644 --- a/src/venue/venue-service.js +++ b/src/venue/venue-service.js @@ -29,7 +29,7 @@ class VenueService extends DatapartySrv.IService { builder.addEndpoint(Path.join(__dirname, './endpoints/key-announce.js')) - //builder.addEndpoint(Path.join(__dirname, './endpoints/create-service.js')) + builder.addEndpoint(Path.join(__dirname, './endpoints/create-package.js')) builder.addTask(Path.join(__dirname,'./tasks/cleanup-ephemeral-sessions.js')) From c022c128e0e64349b2c16361c27b92f2c71a3988 Mon Sep 17 00:00:00 2001 From: sevenbitbyte Date: Sat, 30 May 2026 21:09:28 +0000 Subject: [PATCH 09/49] hash tar file during build --- src/service/service-builder.js | 18 ++++++++++++++++-- src/venue/build.js | 10 +++++----- 2 files changed, 21 insertions(+), 7 deletions(-) diff --git a/src/service/service-builder.js b/src/service/service-builder.js index 0f449fd..0137f93 100644 --- a/src/service/service-builder.js +++ b/src/service/service-builder.js @@ -377,13 +377,27 @@ module.exports = class ServiceBuilder { return hash }) - this.service.compiled.files = fileMap + const tarFileName = this.service.compiled.package.name.replace('/', '-')+'.files.venue.tgz' + const tarPath = Path.join(outputPath, tarFileName) await tar.create({ cwd: this.service.sources.files_root, gzip: true, - file: Path.join(outputPath, this.service.compiled.package.name.replace('/', '-')+'.files.venue.tgz') + file: tarPath }, this.service.sources.files) + const staticTar = fs.readFileSync(tarPath) + + let tarHash = dataparty_crypto.Routines.Utils.hash( staticTar ) + let tarHash64 = dataparty_crypto.Routines.Utils.base64.encode(tarHash) + + this.service.compiled.files = { + [tarFileName]: { + tar: tarFileName, + hash:tarHash64, + size: staticTar.length, + files: fileMap + } + } } } \ No newline at end of file diff --git a/src/venue/build.js b/src/venue/build.js index 4ba65fc..7f303b8 100644 --- a/src/venue/build.js +++ b/src/venue/build.js @@ -119,7 +119,7 @@ async function buildVenuePackage({authorIdentity, venueIdentity, outputPath, exi * 4. venue package push - c */ -async function pushService(devId, build){ +async function pushService(devId, build, staticTar){ let client = new Dataparty.MatchMakerClient( devId, @@ -130,9 +130,6 @@ async function pushService(devId, build){ await client.start() - const staticTar = fs.readFileSync('./dataparty/@dataparty-venue.files.venue.tgz') - - console.log('is staticTar a buffer? ', staticTar instanceof Buffer); // true let uploadResult = await client.restParty.comms.call('create-package', {build, staticTar: staticTar}, { expectClearTextReply: false, @@ -170,7 +167,10 @@ async function main(){ debug('compiled') - await pushService( party.privateIdentity, build ) + const staticTar = fs.readFileSync('./dataparty/@dataparty-venue.files.venue.tgz') + + debug('is staticTar a buffer? ', staticTar instanceof Buffer); // true + //await pushService( party.privateIdentity, build, staticTar ) From 09ab900a4d2187848c80cf5b35f80bc453930a70 Mon Sep 17 00:00:00 2001 From: sevenbitbyte Date: Sun, 31 May 2026 15:19:41 +0000 Subject: [PATCH 10/49] verifying service & tars, saving files to disc --- src/config/json-file.js | 10 ++ src/service/iservice.js | 2 +- src/venue/build.js | 6 +- src/venue/endpoints/create-package.js | 126 ++++++++++++++++++++++---- src/venue/venue-host.js | 2 +- 5 files changed, 125 insertions(+), 21 deletions(-) diff --git a/src/config/json-file.js b/src/config/json-file.js index d52eb4f..a8376c8 100644 --- a/src/config/json-file.js +++ b/src/config/json-file.js @@ -110,6 +110,16 @@ class JsonFileConfig extends IConfig { }) } + fileExists(path){ + var realPath = Path.join(this.basePath, Path.dirname(path), sanitize(Path.basename(path))) + + return fs.existsSync(realPath) + } + + filePath(path){ + return Path.join(this.basePath, Path.dirname(path), sanitize(Path.basename(path))) + } + async handleFileChange(current, previous){ if(this.writing){ return } diff --git a/src/service/iservice.js b/src/service/iservice.js index f8a8b8b..2d46caf 100644 --- a/src/service/iservice.js +++ b/src/service/iservice.js @@ -74,7 +74,7 @@ module.exports = class IService { topics: {}, auth: {}, files: {}, - signatures: [] + //signatures: {} } this.compileSettings = { diff --git a/src/venue/build.js b/src/venue/build.js index 7f303b8..e126d57 100644 --- a/src/venue/build.js +++ b/src/venue/build.js @@ -131,10 +131,10 @@ async function pushService(devId, build, staticTar){ await client.start() - let uploadResult = await client.restParty.comms.call('create-package', {build, staticTar: staticTar}, { + let uploadResult = await client.restParty.comms.call('create-package', {build, staticTar}, { expectClearTextReply: false, sendClearTextRequest: false, - useSessions: false + useSessions: true }) console.log('result', uploadResult) @@ -170,7 +170,7 @@ async function main(){ const staticTar = fs.readFileSync('./dataparty/@dataparty-venue.files.venue.tgz') debug('is staticTar a buffer? ', staticTar instanceof Buffer); // true - //await pushService( party.privateIdentity, build, staticTar ) + await pushService( party.privateIdentity, build, staticTar ) diff --git a/src/venue/endpoints/create-package.js b/src/venue/endpoints/create-package.js index 517257c..9a4022d 100644 --- a/src/venue/endpoints/create-package.js +++ b/src/venue/endpoints/create-package.js @@ -1,10 +1,21 @@ +const fs = require('fs') const Joi = require('joi') const Hoek = require('@hapi/hoek') -const {Message, Routines} = require('@dataparty/crypto') +const {Message, Routines, Identity} = require('@dataparty/crypto') const debug = require('debug')('dataparty.endpoint.create-package') + const IEndpoint = require('../../service/iendpoint') +const typedArraySchema = (value, helpers) => { + // 1. Ensure the value is an instance of a TypedArray (e.g., Uint8Array) + if (!(value instanceof Uint8Array)) { + return helpers.message({ custom: '"value" must be a Uint8Array' }); + } + + return value +} + module.exports = class CreatePkgEndpoint extends IEndpoint { static get Name(){ @@ -88,15 +99,14 @@ module.exports = class CreatePkgEndpoint extends IEndpoint { pre: { decrypt: true, ephemeral_session: true, - //validate: Joi.object().keys(null) validate: Joi.object().keys({ - /*settings: Joi.object().keys({ - enabled: Joi.boolean().default(true).required(), + settings: Joi.object().keys({ + //enabled: Joi.boolean().default(true).required(), //domain: Joi.string().required(), staticPrefix: Joi.string().default('/'), - sendFullErrors: Joi.boolean().default(false).required(), - useNative: Joi.boolean().default(false).required() - }).required(),*/ + sendFullErrors: Joi.boolean().default(false), + useNative: Joi.boolean().default(false) + }), build: Joi.object().keys({ package: Joi.object().keys({ owner: Joi.string(), @@ -117,7 +127,8 @@ module.exports = class CreatePkgEndpoint extends IEndpoint { signatures: Joi.object().keys(null).required(), compileSettings: Joi.object().keys(null) }).required(), - staticTar: Joi.binary() + //staticTar: Joi.binary() + staticTar: Joi.any().custom(typedArraySchema) }) }, post: { @@ -129,22 +140,105 @@ module.exports = class CreatePkgEndpoint extends IEndpoint { static async run(ctx){ + let {signatures, ...buildWithoutSig} = ctx.input.build + + const pkgOwnerIdentityDoc = (await ctx.party.find() + .type('public_key') + .where('hash').equals( ctx.input.build.package.owner ) + .exec() + )[0] + + if(!pkgOwnerIdentityDoc){ + throw new Error('package owner not authorized') + } + + debug('found pkg owner', pkgOwnerIdentityDoc.hash) + + const pkgOwnerIdentity = Identity.fromJSON({ + id: '', + key: { + type: pkgOwnerIdentityDoc.data.type, + hash: pkgOwnerIdentityDoc.data.hash, + public: pkgOwnerIdentityDoc.data.public + } + }) + + + debug('inflated identity') + + //debug('build', buildWithoutSig) + const devSig = Routines.Utils.base64.decode(signatures[ctx.input.build.package.owner]) + + let signedBuildMsg = new Message({ + msg: buildWithoutSig, + sig: devSig + }) + + debug('sigs', signatures) + + debug('verifying package signature') - ctx.debug('hello') - debug('echo') - ctx.debug('ctx.input', ctx.input) + await signedBuildMsg.assertVerified(pkgOwnerIdentity, true) + debug('verified package signature') - //verify sender is admin - const isAdmin = await ctx.runner.auth.isAdmin(actorIdentity) - if(!isAdmin){ - ctx.debug('non-admin user') - return {done: false} + const tarHash = Routines.Utils.hash(ctx.input.staticTar) + const tarHash64 = Routines.Utils.base64.encode( tarHash ) + + const safeFileName = ctx.input.build.package.name.replace('/', '-') + const tarFileName = safeFileName+'.files.venue.tgz' + + const buildFiles = ctx.input.build.files[tarFileName] + + if(buildFiles && buildFiles.hash != tarHash64){ + throw new Error("staticTar hash doesn't match package definition") } + debug('verified staticTar') + + debug('verified package - '+ctx.input.build.package.name+'@'+ctx.input.build.package.version) + + const buildBSON = Routines.BSON.serializeBSONWithoutOptimiser(buildWithoutSig) + + const buildHash = Routines.Utils.base64.encode( + Routines.Utils.hash( + buildBSON + ) + ) + + const safeBuildHash = buildHash.replace('/', '-') + + debug('\t'+'hash', buildHash) + + const buildWorkspace = 'packages/'+safeFileName+'/'+ctx.input.build.package.version+'/'+safeBuildHash + + const config = ctx.party.config + + const workspacePath = await config.touchDir(buildWorkspace) + + + debug('\t'+'workspace - local', buildWorkspace) + debug('\t'+'workspace - global', workspacePath) + + fs.writeFileSync( + Path.join(workspacePath, tarFileName), + ctx.input.staticTar + ) + + /*fs.writeFileSync( + Path.join(workspacePath, safeFileName+'.service.venue.bson'), + Routines.BSON.serializeBSONWithoutOptimiser(ctx.input.build) + )*/ + + fs.writeFileSync( + Path.join(workspacePath, safeFileName+'.service.venue.json'), + JSON.stringify(ctx.input.build, null, 2) + ) + // verify build signature + //ctx.input. // untar listed files diff --git a/src/venue/venue-host.js b/src/venue/venue-host.js index 7118e18..6e72731 100644 --- a/src/venue/venue-host.js +++ b/src/venue/venue-host.js @@ -88,7 +88,7 @@ async function main(){ let party = new Dataparty.TingoParty({ path: path+'/db', model: VenueServiceSchema, - config: new Dataparty.Config.JsonFileConfig({basePath: path+'/config'}), + config: new Dataparty.Config.JsonFileConfig({basePath: path}), noCache: false }) From 1c794adcaf38951a5902accf812b59462d178c73 Mon Sep 17 00:00:00 2001 From: sevenbitbyte Date: Sun, 31 May 2026 17:18:26 +0000 Subject: [PATCH 11/49] services written to db --- src/service/service-builder.js | 11 +++++++ src/venue/endpoints/create-package.js | 42 +++++++++++++++++++++++++-- src/venue/schema/venue_package.js | 15 ++++------ src/venue/venue-service.js | 2 +- 4 files changed, 57 insertions(+), 13 deletions(-) diff --git a/src/service/service-builder.js b/src/service/service-builder.js index 0137f93..08b8f94 100644 --- a/src/service/service-builder.js +++ b/src/service/service-builder.js @@ -7,6 +7,7 @@ const gitRepoInfo = require('git-repo-info') const BouncerDb = require('@dataparty/bouncer-db') const mongoose = BouncerDb.mongoose() const debug = require('debug')('dataparty.service.ServiceBuilder') +const zlib = require('zlib') const dataparty_crypto = require('@dataparty/crypto') @@ -89,6 +90,16 @@ module.exports = class ServiceBuilder { package: this.service.compiled.package, ...this.service.compiled.schemas }, null, 2)) + + // Gzip compression (most common for HTTP) + const compressed = zlib.gzipSync(JSON.stringify(this.service.compiled, null,2)); + + // Brotli compression (better ratio, Node.js 10.5.0+) + const compressedBrotli = zlib.brotliCompressSync(JSON.stringify(this.service.compiled, null,2)); + + console.log('Original:', JSON.stringify(this.service.compiled, null,2).length, 'bytes'); + console.log('Gzip:', compressed.length, 'bytes'); + console.log('Brotli:', compressedBrotli.length, 'bytes'); } return this.service.compiled diff --git a/src/venue/endpoints/create-package.js b/src/venue/endpoints/create-package.js index 9a4022d..fbe7a28 100644 --- a/src/venue/endpoints/create-package.js +++ b/src/venue/endpoints/create-package.js @@ -3,7 +3,7 @@ const Joi = require('joi') const Hoek = require('@hapi/hoek') const {Message, Routines, Identity} = require('@dataparty/crypto') const debug = require('debug')('dataparty.endpoint.create-package') - +const zlib = require('zlib') const IEndpoint = require('../../service/iendpoint') @@ -105,7 +105,8 @@ module.exports = class CreatePkgEndpoint extends IEndpoint { //domain: Joi.string().required(), staticPrefix: Joi.string().default('/'), sendFullErrors: Joi.boolean().default(false), - useNative: Joi.boolean().default(false) + useNative: Joi.boolean().default(false), + defaultConfig: Joi.object().keys(null) }), build: Joi.object().keys({ package: Joi.object().keys({ @@ -199,7 +200,7 @@ module.exports = class CreatePkgEndpoint extends IEndpoint { debug('verified package - '+ctx.input.build.package.name+'@'+ctx.input.build.package.version) - const buildBSON = Routines.BSON.serializeBSONWithoutOptimiser(buildWithoutSig) + const buildBSON = Routines.BSON.serializeBSONWithoutOptimiser(/*ctx.input.build*/buildWithoutSig) const buildHash = Routines.Utils.base64.encode( Routines.Utils.hash( @@ -235,6 +236,41 @@ module.exports = class CreatePkgEndpoint extends IEndpoint { Path.join(workspacePath, safeFileName+'.service.venue.json'), JSON.stringify(ctx.input.build, null, 2) ) + + const compressedBrotliBuild = zlib.brotliCompressSync(JSON.stringify(ctx.input.build)) + + const build = ctx.input.build + const serviceId = build.package.name + '@' + build.package.version + debug('addService', serviceId) + + let srvDoc = (await ctx.party.find() + .type('venue_pkg') + .where('package.name').equals(build.package.name) + .where('package.version').equals(build.package.version) + .where('hash').equals(buildHash) + .exec())[0] + + + if(!srvDoc){ + debug('creating service') + + const {owner, ...pkgWithoutOwner} = build.package + + srvDoc = await ctx.party.createDocument('venue_pkg', { + owner: build.package.owner, + 'created': Date.now(), + venue: ctx.party.identity.key.hash, + hash: buildHash, + workspace: workspacePath, + settings: ctx.input.settings, + package: pkgWithoutOwner, + compressedBuild: Routines.Utils.base64.encode(compressedBrotliBuild) + }) + + debug('service created') + } else { + debug('need to update service') + } // verify build signature diff --git a/src/venue/schema/venue_package.js b/src/venue/schema/venue_package.js index a56929e..a205839 100644 --- a/src/venue/schema/venue_package.js +++ b/src/venue/schema/venue_package.js @@ -17,8 +17,10 @@ class VenuePkg extends ISchema { created: {type: Number, required: true}, changed: {type: Number}, venue: {type: String}, + workspace: {type: String, required: true}, + hash: {type: String, required: true, index: true}, settings: { - enabled: {type: Boolean, required: true}, + //enabled: {type: Boolean, required: true}, staticPrefix: String, sendFullErrors: {type: Boolean, required: true}, useNative: {type: Boolean, required: true}, @@ -26,16 +28,11 @@ class VenuePkg extends ISchema { }, package: { name: {type: String, required: true, index: true}, - version: {type: String, required: true}, + version: {type: String, required: true, index: true}, githash: {type: String, required: true}, - branch: {type: String, required: true} + branch: {type: String, required: true}, }, - compressedBuild: {type: String, required: true}, //! zlib compressed - signature: { - timestamp: {type: Number, required: true}, - type: {type: String, required: true, maxlength: 10}, - value: {type: String, required: true} - } + compressedBuild: {type: String, required: true} //! brotli compressed } } diff --git a/src/venue/venue-service.js b/src/venue/venue-service.js index d100d12..c0f0a30 100644 --- a/src/venue/venue-service.js +++ b/src/venue/venue-service.js @@ -14,7 +14,7 @@ class VenueService extends DatapartySrv.IService { builder.addSchema(Path.join(__dirname, './schema/public-key.js')) builder.addSchema(Path.join(__dirname, './schema/session-key.js')) - builder.addSchema(Path.join(__dirname, './schema/venue_service.js')) + builder.addSchema(Path.join(__dirname, './schema/venue_package.js')) builder.addMiddleware(DatapartySrv.middleware_paths.pre.decrypt) builder.addMiddleware(DatapartySrv.middleware_paths.pre.validate) From 2b68a1d77d22f11483b3712de8e40717377af5df Mon Sep 17 00:00:00 2001 From: sevenbitbyte Date: Sun, 31 May 2026 17:32:01 +0000 Subject: [PATCH 12/49] regular expression for safeworkspace path and move file save to after db is updated --- src/venue/endpoints/create-package.js | 30 ++++++++++++++------------- 1 file changed, 16 insertions(+), 14 deletions(-) diff --git a/src/venue/endpoints/create-package.js b/src/venue/endpoints/create-package.js index fbe7a28..30ff2bd 100644 --- a/src/venue/endpoints/create-package.js +++ b/src/venue/endpoints/create-package.js @@ -208,7 +208,7 @@ module.exports = class CreatePkgEndpoint extends IEndpoint { ) ) - const safeBuildHash = buildHash.replace('/', '-') + const safeBuildHash = buildHash.replace(/\//g, "-") debug('\t'+'hash', buildHash) @@ -222,20 +222,7 @@ module.exports = class CreatePkgEndpoint extends IEndpoint { debug('\t'+'workspace - local', buildWorkspace) debug('\t'+'workspace - global', workspacePath) - fs.writeFileSync( - Path.join(workspacePath, tarFileName), - ctx.input.staticTar - ) - - /*fs.writeFileSync( - Path.join(workspacePath, safeFileName+'.service.venue.bson'), - Routines.BSON.serializeBSONWithoutOptimiser(ctx.input.build) - )*/ - fs.writeFileSync( - Path.join(workspacePath, safeFileName+'.service.venue.json'), - JSON.stringify(ctx.input.build, null, 2) - ) const compressedBrotliBuild = zlib.brotliCompressSync(JSON.stringify(ctx.input.build)) @@ -271,6 +258,21 @@ module.exports = class CreatePkgEndpoint extends IEndpoint { } else { debug('need to update service') } + + fs.writeFileSync( + Path.join(workspacePath, tarFileName), + ctx.input.staticTar + ) + + /*fs.writeFileSync( + Path.join(workspacePath, safeFileName+'.service.venue.bson'), + Routines.BSON.serializeBSONWithoutOptimiser(ctx.input.build) + )*/ + + fs.writeFileSync( + Path.join(workspacePath, safeFileName+'.service.venue.json'), + JSON.stringify(ctx.input.build, null, 2) + ) // verify build signature From 189090796e51d762bdf98f4dab8b09ec3173a2c5 Mon Sep 17 00:00:00 2001 From: sevenbitbyte Date: Tue, 2 Jun 2026 21:40:04 +0000 Subject: [PATCH 13/49] initial project schema --- src/venue/schema/venue-project.js | 82 +++++++++++++++++++++++++++++++ 1 file changed, 82 insertions(+) create mode 100644 src/venue/schema/venue-project.js diff --git a/src/venue/schema/venue-project.js b/src/venue/schema/venue-project.js new file mode 100644 index 0000000..51d77fa --- /dev/null +++ b/src/venue/schema/venue-project.js @@ -0,0 +1,82 @@ +'use strict' + +const debug = require('debug')('venue.venue_project') + +const ISchema = require('../../bouncer/ischema') + +const Utils = ISchema.Utils + + +class VenueProject extends ISchema { + + static get Type () { return 'venue_project' } + + static get Schema(){ + return { + owner: {type: String, required: true, index: true}, //public_key.key.hash + created: {type: Number, required: true}, + changed: {type: Number}, + venue: {type: String}, + domain: {type: String, index: true, unique: true}, + + // i2p enable? + // standalone hosting? + + workspace: {type: String, required: true}, + //hash: {type: String, required: true, index: true}, + party: [{ + name: String, + type: {type: String, enum: ['TingoParty', 'LokiParty', 'PeerParty', 'MongoParty']}, + tingo: { + path: String + }, + loki: { + path: String + }, + peer: { + venue: String, + remoteIdentity: String + }, + key: { + hash: String, + securePrivate: String + }, + settings: { + noCache: Boolean, + }, + defaultConfig: Object + }], + routes: [{ + prefix: String, + party: String, + package: { + owner: String, + name: {type: String, required: true, index: true}, + version: String, + branch: String, + hash: String, + }, + settings: { + sendFullErrors: {type: Boolean, required: true}, + useNative: {type: Boolean, required: true}, + } + }] + } + } + + static setupSchema(schema){ + //schema.index({ 'package.name': 1 }, {unique: true}) + return schema + } + + static permissions (context) { + return { + read: false, + new: false, + change: false + } + } +} + + +module.exports = VenueProject \ No newline at end of file From 7a08ea0663bc95bd3c4b6ad87f7e7f2772a5439c Mon Sep 17 00:00:00 2001 From: sevenbitbyte Date: Tue, 9 Jun 2026 13:20:29 +0000 Subject: [PATCH 14/49] move reasable package parts --- .../endpoints/key-announce.js | 0 src/service/index.js | 22 +++++++++++++++---- .../middleware/pre/ephemeral-session.js | 2 +- src/{venue => service}/schema/public-key.js | 0 src/{venue => service}/schema/session-key.js | 0 src/venue/endpoints/create-package.js | 2 +- src/venue/venue-service.js | 13 +++++++---- 7 files changed, 29 insertions(+), 10 deletions(-) rename src/{venue => service}/endpoints/key-announce.js (100%) rename src/{venue => service}/middleware/pre/ephemeral-session.js (97%) rename src/{venue => service}/schema/public-key.js (100%) rename src/{venue => service}/schema/session-key.js (100%) diff --git a/src/venue/endpoints/key-announce.js b/src/service/endpoints/key-announce.js similarity index 100% rename from src/venue/endpoints/key-announce.js rename to src/service/endpoints/key-announce.js diff --git a/src/service/index.js b/src/service/index.js index bf2ed65..9f00a91 100644 --- a/src/service/index.js +++ b/src/service/index.js @@ -24,7 +24,8 @@ exports.MiddlewareRunner= require('./middleware-runner') exports.middleware = { pre: { decrypt: require('./middleware/pre/decrypt'), - validate: require('./middleware/pre/validate') + validate: require('./middleware/pre/validate'), + ephemeral_session: require('./middleware/pre/ephemeral-session.js') }, post: { validate: require('./middleware/post/validate.js'), @@ -35,7 +36,8 @@ exports.middleware = { exports.middleware_paths = { pre: { decrypt: Path.join(__dirname, './middleware/pre/decrypt.js'), - validate: Path.join(__dirname, './middleware/pre/validate.js') + validate: Path.join(__dirname, './middleware/pre/validate.js'), + ephemeral_session: Path.join(__dirname, './middleware/pre/ephemeral-session.js') }, post: { validate: Path.join(__dirname, './middleware/post/validate.js'), @@ -47,12 +49,24 @@ exports.endpoint = { echo: require('./endpoints/echo'), secureecho: require('./endpoints/secure-echo'), identity: require('./endpoints/service-identity'), - version: require('./endpoints/service-version') + version: require('./endpoints/service-version'), + key_announce: require('./endpoints/key-announce'), } exports.endpoint_paths = { echo: Path.join(__dirname, './endpoints/echo.js'), secureecho: Path.join(__dirname, './endpoints/secure-echo.js'), identity: Path.join(__dirname, './endpoints/service-identity.js'), - version: Path.join(__dirname, './endpoints/service-version.js') + version: Path.join(__dirname, './endpoints/service-version.js'), + key_announce: Path.join(__dirname, './endpoints/key-announce.js') +} + +exports.schema = { + public_key: require('./schema/public-key'), + session_key: require('./schema/session-key') +} + +exports.schema_paths = { + public_key: Path.join(__dirname, './schema/public-key.js'), + session_key: Path.join(__dirname, './schema/session-key.js') } \ No newline at end of file diff --git a/src/venue/middleware/pre/ephemeral-session.js b/src/service/middleware/pre/ephemeral-session.js similarity index 97% rename from src/venue/middleware/pre/ephemeral-session.js rename to src/service/middleware/pre/ephemeral-session.js index de0fb2a..fd099cf 100644 --- a/src/venue/middleware/pre/ephemeral-session.js +++ b/src/service/middleware/pre/ephemeral-session.js @@ -3,7 +3,7 @@ const Hoek = require('@hapi/hoek') const {Identity} = require('@dataparty/crypto') const debug = require('debug')('dataparty.middleware.pre.ephemeral-session') -const IMiddleware = require('../../../service/imiddleware') +const IMiddleware = require('../../imiddleware') module.exports = class Decrypt extends IMiddleware { diff --git a/src/venue/schema/public-key.js b/src/service/schema/public-key.js similarity index 100% rename from src/venue/schema/public-key.js rename to src/service/schema/public-key.js diff --git a/src/venue/schema/session-key.js b/src/service/schema/session-key.js similarity index 100% rename from src/venue/schema/session-key.js rename to src/service/schema/session-key.js diff --git a/src/venue/endpoints/create-package.js b/src/venue/endpoints/create-package.js index 30ff2bd..4eedd2d 100644 --- a/src/venue/endpoints/create-package.js +++ b/src/venue/endpoints/create-package.js @@ -256,7 +256,7 @@ module.exports = class CreatePkgEndpoint extends IEndpoint { debug('service created') } else { - debug('need to update service') + debug('need to update service?') } fs.writeFileSync( diff --git a/src/venue/venue-service.js b/src/venue/venue-service.js index c0f0a30..a304e47 100644 --- a/src/venue/venue-service.js +++ b/src/venue/venue-service.js @@ -12,22 +12,27 @@ class VenueService extends DatapartySrv.IService { let builder = new DatapartySrv.ServiceBuilder(this) - builder.addSchema(Path.join(__dirname, './schema/public-key.js')) - builder.addSchema(Path.join(__dirname, './schema/session-key.js')) + //builder.addSchema(Path.join(__dirname, './schema/public-key.js')) + //builder.addSchema(Path.join(__dirname, './schema/session-key.js')) builder.addSchema(Path.join(__dirname, './schema/venue_package.js')) + builder.addSchema(DatapartySrv.schema_paths.public_key) + builder.addSchema(DatapartySrv.schema_paths.session_key) + builder.addMiddleware(DatapartySrv.middleware_paths.pre.decrypt) builder.addMiddleware(DatapartySrv.middleware_paths.pre.validate) + builder.addMiddleware(DatapartySrv.middleware_paths.pre.ephemeral_session) - builder.addMiddleware(Path.join(__dirname, './middleware/pre/ephemeral-session.js')) + //builder.addMiddleware(Path.join(__dirname, './middleware/pre/ephemeral-session.js')) builder.addMiddleware(DatapartySrv.middleware_paths.post.validate) builder.addMiddleware(DatapartySrv.middleware_paths.post.encrypt) builder.addEndpoint(DatapartySrv.endpoint_paths.identity) builder.addEndpoint(DatapartySrv.endpoint_paths.version) + builder.addEndpoint(DatapartySrv.endpoint_paths.key_announce) - builder.addEndpoint(Path.join(__dirname, './endpoints/key-announce.js')) + //builder.addEndpoint(Path.join(__dirname, './endpoints/key-announce.js')) builder.addEndpoint(Path.join(__dirname, './endpoints/create-package.js')) From 237adf2b6764e5598bcdb0c3985b4780b94bff0a Mon Sep 17 00:00:00 2001 From: sevenbitbyte Date: Wed, 10 Jun 2026 01:42:42 +0000 Subject: [PATCH 15/49] venued commands --- package.json | 14 +- src/bouncer/ischema.js | 2 +- src/service/endpoints/key-announce.js | 2 +- src/service/service-builder.js | 18 +- src/service/service-host.js | 10 +- src/venue/bin/commands/venued-admin-add.js | 85 ++++++ src/venue/bin/commands/venued-admin-list.js | 69 +++++ src/venue/bin/commands/venued-host.js | 277 ++++++++++++++++++++ src/venue/bin/venue.js | 71 +++++ src/venue/bin/venued.js | 82 ++++++ src/venue/schema/venue_package.js | 2 +- 11 files changed, 617 insertions(+), 15 deletions(-) create mode 100644 src/venue/bin/commands/venued-admin-add.js create mode 100644 src/venue/bin/commands/venued-admin-list.js create mode 100644 src/venue/bin/commands/venued-host.js create mode 100755 src/venue/bin/venue.js create mode 100755 src/venue/bin/venued.js diff --git a/package.json b/package.json index 7c295b1..7b16dcd 100644 --- a/package.json +++ b/package.json @@ -47,6 +47,10 @@ "dist", "src/*" ], + "bin": { + "venue": "./src/venue/bin/venue.js", + "venued": "./src/venue/bin/venued.js" + }, "scripts": { "test": "npx lab", "build": "npx parcel build --no-scope-hoist", @@ -63,10 +67,10 @@ }, "dependencies": { "@babel/runtime": "^7.28.4", - "@dataparty/bouncer-db": "1.0.1", + "@dataparty/bouncer-db": "https://github.com/datapartyjs/bouncer-db#mongoose-latest", "@dataparty/crypto": "github:datapartyjs/dataparty-crypto", "@dataparty/tasker": "^0.0.3", - "@diva.exchange/i2p-sam": "^4.1.8", + "@diva.exchange/i2p-sam": "5.5.2", "@markwylde/liferaft": "^1.3.4", "@sevenbitbyte/ncc": "0.0.2", "ajv": "6.12.5", @@ -77,6 +81,7 @@ "buffer": "^6.0.3", "bufferutil": "^4.0.8", "colors": "1.3.1", + "command-tree": "github:datapartyjs/command-tree", "cors": "^2.8.5", "debug": "^3.1.0", "dom-storage": "^2.1.0", @@ -84,12 +89,13 @@ "express": "^4.17.1", "express-ipfilter": "^1.3.2", "express-list-routes": "^1.1.9", + "fast-safe-stringify": "^2.1.1", "git-repo-info": "^2.1.1", "glob": "^13.0.6", "joi": "^18.2.1", "joi-objectid": "^4.0.2", "jshashes": "^1.0.8", - "jsonpath-plus": "^0.20.1", + "jsonpath-plus": "10.4.0", "last-eventemitter": "^1.1.1", "lodash": "^4.17.21", "lokijs": "1.5.12", @@ -120,7 +126,7 @@ "zangodb": "https://github.com/sevenbitbyte/zangodb#hash-patch" }, "devDependencies": { - "@dataparty/bouncer-model": "1.4.3", + "@dataparty/bouncer-model": "https://github.com/datapartyjs/bouncer-model#mongoose-latest", "@hapi/code": "^9.0.1", "@hapi/joi": "^17.1.1", "@hapi/lab": "^25.0.1", diff --git a/src/bouncer/ischema.js b/src/bouncer/ischema.js index 78fd0a8..18c080b 100644 --- a/src/bouncer/ischema.js +++ b/src/bouncer/ischema.js @@ -1,4 +1,4 @@ -const debug = require('debug')('bouncer.ISchema') +const debug = require('debug')('dataparty.bouncer.ISchema') const MgoUtils = require('../utils/mongoose-scheme-utils') module.exports = class ISchema { diff --git a/src/service/endpoints/key-announce.js b/src/service/endpoints/key-announce.js index 6b281b6..b2768f5 100644 --- a/src/service/endpoints/key-announce.js +++ b/src/service/endpoints/key-announce.js @@ -5,7 +5,7 @@ const debug = require('debug')('dataparty.endpoint.key-announce') const {Identity, Message, Routines} = require('@dataparty/crypto') //const IEndpoint = require('@dataparty/api/src/service/iendpoint') -const IEndpoint = require('../../service/iendpoint') +const IEndpoint = require('../iendpoint') const KeyVerifier = Joi.object().keys({ id: Joi.string().max(100), diff --git a/src/service/service-builder.js b/src/service/service-builder.js index 08b8f94..c78d3fe 100644 --- a/src/service/service-builder.js +++ b/src/service/service-builder.js @@ -9,6 +9,8 @@ const mongoose = BouncerDb.mongoose() const debug = require('debug')('dataparty.service.ServiceBuilder') const zlib = require('zlib') +const safeStringify = require('fast-safe-stringify') + const dataparty_crypto = require('@dataparty/crypto') const { @@ -178,24 +180,32 @@ module.exports = class ServiceBuilder { this.service.compiled.schemas.Permissions[model.Type] = await model.permissions() this.service.compiled.schemas.JSONSchema.push(jsonSchema) + const safePaths = JSON.parse(safeStringify(schema.paths)) + + //debug(schema.paths) debug('\t','type',model.Type) let indexed = JSONPath({ path: '$..options.index', - json: schema.paths, + json: safePaths, resultType: 'pointer' - }).map(p=>{return p.split('/')[1]}) + }).map(p=>{ + + debug('\t\t','p',p) + return p + }) + //return p.split('.')[1]}) debug('\t\tindexed', indexed) let unique = JSONPath({ path: '$..options.unique', - json: schema.paths, + json: safePaths, resultType: 'pointer' }).map(p=>{ debug(typeof p) if(typeof p == 'string'){ - return p.split('/')[1] + return p.split('.')[1] } return p diff --git a/src/service/service-host.js b/src/service/service-host.js index 10033fa..6ebf54f 100644 --- a/src/service/service-host.js +++ b/src/service/service-host.js @@ -58,8 +58,9 @@ class ServiceHost { i2pSamHost = '127.0.0.1', i2pSamPort = 7656, i2pKey = null, - i2pForwardHost = 'localhost', + i2pForwardHost = '127.0.0.1', i2pForwardPort = null, + i2pOptions = null, wsEnabled = true, wsPort = null, wsUpgradePath = '/ws', @@ -146,6 +147,9 @@ class ServiceHost { forward: { host: i2pForwardHost ? i2pForwardHost : this.apiServerUri.hostname, port: i2pForwardPort ? i2pForwardPort : parseInt( this.apiServerUri.port ) + }, + session: { + options: i2pOptions } } } @@ -250,9 +254,7 @@ class ServiceHost { this.i2p = await SAM.createForward(this.i2pSettings) this.i2pUri = this.i2p.getB32Address() - this.i2pSettings.privateKey = null // clear no longer needed - - + this.i2pSettings.sam.privateKey = null // clear no longer needed this.i2p.on('error', this.reportI2pError.bind(this)) diff --git a/src/venue/bin/commands/venued-admin-add.js b/src/venue/bin/commands/venued-admin-add.js new file mode 100644 index 0000000..209e6fe --- /dev/null +++ b/src/venue/bin/commands/venued-admin-add.js @@ -0,0 +1,85 @@ +const CmdTree = require('command-tree') +const Hoek = require('@hapi/hoek') +const debug = require('debug')('venued.admin-add') +const Path = require('path') +const OS = require('os') +const fs = require('fs') + +const { execSync } = require('child_process') + +const Dataparty = require('../../../../') + +const HOMEDIR = OS.homedir() +const DEFAULT_FOLDER = (HOMEDIR.indexOf('opt')==-1) ? '.venued' : '' +const DEFAULT_PATH = Path.join( HOMEDIR, DEFAULT_FOLDER ) + + +const DEFINITION = { + h: { + description: 'Show help', + alias: 'help', + type: 'help' + }, + p: { + alias: 'path', + description: 'venued path', + default: DEFAULT_PATH + } +} + + +class VenuedAdminAdd extends CmdTree.Command { + constructor(context){ + super({...VenuedAdminAdd.Definition, context}) + debug('constructor') + } + + static get Command(){ + return 'admin add' + } + + static get Definition(){ + return { + usage: `venued admin add [key-hash]`, + description: 'Add admin key', + definition: DEFINITION + } + } + + async run({parsed}){ + //debug('context -', this.context) + //console.log('parsed -', parsed) + + if (parsed.h) { + throw new CmdTree.Error.HelpRequest('help request') + } + + const config = new Dataparty.Config.JsonFileConfig({basePath: parsed.path}) + + await config.start() + + + let admins = (await config.read('admins')) || [] + + + let newAdmins = [] + + + for(let admin of parsed._.slice(2)){ + if(admins.indexOf(admin) != -1){ continue } + + newAdmins.push(admin) + } + + admins = admins.concat(newAdmins) + + await config.write('admins', admins) + await config.save() + + //console.log('admin added -', newAdmins) + + return {newAdmins} + } +} + +module.exports = VenuedAdminAdd \ No newline at end of file diff --git a/src/venue/bin/commands/venued-admin-list.js b/src/venue/bin/commands/venued-admin-list.js new file mode 100644 index 0000000..88d1c29 --- /dev/null +++ b/src/venue/bin/commands/venued-admin-list.js @@ -0,0 +1,69 @@ +const CmdTree = require('command-tree') +const Hoek = require('@hapi/hoek') +const debug = require('debug')('venued.admin-list') +const Path = require('path') +const OS = require('os') +const fs = require('fs') + +const { execSync } = require('child_process') + +const Dataparty = require('../../../../') + +const HOMEDIR = OS.homedir() +const DEFAULT_FOLDER = (HOMEDIR.indexOf('opt')==-1) ? '.venued' : '' +const DEFAULT_PATH = Path.join( HOMEDIR, DEFAULT_FOLDER ) + + +const DEFINITION = { + h: { + description: 'Show help', + alias: 'help', + type: 'help' + }, + p: { + alias: 'path', + description: 'venued path', + default: DEFAULT_PATH + } +} + + +class VenuedAdminList extends CmdTree.Command { + constructor(context){ + super({...VenuedAdminList.Definition, context}) + debug('constructor') + } + + static get Command(){ + return 'admin list' + } + + static get Definition(){ + return { + usage: `venued admin list [key-hash]`, + description: 'Add admin key', + definition: DEFINITION + } + } + + async run({parsed}){ + //debug('context -', this.context) + //debug('parsed -', parsed) + + if (parsed.h) { + throw new CmdTree.Error.HelpRequest('help request') + } + + let config = new Dataparty.Config.JsonFileConfig({basePath: parsed.path}) + + await config.start() + + let admins = (await config.read('admins')) || [] + + config=null + + return {admins} + } +} + +module.exports = VenuedAdminList \ No newline at end of file diff --git a/src/venue/bin/commands/venued-host.js b/src/venue/bin/commands/venued-host.js new file mode 100644 index 0000000..8ebadd3 --- /dev/null +++ b/src/venue/bin/commands/venued-host.js @@ -0,0 +1,277 @@ +const CmdTree = require('command-tree') +const Hoek = require('@hapi/hoek') +const debug = require('debug')('venued.host') +const Path = require('path') +const OS = require('os') +const fs = require('fs') + +const { execSync } = require('child_process') + +const Dataparty = require('../../../../') + +const HOMEDIR = OS.homedir() +const DEFAULT_FOLDER = (HOMEDIR.indexOf('opt')==-1) ? '.venued' : '' +const DEFAULT_PATH = Path.join( HOMEDIR, DEFAULT_FOLDER ) + + +const DEFINITION = { + h: { + description: 'Show help', + alias: 'help', + type: 'help' + }, + p: { + alias: 'path', + description: 'venued path', + default: DEFAULT_PATH + }, + l: { + alias: 'listen', + description: 'listen uri this can be an https:// with any local IP or file:/// to use a Unix socket', + default: 'https://0.0.0.0:3000' + }, + k: { + alias: 'ssl-key', + description: 'SSL key in pem format', + default: Path.join(DEFAULT_PATH, 'key.pem') + }, + c: { + alias: 'ssl-cert', + description: 'SSL Certificate in pem format', + default: Path.join(DEFAULT_PATH, 'cert.pem') + }, + A: { + alias: 'allow-ip', + description: 'IP to add to allow list', + multiple: true + }, + 'db-type': { + description: 'Type of database', + valid: ['tingo', 'loki', 'mongo', 'peer'], + default: 'tingo' + }, + 'db-uri': { + description: 'url to connect to database', + default: Path.join(DEFAULT_PATH, 'db/') + }, + 'db-peer': { + description: 'Peer database identity hash' + }, + 'service-code': { + description: 'path to service implementation', + default: Path.join(__dirname, '../../venue-service.js') + }, + 'service-build': { + description: 'path to service build', + default: Path.join(__dirname, '../../dataparty/@dataparty-venue.service.venue.json') + }, + 'full-errors': { + description: 'full server side error messages sent to client', + type:'boolean', + default: false + }, + i2p: { + description: 'Enable i2p hosting', + type: 'boolean', + default: false + }, + 'i2p-host':{ + default: '127.0.0.1' + }, + 'i2p-port': { + default: 7656 + }, + 'trust-proxy': { + type: 'boolean', + default: false + } +} + +function getPartyByType(type){ + if(type == 'tingo'){ + return Dataparty.TingoParty + } +} + +class VenuedHost extends CmdTree.Command { + constructor(context){ + super({...VenuedHost.Definition, context}) + debug('constructor') + } + + static get Command(){ + return 'host' + } + + static get Definition(){ + return { + usage: `venued host [options]`, + description: 'Start the hosting venued', + definition: DEFINITION + } + } + + async run({parsed}){ + //debug('context -', this.context) + //debug('parsed -', parsed) + + if (parsed.h) { + throw new CmdTree.Error.HelpRequest('help request') + } + + /*if (!parsed.name){ + throw new CmdTree.Error.UsageError('no name provided') + }*/ + + + const ServiceCode = require(parsed['service-code']) + const ServiceBuild = require(parsed['service-build']) + const ServiceSchema = { + package: ServiceBuild.package, + ...ServiceBuild.schemas + } + + const PARTY = getPartyByType(parsed['db-type']) + + const config = new Dataparty.Config.JsonFileConfig({basePath: parsed.path}) + + await config.start() + + //if(parsed['db-uri'][0] == '/'){ + config.touchDir( 'db' ) + //} + + const party = new PARTY({ + path: parsed['db-uri'], + model: ServiceSchema, + config: config, + noCache: false + }) + + const service = new ServiceCode( ServiceSchema.package, ServiceBuild ) + + debug('loaded service') + + debug('party db location', parsed['db-uri']) + + const CustomIpFilter = { + options: { + mode: 'allow', + //trustProxy: true + }, + ips: [ + '173.245.48.0/20', + '103.21.244.0/22', + '103.22.200.0/22', + '103.31.4.0/22', + '141.101.64.0/18', + '108.162.192.0/18', + '190.93.240.0/20', + '188.114.96.0/20', + '197.234.240.0/22', + '198.41.128.0/17', + '162.158.0.0/15', + '104.16.0.0/13', + '104.24.0.0/14', + '172.64.0.0/13', + '131.0.72.0/22', + '2400:cb00::/32', + '2606:4700::/32', + '2803:f800::/32', + '2405:b500::/32', + '2405:8100::/32', + '2a06:98c0::/29', + '2c0f:f248::/32', + '10.115.68.55/32' //! + ] + } + + if(parsed['allow-ip']){ + for(let ip of parsed['allow-ip']){ + CustomIpFilter.ips.push(ip) + } + } + + + const runner = new Dataparty.ServiceRunnerNode({ + party, service, + sendFullErrors: parsed['full-errors'], + useNative: false, + prefix: 'venue/' + }) + + + let runnerRouter = new Dataparty.RunnerRouter(runner) + + if(!fs.existsSync(parsed['ssl-key'])){ + execSync('openssl req -newkey rsa:2048 -new -nodes -x509 -days 3650 -keyout key.pem -out cert.pem -subj "/C=US/ST=State/L=City/O=Organization/OU=Unit/CN=example.com"', + {cwd: parsed.path} + ) + } + + const ssl_key = fs.readFileSync( parsed['ssl-key'], 'utf8') + const ssl_cert = fs.readFileSync( parsed['ssl-cert'], 'utf8') + + + + if(parsed.i2p && !await config.read('i2p.sam')){ + debug('i2p - creating key') + + const SAM = require('@diva.exchange/i2p-sam') + + const i2pSettings = { + sam: { + host: parsed['i2p-host'], + portTCP: parsed['i2p-port'], + }, + session: { + options: 'i2cp.leaseSetEncType=6,4' + } + } + + let i2p = await SAM.createLocalDestination(i2pSettings) + + + await Promise.all([ + config.write('i2p.address', i2p.address), + config.write('i2p.sam.publicKey', i2p.public), + config.write('i2p.sam.privateKey', i2p.private), + ]) + + await config.save() + } + + const host = new Dataparty.ServiceHost({ + runner: runnerRouter, + trust_proxy: parsed['trust-proxy/'], + wsEnabled: true, + ssl_key, ssl_cert, + listenUri: parsed.listen, + staticPath: Path.join(__dirname,'../../public'), + staticPrefix: '/venue/', + ipFilter: CustomIpFilter, + i2pEnabled: parsed.i2p, + i2pSamHost: parsed['i2p-host'], + i2pSamPort: parsed['i2p-port'], + i2pForwardHost: '127.0.0.1', + i2pForwardPort: 3000, + //i2pOptions: 'i2cp.leaseSetEncType=6,4', + i2pOptions: 'i2cp.leaseSetEncType=4,0', + i2pKey: await config.read('i2p.sam') + }) + + await party.start() + await runner.start() + await host.start() + + debug('started') + console.log('partying') + console.log(Path.join(__dirname,'../public')) + + this.context.exiting = false + + return {} + } +} + +module.exports = VenuedHost \ No newline at end of file diff --git a/src/venue/bin/venue.js b/src/venue/bin/venue.js new file mode 100755 index 0000000..393955f --- /dev/null +++ b/src/venue/bin/venue.js @@ -0,0 +1,71 @@ +#!/usr/bin/env node + +const Pkg = require('../../../package.json') +const debug = require('debug')('venue') +const CommandTree = require('command-tree').CommandTree + + +const commandTree = new CommandTree({ usage: 'venue [command] \nVersion: ' + Pkg.version }) + +/*commandTree.addCommand(require('./project/project-init')) +commandTree.addCommand(require('./project/project-show')) +commandTree.addCommand(require('./project/project-mount')) +commandTree.addCommand(require('./developer/developer-add')) +commandTree.addCommand(require('./team/team-add')) +commandTree.addCommand(require('./cloud/cloud-add')) +commandTree.addCommand(require('./cloud/cloud-list')) +commandTree.addCommand(require('./package/package-add')) +commandTree.addCommand(require('./service/service-add'))*/ + + +/* +venued host --path /opt/venue +venued get version/identity/config +venued admin add/rm/list + +venue remote add --url [] --ws [] --identity <> +venue remote rm +venue remote switch + +venue package build --output [] +venue package deploy + +venue project build +venue project deploy +*/ + +async function main(){ + + if(process.argv.length < 3 || process.argv[2] == 'help' || process.argv[2] == '--help'){ + console.log(commandTree.getHelp()) + if(process.send){ process.send(commandTree.getHelp()) } + return + } + + + const output = await commandTree.run({context: { + + }}) + + if(output){ + console.log(output) + + if(process.send){ process.send({output}) } + } + +} + +// Run main +main().catch((error) => { + console.log(error) + console.error(error.message) + debug(error) + console.log(commandTree.getHelp()) + if(process.send){ + process.send({ + error: error, + output: commandTree.getHelp() + }) + } + //process.exit() +}) diff --git a/src/venue/bin/venued.js b/src/venue/bin/venued.js new file mode 100755 index 0000000..a189251 --- /dev/null +++ b/src/venue/bin/venued.js @@ -0,0 +1,82 @@ +#!/usr/bin/env -S node --trace-warnings + +const Pkg = require('../../../package.json') +const debug = require('debug')('venue') +const CommandTree = require('command-tree').CommandTree + + +const commandTree = new CommandTree({ usage: 'venued [command] \nVersion: ' + Pkg.version }) + +commandTree.addCommand(require('./commands/venued-host')) +commandTree.addCommand(require('./commands/venued-admin-add')) +commandTree.addCommand(require('./commands/venued-admin-list')) + +/*commandTree.addCommand(require('./project/project-init')) +commandTree.addCommand(require('./project/project-show')) +commandTree.addCommand(require('./project/project-mount')) +commandTree.addCommand(require('./developer/developer-add')) +commandTree.addCommand(require('./team/team-add')) +commandTree.addCommand(require('./cloud/cloud-add')) +commandTree.addCommand(require('./cloud/cloud-list')) +commandTree.addCommand(require('./package/package-add')) +commandTree.addCommand(require('./service/service-add'))*/ + + +/* +venued host --path /opt/venue +venued get version/identity/config +venued admin add/rm/list + +venue remote add --url [] --ws [] --identity <> +venue remote rm +venue remote switch + +venue package build --output [] +venue package deploy + +venue project build +venue project deploy +*/ + +let context = { + exiting: true +} + +async function main(){ + + if(process.argv.length < 3 || process.argv[2] == 'help' || process.argv[2] == '--help'){ + console.log(commandTree.getHelp()) + if(process.send){ process.send(commandTree.getHelp()) } + return + } + + + const output = await commandTree.run({context}) + + if(output){ + console.log(output) + + if(process.send){ process.send({output}) } + } + +} + +// Run main +main().catch((error) => { + console.log(error) + console.error(error.message) + debug(error) + console.log(commandTree.getHelp()) + if(process.send){ + process.send({ + error: error, + output: commandTree.getHelp() + }) + } + //process.exit() +}).finally(()=>{ + + if(context.exiting){ + process.exit() + } +}) diff --git a/src/venue/schema/venue_package.js b/src/venue/schema/venue_package.js index a205839..e7802bd 100644 --- a/src/venue/schema/venue_package.js +++ b/src/venue/schema/venue_package.js @@ -4,7 +4,7 @@ const debug = require('debug')('venue.venue_pkg') const ISchema = require('../../bouncer/ischema') -const Utils = ISchema.Utils +//const Utils = ISchema.Utils class VenuePkg extends ISchema { From e94db4e5fa7abdb443d877cc08335fd3738e0597 Mon Sep 17 00:00:00 2001 From: sevenbitbyte Date: Wed, 10 Jun 2026 07:27:19 +0000 Subject: [PATCH 16/49] moar commands --- src/service/service-host.js | 35 ++--- src/venue/bin/commands/venue-identity-gen.js | 85 ++++++++++++ src/venue/bin/commands/venue-identity-list.js | 56 ++++++++ src/venue/bin/commands/venue-identity-show.js | 85 ++++++++++++ src/venue/bin/commands/venue-remote-add.js | 94 +++++++++++++ src/venue/bin/commands/venue-remote-list.js | 71 ++++++++++ src/venue/bin/commands/venue-remote-show.js | 66 +++++++++ src/venue/bin/commands/venued-admin-list.js | 2 +- src/venue/bin/commands/venued-host.js | 11 +- src/venue/bin/venue.js | 126 ++++++++++++++---- src/venue/bin/venued.js | 2 +- src/venue/build.js | 1 + src/venue/schema/venue-project.js | 1 + 13 files changed, 593 insertions(+), 42 deletions(-) create mode 100644 src/venue/bin/commands/venue-identity-gen.js create mode 100644 src/venue/bin/commands/venue-identity-list.js create mode 100644 src/venue/bin/commands/venue-identity-show.js create mode 100644 src/venue/bin/commands/venue-remote-add.js create mode 100644 src/venue/bin/commands/venue-remote-list.js create mode 100644 src/venue/bin/commands/venue-remote-show.js diff --git a/src/service/service-host.js b/src/service/service-host.js index 6ebf54f..109edb7 100644 --- a/src/service/service-host.js +++ b/src/service/service-host.js @@ -250,27 +250,32 @@ class ServiceHost { if(this.i2pEnabled && this.i2p == null){ debug('starting i2p forward', this.i2pSettings) - const SAM = require('@diva.exchange/i2p-sam') - this.i2p = await SAM.createForward(this.i2pSettings) - this.i2pUri = this.i2p.getB32Address() - this.i2pSettings.sam.privateKey = null // clear no longer needed + async function setup_i2p(){ + const SAM = require('@diva.exchange/i2p-sam') - this.i2p.on('error', this.reportI2pError.bind(this)) + this.i2p = await SAM.createForward(this.i2pSettings) + this.i2pUri = this.i2p.getB32Address() + this.i2pSettings.sam.privateKey = null // clear no longer needed + this.i2p.on('error', this.reportI2pError.bind(this)) - this.i2p.on('close', ()=>{ - debug('i2p closed') - }) - this.i2p.on('data', (data)=>{ - debug('i2p data') - debug(data.toString()) - }) + this.i2p.on('close', ()=>{ + debug('i2p closed') + }) + + this.i2p.on('data', (data)=>{ + debug('i2p data') + debug(data.toString()) + }) + + debug('i2p started') + debug('\t', 'address', this.i2pUri) + debug('\t', 'key', this.i2p.getPublicKey()) + } - debug('i2p started') - debug('\t', 'address', this.i2pUri) - debug('\t', 'key', this.i2p.getPublicKey()) + setup_i2p() } if(this.mdnsEnabled && this.apiServer && this.apiServerUri.protocol != 'file:'){ diff --git a/src/venue/bin/commands/venue-identity-gen.js b/src/venue/bin/commands/venue-identity-gen.js new file mode 100644 index 0000000..6cb6d2b --- /dev/null +++ b/src/venue/bin/commands/venue-identity-gen.js @@ -0,0 +1,85 @@ +const CmdTree = require('command-tree') +const Hoek = require('@hapi/hoek') +const debug = require('debug')('venue.identity-gen') +const Path = require('path') +const OS = require('os') +const fs = require('fs') + +const prompt = require('prompt') +const argon2 = require('argon2') + +const { execSync } = require('child_process') + +const Dataparty = require('../../../../') +const dataparty_crypto = require('@dataparty/crypto') + +const DEFINITION = { + h: { + description: 'Show help', + alias: 'help', + type: 'help' + }, + nopassword: { + type: 'boolean', + default: false + }, + force: { + type: 'boolean', + default: false + } +} + + +class VenueIdentityGen extends CmdTree.Command { + constructor(context){ + super({...VenueIdentityGen.Definition, context}) + debug('constructor') + } + + static get Command(){ + return 'identity gen' + } + + static get Definition(){ + return { + usage: `venue identity gen [name]`, + description: 'Create an identity', + definition: DEFINITION + } + } + + async run({parsed}){ + //debug('context -', this.context) + //console.log('parsed -', parsed) + + if (parsed.h) { + throw new CmdTree.Error.HelpRequest('help request') + } + + if (parsed._.length != 3){ + throw new CmdTree.Error.UsageError('You must supply a name for the identity') + } + + const keyName = parsed._[2] + + if(await this.context.secureConfig.read('identity.'+keyName+'.phrase') && !parsed.force){ + throw new CmdTree.Error.UsageError('Key already exists!') + } + + const phrase = await dataparty_crypto.Routines.generateMnemonic() + + const password = parsed.nopassword ? null : await this.context.collectPassword() + + let key = await dataparty_crypto.Identity.fromMnemonic(phrase, password, argon2) + + key.id = keyName + + await this.context.secureConfig.write('identity.'+keyName+'.phrase', phrase) + + return {...key.toJSON()} + } +} + +module.exports = VenueIdentityGen + + diff --git a/src/venue/bin/commands/venue-identity-list.js b/src/venue/bin/commands/venue-identity-list.js new file mode 100644 index 0000000..171fdf3 --- /dev/null +++ b/src/venue/bin/commands/venue-identity-list.js @@ -0,0 +1,56 @@ +const CmdTree = require('command-tree') +const Hoek = require('@hapi/hoek') +const debug = require('debug')('venue.identity-list') +const Path = require('path') +const OS = require('os') +const fs = require('fs') + +const prompt = require('prompt') +const argon2 = require('argon2') + +const { execSync } = require('child_process') + +const Dataparty = require('../../../../') +const dataparty_crypto = require('@dataparty/crypto') + +const DEFINITION = { + h: { + description: 'Show help', + alias: 'help', + type: 'help' + } +} + + +class VenueIdentityList extends CmdTree.Command { + constructor(context){ + super({...VenueIdentityList.Definition, context}) + debug('constructor') + } + + static get Command(){ + return 'identity list' + } + + static get Definition(){ + return { + usage: `venue identity list [name]`, + description: 'List identity nicknames', + definition: DEFINITION + } + } + + async run({parsed}){ + if (parsed.h) { + throw new CmdTree.Error.HelpRequest('help request') + } + + const names = Object.keys(await this.context.secureConfig.read('identity')) + + return {names} + } +} + +module.exports = VenueIdentityList + + diff --git a/src/venue/bin/commands/venue-identity-show.js b/src/venue/bin/commands/venue-identity-show.js new file mode 100644 index 0000000..e289c74 --- /dev/null +++ b/src/venue/bin/commands/venue-identity-show.js @@ -0,0 +1,85 @@ +const CmdTree = require('command-tree') +const Hoek = require('@hapi/hoek') +const debug = require('debug')('venue.identity-show') +const Path = require('path') +const OS = require('os') +const fs = require('fs') + +const prompt = require('prompt') +const argon2 = require('argon2') + +const { execSync } = require('child_process') + +const Dataparty = require('../../../../') +const dataparty_crypto = require('@dataparty/crypto') + +const DEFINITION = { + h: { + description: 'Show help', + alias: 'help', + type: 'help' + }, + nopassword: { + type: 'boolean', + default: false + } +} + + +class VenueIdentityShow extends CmdTree.Command { + constructor(context){ + super({...VenueIdentityShow.Definition, context}) + debug('constructor') + } + + static get Command(){ + return 'identity show' + } + + static get Definition(){ + return { + usage: `venue identity show [name]`, + description: 'Show an identity', + definition: DEFINITION + } + } + + async run({parsed}){ + //debug('context -', this.context) + //console.log('parsed -', parsed) + + if (parsed.h) { + throw new CmdTree.Error.HelpRequest('help request') + } + + if (parsed._.length != 3){ + throw new CmdTree.Error.UsageError('You must supply a name for the identity') + } + + const keyName = parsed._[2] + + const phrase = await this.context.secureConfig.read('identity.'+keyName+'.phrase') + + if(!phrase){ + throw new CmdTree.Error.UsageError("Key doesn't exists!") + } + + const {password} = parsed.nopassword ? {password:null} : await prompt.get({ + properties: { + password: { + message: 'Enter password for identity['+keyName+']', + hidden: true + } + }}) + + let key = await dataparty_crypto.Identity.fromMnemonic(phrase, password, argon2) + + key.id = keyName + + return {...key.toJSON()} + } +} + +module.exports = VenueIdentityShow + + diff --git a/src/venue/bin/commands/venue-remote-add.js b/src/venue/bin/commands/venue-remote-add.js new file mode 100644 index 0000000..96a8093 --- /dev/null +++ b/src/venue/bin/commands/venue-remote-add.js @@ -0,0 +1,94 @@ +const CmdTree = require('command-tree') +const Hoek = require('@hapi/hoek') +const debug = require('debug')('venue.remote-add') +const Path = require('path') +const OS = require('os') +const fs = require('fs') + +const prompt = require('prompt') +const argon2 = require('argon2') + +const Dataparty = require('../../../../') +const dataparty_crypto = require('@dataparty/crypto') + +const DEFINITION = { + h: { + description: 'Show help', + alias: 'help', + type: 'help' + }, + i2p: { + type: 'string', + description: 'i2p address' + }, + ws: { + type: 'string', + description: 'websocket url' + }, + url: { + type: 'string', + description: 'api base url' + }, + hash: { + type: 'string', + description: 'key hash of remote' + }, + force: { + type: 'boolean', + default: false + } +} + + +class VenueRemoteAdd extends CmdTree.Command { + constructor(context){ + super({...VenueRemoteAdd.Definition, context}) + } + + static get Command(){ + return 'remote add' + } + + static get Definition(){ + return { + usage: `venue remote add [name]`, + description: 'Add remote party', + definition: DEFINITION + } + } + + async run({parsed}){ + if (parsed.h) { + throw new CmdTree.Error.HelpRequest('help request') + } + + if (parsed._.length != 3){ + throw new CmdTree.Error.UsageError('You must supply a name for the remote') + } + + const remoteName = parsed._[2] + + if(await this.context.secureConfig.read('remote.'+remoteName) && !parsed.force){ + throw new CmdTree.Error.UsageError('Remote already exists!') + } + + const identityUrl = parsed.url+'/identity' + const versionUrl = parsed.url+'/version' + + const identity = await Dataparty.Comms.RestComms.HttpGet(identityUrl) + const version = await Dataparty.Comms.RestComms.HttpGet(versionUrl) + + const remote = { + url: parsed.url, + ws: parsed.ws, + i2p: parsed.i2p, + identity, version + } + + await this.context.secureConfig.write('remote.'+remoteName, remote) + + return { remote } + } +} + +module.exports = VenueRemoteAdd diff --git a/src/venue/bin/commands/venue-remote-list.js b/src/venue/bin/commands/venue-remote-list.js new file mode 100644 index 0000000..ed9b1fe --- /dev/null +++ b/src/venue/bin/commands/venue-remote-list.js @@ -0,0 +1,71 @@ +const CmdTree = require('command-tree') +const Hoek = require('@hapi/hoek') +const debug = require('debug')('venue.remote-list') +const Path = require('path') +const OS = require('os') +const fs = require('fs') + +const prompt = require('prompt') +const argon2 = require('argon2') + +const Dataparty = require('../../../../') +const dataparty_crypto = require('@dataparty/crypto') + +const DEFINITION = { + h: { + description: 'Show help', + alias: 'help', + type: 'help' + }, + i2p: { + type: 'string', + description: 'i2p address' + }, + ws: { + type: 'string', + description: 'websocket url' + }, + url: { + type: 'string', + description: 'api base url' + }, + hash: { + type: 'string', + description: 'key hash of remote' + }, + force: { + type: 'boolean', + default: false + } +} + + +class VenueRemoteList extends CmdTree.Command { + constructor(context){ + super({...VenueRemoteList.Definition, context}) + } + + static get Command(){ + return 'remote list' + } + + static get Definition(){ + return { + usage: `venue remote list [name]`, + description: 'List remote parties', + definition: DEFINITION + } + } + + async run({parsed}){ + if (parsed.h) { + throw new CmdTree.Error.HelpRequest('help request') + } + + const names = Object.keys(await this.context.secureConfig.read('remote')) + + return { names } + } +} + +module.exports = VenueRemoteList diff --git a/src/venue/bin/commands/venue-remote-show.js b/src/venue/bin/commands/venue-remote-show.js new file mode 100644 index 0000000..c8f1e63 --- /dev/null +++ b/src/venue/bin/commands/venue-remote-show.js @@ -0,0 +1,66 @@ +const CmdTree = require('command-tree') +const Hoek = require('@hapi/hoek') +const debug = require('debug')('venue.remote-show') +const Path = require('path') +const OS = require('os') +const fs = require('fs') + +const prompt = require('prompt') +const argon2 = require('argon2') + +const { execSync } = require('child_process') + +const Dataparty = require('../../../../') +const dataparty_crypto = require('@dataparty/crypto') + +const DEFINITION = { + h: { + description: 'Show help', + alias: 'help', + type: 'help' + } +} + + +class VenueRemoteShow extends CmdTree.Command { + constructor(context){ + super({...VenueRemoteShow.Definition, context}) + debug('constructor') + } + + static get Command(){ + return 'remote show' + } + + static get Definition(){ + return { + usage: `venue remote show [name]`, + description: 'Show a remote party', + definition: DEFINITION + } + } + + async run({parsed}){ + //debug('context -', this.context) + //console.log('parsed -', parsed) + + if (parsed.h) { + throw new CmdTree.Error.HelpRequest('help request') + } + + if (parsed._.length != 3){ + throw new CmdTree.Error.UsageError('You must supply a name for the remote') + } + + const remoteName = parsed._[2] + + const remote = await this.context.secureConfig.read('remote.'+remoteName) + + + return {remote} + } +} + +module.exports = VenueRemoteShow + + diff --git a/src/venue/bin/commands/venued-admin-list.js b/src/venue/bin/commands/venued-admin-list.js index 88d1c29..da0555e 100644 --- a/src/venue/bin/commands/venued-admin-list.js +++ b/src/venue/bin/commands/venued-admin-list.js @@ -41,7 +41,7 @@ class VenuedAdminList extends CmdTree.Command { static get Definition(){ return { usage: `venued admin list [key-hash]`, - description: 'Add admin key', + description: 'List admin keys', definition: DEFINITION } } diff --git a/src/venue/bin/commands/venued-host.js b/src/venue/bin/commands/venued-host.js index 8ebadd3..c53d66f 100644 --- a/src/venue/bin/commands/venued-host.js +++ b/src/venue/bin/commands/venued-host.js @@ -266,11 +266,18 @@ class VenuedHost extends CmdTree.Command { debug('started') console.log('partying') - console.log(Path.join(__dirname,'../public')) + console.log('\t', parsed.listen) + + const i2pAddress = await config.read('i2p.address') + if(i2pAddress){ + console.log('\t', i2pAddress) + } + + //console.log(Path.join(__dirname,'../public')) this.context.exiting = false - return {} + return } } diff --git a/src/venue/bin/venue.js b/src/venue/bin/venue.js index 393955f..d99e4fc 100755 --- a/src/venue/bin/venue.js +++ b/src/venue/bin/venue.js @@ -3,36 +3,79 @@ const Pkg = require('../../../package.json') const debug = require('debug')('venue') const CommandTree = require('command-tree').CommandTree +const prompt = require('prompt') +const argon2 = require('argon2') +const OS = require('os') +const Path = require('path') + + +const Dataparty = require('../../../') const commandTree = new CommandTree({ usage: 'venue [command] \nVersion: ' + Pkg.version }) -/*commandTree.addCommand(require('./project/project-init')) -commandTree.addCommand(require('./project/project-show')) -commandTree.addCommand(require('./project/project-mount')) -commandTree.addCommand(require('./developer/developer-add')) -commandTree.addCommand(require('./team/team-add')) -commandTree.addCommand(require('./cloud/cloud-add')) -commandTree.addCommand(require('./cloud/cloud-list')) -commandTree.addCommand(require('./package/package-add')) -commandTree.addCommand(require('./service/service-add'))*/ +commandTree.addCommand(require('./commands/venue-identity-gen')) +commandTree.addCommand(require('./commands/venue-identity-list')) +commandTree.addCommand(require('./commands/venue-identity-show')) + +commandTree.addCommand(require('./commands/venue-remote-add')) +commandTree.addCommand(require('./commands/venue-remote-list')) +commandTree.addCommand(require('./commands/venue-remote-show')) + +const HOMEDIR = OS.homedir() +const DEFAULT_FOLDER = '.venue' +const DEFAULT_PATH = Path.join( HOMEDIR, DEFAULT_FOLDER ) + +let secureConfig = null +async function collectPassword(info=''){ + let password = '' -/* -venued host --path /opt/venue -venued get version/identity/config -venued admin add/rm/list + while(1){ + let passes = await prompt.get({ + properties: { + password1: { + message: 'Set'+info+' password', + hidden: true + }, + password2: { + message: 'Confim'+info+' password', + hidden: true + } + } + }) -venue remote add --url [] --ws [] --identity <> -venue remote rm -venue remote switch + if(passes.password1 == passes.password2){ -venue package build --output [] -venue package deploy + password = passes.password1 + break + } + + console.log("passwords don't match") + } + + return password +} -venue project build -venue project deploy -*/ +async function onSetupRequired(){ + + console.log('setup-required') + + const password = await collectPassword(' keychain') + + await secureConfig.setPassword(password, { + created: Date.now() + }) + + console.log('password set') + + + await secureConfig.unlock(password) +} + +let context = { + exiting: true +} async function main(){ @@ -41,10 +84,42 @@ async function main(){ if(process.send){ process.send(commandTree.getHelp()) } return } + + + let config = new Dataparty.Config.JsonFileConfig({basePath: DEFAULT_PATH}) + secureConfig = new Dataparty.Config.SecureConfig({ + config, + timeoutMs: 60*1000*5, + argon: argon2 + }) + + secureConfig.on('setup-required', onSetupRequired) + + console.log('starting') + + await config.start() + await secureConfig.start() + + + if(await secureConfig.isInitialized() && secureConfig.isLocked()){ + + const {password} = await prompt.get({ + properties: { + password: { + message: 'Enter password', + hidden: true + } + }}) + + await secureConfig.unlock(password) + } + await secureConfig.waitForUnlocked('startup') + const output = await commandTree.run({context: { - + secureConfig, collectPassword, + ...context }}) if(output){ @@ -68,4 +143,9 @@ main().catch((error) => { }) } //process.exit() -}) +}).finally(()=>{ + + if(context.exiting){ + process.exit() + } +}) \ No newline at end of file diff --git a/src/venue/bin/venued.js b/src/venue/bin/venued.js index a189251..fbe37cd 100755 --- a/src/venue/bin/venued.js +++ b/src/venue/bin/venued.js @@ -1,4 +1,4 @@ -#!/usr/bin/env -S node --trace-warnings +#!/usr/bin/env node const Pkg = require('../../../package.json') const debug = require('debug')('venue') diff --git a/src/venue/build.js b/src/venue/build.js index e126d57..98c55ab 100644 --- a/src/venue/build.js +++ b/src/venue/build.js @@ -59,6 +59,7 @@ venue: { - default-config.json - public/ - package/NAME + - project/NAME - venue.json { projects: { diff --git a/src/venue/schema/venue-project.js b/src/venue/schema/venue-project.js index 51d77fa..9166252 100644 --- a/src/venue/schema/venue-project.js +++ b/src/venue/schema/venue-project.js @@ -17,6 +17,7 @@ class VenueProject extends ISchema { created: {type: Number, required: true}, changed: {type: Number}, venue: {type: String}, + name: {type: String}, domain: {type: String, index: true, unique: true}, // i2p enable? From b6e4ac74409bf403113607276b495f952b3535fd Mon Sep 17 00:00:00 2001 From: sevenbitbyte Date: Wed, 10 Jun 2026 09:25:31 +0000 Subject: [PATCH 17/49] venue package build --- package.json | 1 + src/service/service-builder.js | 20 ++++- src/venue/bin/commands/pkg-build.js | 122 ++++++++++++++++++++++++++++ src/venue/bin/venue.js | 2 + src/venue/build.js | 4 +- 5 files changed, 142 insertions(+), 7 deletions(-) create mode 100644 src/venue/bin/commands/pkg-build.js diff --git a/package.json b/package.json index 7b16dcd..88e0447 100644 --- a/package.json +++ b/package.json @@ -90,6 +90,7 @@ "express-ipfilter": "^1.3.2", "express-list-routes": "^1.1.9", "fast-safe-stringify": "^2.1.1", + "find-up-json": "^2.0.5", "git-repo-info": "^2.1.1", "glob": "^13.0.6", "joi": "^18.2.1", diff --git a/src/service/service-builder.js b/src/service/service-builder.js index c78d3fe..8e5e2bb 100644 --- a/src/service/service-builder.js +++ b/src/service/service-builder.js @@ -52,7 +52,7 @@ module.exports = class ServiceBuilder { debug('compiling sources',this.service.sources) - await Promise.all([ + let results = await Promise.all([ this.compileMiddleware('pre'), this.compileMiddleware('post'), this.compileList('documents'), @@ -82,6 +82,7 @@ module.exports = class ServiceBuilder { } } + let files = [] if(writeFile){ const buildOutput = outputPath+'/'+ this.service.compiled.package.name.replace('/', '-') +'.service.venue.json' @@ -100,11 +101,16 @@ module.exports = class ServiceBuilder { const compressedBrotli = zlib.brotliCompressSync(JSON.stringify(this.service.compiled, null,2)); console.log('Original:', JSON.stringify(this.service.compiled, null,2).length, 'bytes'); - console.log('Gzip:', compressed.length, 'bytes'); - console.log('Brotli:', compressedBrotli.length, 'bytes'); + console.log('Gzip:', compressed.length, 'bytes') + console.log('Brotli:', compressedBrotli.length, 'bytes') + + let tarFile = results[ results.length - 1 ] + files.push(buildOutput) + files.push(schemaOutput) + if(tarFile){files.push(tarFile)} } - return this.service.compiled + return {build: this.service.compiled, files} } @@ -384,6 +390,8 @@ module.exports = class ServiceBuilder { async compressFiles(outputPath, writeFile){ + if(!this.service.sources.files){ return } + let fileMap={} let files = this.service.sources.files.map(file=>{ @@ -398,6 +406,8 @@ module.exports = class ServiceBuilder { return hash }) + if(!files || files.length < 1){ return } + const tarFileName = this.service.compiled.package.name.replace('/', '-')+'.files.venue.tgz' const tarPath = Path.join(outputPath, tarFileName) @@ -420,5 +430,7 @@ module.exports = class ServiceBuilder { files: fileMap } } + + return tarPath } } \ No newline at end of file diff --git a/src/venue/bin/commands/pkg-build.js b/src/venue/bin/commands/pkg-build.js new file mode 100644 index 0000000..11a9db7 --- /dev/null +++ b/src/venue/bin/commands/pkg-build.js @@ -0,0 +1,122 @@ +const CmdTree = require('command-tree') +const Hoek = require('@hapi/hoek') +const debug = require('debug')('venue.package-build') +const Path = require('path') +const OS = require('os') +const fs = require('fs') +const mkdirp = require('mkdirp') + +const prompt = require('prompt') +const argon2 = require('argon2') + +const { execSync } = require('child_process') +const findUp = require('find-up-json').default + + +const Dataparty = require('../../../../') +const dataparty_crypto = require('@dataparty/crypto') + +const DEFINITION = { + h: { + description: 'Show help', + alias: 'help', + type: 'help' + }, + o: { + alias: 'output', + type: 'string', + default: Path.join(process.cwd(), 'dist/') + }, + nopassword: { + type: 'boolean', + default: false + }, + identity: { + type: 'string', + description: 'developer release identity', + require: true + }, + name: { + description: 'package name' + }, + version: { + description: 'package version' + } +} + + +class VenuePackageBuild extends CmdTree.Command { + constructor(context){ + super({...VenuePackageBuild.Definition, context}) + debug('constructor') + } + + static get Command(){ + return 'package build' + } + + static get Definition(){ + return { + usage: `venue package build [service-code.js]`, + description: 'Build a package', + definition: DEFINITION + } + } + + async run({parsed}){ + //debug('context -', this.context) + //console.log('parsed -', parsed) + + if (parsed.h) { + throw new CmdTree.Error.HelpRequest('help request') + } + + if (parsed._.length != 3){ + throw new CmdTree.Error.UsageError('You must supply service code') + } + + const keyName = parsed.identity + + const phrase = await this.context.secureConfig.read('identity.'+keyName+'.phrase') + + if(!phrase){ + throw new CmdTree.Error.UsageError("Key doesn't exist!") + } + + const {password} = parsed.nopassword ? {password:null} : await prompt.get({ + properties: { + password: { + message: 'Enter password for identity['+keyName+']', + hidden: true + } + }}) + + let key = await dataparty_crypto.Identity.fromMnemonic(phrase, password, argon2) + + key.id = keyName + + const serviceClassPath = Path.resolve(parsed._[2]) + + let foundUp = findUp('package.json', Path.dirname(serviceClassPath)) + + let pkgJson = foundUp.content + + const ServiceClass = require( serviceClassPath ) + + const service = new ServiceClass({ + name: parsed.name ? parsed.name : pkgJson.name, + version: parsed.version ? parsed.version : pkgJson.version, + }) + + await mkdirp(parsed.output) + + const builder = new Dataparty.ServiceBuilder(service) + const build = await builder.compile(parsed.output, true, key) + + return {files: build.files} + } +} + +module.exports = VenuePackageBuild + + diff --git a/src/venue/bin/venue.js b/src/venue/bin/venue.js index d99e4fc..fc715db 100755 --- a/src/venue/bin/venue.js +++ b/src/venue/bin/venue.js @@ -22,6 +22,8 @@ commandTree.addCommand(require('./commands/venue-remote-add')) commandTree.addCommand(require('./commands/venue-remote-list')) commandTree.addCommand(require('./commands/venue-remote-show')) +commandTree.addCommand(require('./commands/pkg-build')) + const HOMEDIR = OS.homedir() const DEFAULT_FOLDER = '.venue' const DEFAULT_PATH = Path.join( HOMEDIR, DEFAULT_FOLDER ) diff --git a/src/venue/build.js b/src/venue/build.js index 98c55ab..04e0416 100644 --- a/src/venue/build.js +++ b/src/venue/build.js @@ -174,11 +174,9 @@ async function main(){ await pushService( party.privateIdentity, build, staticTar ) - - } main().catch(err=>{ console.error('CRASH') console.error(err) -}) \ No newline at end of file +}) From 690a5428f0522f868b08562632f5b8d56b27893f Mon Sep 17 00:00:00 2001 From: sevenbitbyte Date: Fri, 12 Jun 2026 01:26:53 +0000 Subject: [PATCH 18/49] i2p tweaks --- scripts/install-i2p-ubuntu-24.sh | 6 ++++++ src/service/service-host.js | 7 +++++-- src/venue/bin/commands/venued-host.js | 9 +++++---- 3 files changed, 16 insertions(+), 6 deletions(-) create mode 100755 scripts/install-i2p-ubuntu-24.sh diff --git a/scripts/install-i2p-ubuntu-24.sh b/scripts/install-i2p-ubuntu-24.sh new file mode 100755 index 0000000..55019f1 --- /dev/null +++ b/scripts/install-i2p-ubuntu-24.sh @@ -0,0 +1,6 @@ +#!/bin/bash + +sudo add-apt-repository ppa:purplei2p/i2pd +sudo apt update +sudo apt install --yes i2pd + diff --git a/src/service/service-host.js b/src/service/service-host.js index 109edb7..85bb43e 100644 --- a/src/service/service-host.js +++ b/src/service/service-host.js @@ -142,9 +142,12 @@ class ServiceHost { host: i2pSamHost, portTCP: i2pSamPort, publicKey: reach(i2pKey, 'publicKey'), - privateKey: reach(i2pKey, 'privateKey') + privateKey: reach(i2pKey, 'privateKey'), + versionMin: '3.1', + versionMax: '3.3' }, forward: { + silent: true, host: i2pForwardHost ? i2pForwardHost : this.apiServerUri.hostname, port: i2pForwardPort ? i2pForwardPort : parseInt( this.apiServerUri.port ) }, @@ -275,7 +278,7 @@ class ServiceHost { debug('\t', 'key', this.i2p.getPublicKey()) } - setup_i2p() + setup_i2p.bind(this)() } if(this.mdnsEnabled && this.apiServer && this.apiServerUri.protocol != 'file:'){ diff --git a/src/venue/bin/commands/venued-host.js b/src/venue/bin/commands/venued-host.js index c53d66f..2ef584c 100644 --- a/src/venue/bin/commands/venued-host.js +++ b/src/venue/bin/commands/venued-host.js @@ -226,6 +226,7 @@ class VenuedHost extends CmdTree.Command { }, session: { options: 'i2cp.leaseSetEncType=6,4' + //options: 'i2cp.leaseSetEncType=4' } } @@ -254,9 +255,9 @@ class VenuedHost extends CmdTree.Command { i2pSamHost: parsed['i2p-host'], i2pSamPort: parsed['i2p-port'], i2pForwardHost: '127.0.0.1', - i2pForwardPort: 3000, - //i2pOptions: 'i2cp.leaseSetEncType=6,4', - i2pOptions: 'i2cp.leaseSetEncType=4,0', + i2pForwardPort: '3000', + i2pOptions: 'i2cp.leaseSetEncType=6,4', + //i2pOptions: 'i2cp.leaseSetEncType=4', i2pKey: await config.read('i2p.sam') }) @@ -281,4 +282,4 @@ class VenuedHost extends CmdTree.Command { } } -module.exports = VenuedHost \ No newline at end of file +module.exports = VenuedHost From df68ba395e974d86d3c976faf53c9abc62b88920 Mon Sep 17 00:00:00 2001 From: sevenbitbyte Date: Sat, 13 Jun 2026 02:17:04 +0000 Subject: [PATCH 19/49] initial create project --- src/venue/endpoints/create-project.js | 308 ++++++++++++++++++++++++++ src/venue/schema/venue-project.js | 83 ------- src/venue/schema/venue_project.js | 99 +++++++++ src/venue/venue-service.js | 2 + 4 files changed, 409 insertions(+), 83 deletions(-) create mode 100644 src/venue/endpoints/create-project.js delete mode 100644 src/venue/schema/venue-project.js create mode 100644 src/venue/schema/venue_project.js diff --git a/src/venue/endpoints/create-project.js b/src/venue/endpoints/create-project.js new file mode 100644 index 0000000..ab0d229 --- /dev/null +++ b/src/venue/endpoints/create-project.js @@ -0,0 +1,308 @@ +const fs = require('fs') +const Joi = require('joi') +const Hoek = require('@hapi/hoek') +const {Message, Routines, Identity} = require('@dataparty/crypto') +const debug = require('debug')('dataparty.endpoint.create-project') +const zlib = require('zlib') + +const IEndpoint = require('../../service/iendpoint') + +const typedArraySchema = (value, helpers) => { + // 1. Ensure the value is an instance of a TypedArray (e.g., Uint8Array) + if (!(value instanceof Uint8Array)) { + return helpers.message({ custom: '"value" must be a Uint8Array' }); + } + + return value +} + +module.exports = class CreateProjectEndpoint extends IEndpoint { + + static get Name(){ + return 'create-project' + } + + + static get Description(){ + return 'Create venue project' + } + + + + static get MiddlewareConfig(){ + return { + pre: { + decrypt: true, + ephemeral_session: true, + validate: Joi.object().keys({ + project: Joi.object().keys({ + owner: Joi.string(), + created: Joi.number(), + changed: Joi.number(), + + name: Joi.string(), + venue: Joi.string(), + domain: Joi.string(), + + i2p: Joi.object().keys({ + address: Joi.string(), + public: Joi.string(), + securePrivate: Joi.string() + }), + party: Joi.array().items(Joi.object().keys({ + name: Joi.string(), + type: Joi.string(), + tingo: { path: Joi.string() }, + loki: { path: Joi.string() }, + peer: { + venue: Joi.string(), + remoteIdentity: Joi.string() + }, + key: { + hash: Joi.string(), + securePrivate: Joi.string() + }, + settings: { noCache: Joi.string() }, + defaultConfig: Joi.string() + })), + routes: Joi.array().items(Joi.object().keys({ + prefix: Joi.string(), + party: Joi.string(), + package: Joi.object().keys({ + owner: Joi.string(), + name: Joi.string().required(), + version: Joi.string(), + branch: Joi.string(), + hash: Joi.string() + }), + settings: { + sendFullErrors: Joi.boolean().required(), + useNative: Joi.boolean().required() + } + })), + files: Joi.object().pattern(Joi.string(), Joi.object().keys({ + tar: Joi.string(), + hash: Joi.string().required(), + size: Joi.number().required(), + files: Joi.object().pattern(Joi.string(), Joi.object().keys({ + hash: Joi.string().required(), + size: Joi.number().required() + })) + })) + }).required(), + signatures: Joi.object().pattern(Joi.string(), Joi.string()).required(), + staticTar: Joi.any().custom(typedArraySchema) + }) + }, + post: { + encrypt: true, + validate: Joi.object().keys(null).description('any output allowed') + } + } + } + + static async run(ctx){ + + if(ctx.party.identity.key.hash != ctx.input.project.hash){ + throw new Error('project venue does not match this host') + } + + let {signatures, ...projectWithoutSig} = ctx.input.project + + const projectOwnerIdentityDoc = (await ctx.party.find() + .type('public_key') + .where('hash').equals( ctx.input.project.owner ) + .exec() + )[0] + + if(!projectOwnerIdentityDoc){ + throw new Error('project owner not authorized') + } + + debug('found project owner', projectOwnerIdentityDoc.hash) + + const projectOwnerIdentity = Identity.fromJSON({ + id: '', + key: { + type: projectOwnerIdentityDoc.data.type, + hash: projectOwnerIdentityDoc.data.hash, + public: projectOwnerIdentityDoc.data.public + } + }) + + + debug('inflated identity') + + const devSig = Routines.Utils.base64.decode(signatures[ctx.input.project.owner]) + + let signedProjectMsg = new Message({ + msg: projectWithoutSig, + sig: devSig + }) + + debug('sigs', signatures) + + debug('verifying package signature') + + await signedProjectMsg.assertVerified(projectOwnerIdentity, true) + + debug('verified package signature') + + const tarHash = Routines.Utils.hash(ctx.input.staticTar) + const tarHash64 = Routines.Utils.base64.encode( tarHash ) + + const safeFileName = ctx.input.project.name.replace('/', '-') + const tarFileName = safeFileName+'.files.venue.tgz' + + const projectFiles = ctx.input.project.files[tarFileName] + + if(projectFiles && projectFiles.hash != tarHash64){ + throw new Error("staticTar hash doesn't match project definition") + } + + debug('verified staticTar') + + debug('verified project - '+ctx.input.project.name+'@'+ctx.input.project.version) + + const projectBSON = Routines.BSON.serializeBSONWithoutOptimiser(projectWithoutSig) + + const projectHash = Routines.Utils.base64.encode( + Routines.Utils.hash( + projectBSON + ) + ) + + const safeProjectHash = projectHash.replace(/\//g, "-") + + debug('\t'+'hash', projectHash) + + const projectWorkspace = 'projects'+safeFileName+'/'+ctx.input.project.version+'/'+safeProjectHash + + const config = ctx.party.config + + const workspacePath = await config.touchDir(projectWorkspace) + + + debug('\t'+'workspace - local', projectWorkspace) + debug('\t'+'workspace - global', workspacePath) + + + + const compressedBrotliBuild = zlib.brotliCompressSync(JSON.stringify(ctx.input.project)) + + const project = ctx.input.project + const projectId = project.name + '@' + project.version + debug('addProject', projectId) + + let projectDoc = (await ctx.party.find() + .type('venue_project') + .where('project.name').equals(project.name) + .where('project.version').equals(project.version) + .where('hash').equals(projectHash) + .exec())[0] + + + if(!projectDoc){ + debug('creating project') + + const {owner, ...pkgWithoutOwner} = project.package + + projectDoc = await ctx.party.createDocument('venue_pkg', { + owner: project.owner, + created: Date.now(), + hash: projectHash, + workspace: workspacePath, + project: ctx.input.project, + signatures: ctx.input.signatures + }) + + debug('project created') + } else { + debug('need to update project?') + } + + fs.writeFileSync( + Path.join(workspacePath, tarFileName), + ctx.input.staticTar + ) + + /*fs.writeFileSync( + Path.join(workspacePath, safeFileName+'.service.venue.bson'), + Routines.BSON.serializeBSONWithoutOptimiser(ctx.input.build) + )*/ + + fs.writeFileSync( + Path.join(workspacePath, safeFileName+'.project.venue.json'), + JSON.stringify(ctx.input.build, null, 2) + ) + + // verify build signature + + //ctx.input. + + // untar listed files + + // verify static file checksums match verified signatures + + // create db entry + + /* + + const compiledSrv = JSON.parse(ctx.input.service) + const projectId = compiledSrv.package.name + '-' + compiledSrv.package.version + debug('addService', projectId) + + let projectDoc = (await ctx.party.find() + .type('venue_srv') + .where('name').equals(compiledSrv.package.name) + .exec())[0] + + + + if(!projectDoc){ + debug('creating service') + projectDoc = await ctx.party.createDocument('venue_srv', { + name: compiledSrv.package.name, + 'created': (new Date()).toISOString(), + package: compiledSrv.package, + schemas: compiledSrv.schemas, + endpoints: compiledSrv.endpoints, + midddleware: compiledSrv.middleware, + middleware_order: compiledSrv.middleware_order + }) + + debug('service created') + } + else{ + + + try{ + + debug('updating service') + debug(projectDoc.data) + await projectDoc.mergeData({ + package: compiledSrv.package, + schemas: compiledSrv.schemas, + endpoints: compiledSrv.endpoints, + midddleware: compiledSrv.middleware, + middleware_order: compiledSrv.middleware_order + }) + + //debug(projectDoc.data) + + debug('saving doc') + + + await projectDoc.save() + } + catch(err){ + console.log(err) + } + debug('updated service') + }*/ + + return {done: true} + + //return {srv:projectDoc.data} + } +} \ No newline at end of file diff --git a/src/venue/schema/venue-project.js b/src/venue/schema/venue-project.js deleted file mode 100644 index 9166252..0000000 --- a/src/venue/schema/venue-project.js +++ /dev/null @@ -1,83 +0,0 @@ -'use strict' - -const debug = require('debug')('venue.venue_project') - -const ISchema = require('../../bouncer/ischema') - -const Utils = ISchema.Utils - - -class VenueProject extends ISchema { - - static get Type () { return 'venue_project' } - - static get Schema(){ - return { - owner: {type: String, required: true, index: true}, //public_key.key.hash - created: {type: Number, required: true}, - changed: {type: Number}, - venue: {type: String}, - name: {type: String}, - domain: {type: String, index: true, unique: true}, - - // i2p enable? - // standalone hosting? - - workspace: {type: String, required: true}, - //hash: {type: String, required: true, index: true}, - party: [{ - name: String, - type: {type: String, enum: ['TingoParty', 'LokiParty', 'PeerParty', 'MongoParty']}, - tingo: { - path: String - }, - loki: { - path: String - }, - peer: { - venue: String, - remoteIdentity: String - }, - key: { - hash: String, - securePrivate: String - }, - settings: { - noCache: Boolean, - }, - defaultConfig: Object - }], - routes: [{ - prefix: String, - party: String, - package: { - owner: String, - name: {type: String, required: true, index: true}, - version: String, - branch: String, - hash: String, - }, - settings: { - sendFullErrors: {type: Boolean, required: true}, - useNative: {type: Boolean, required: true}, - } - }] - } - } - - static setupSchema(schema){ - //schema.index({ 'package.name': 1 }, {unique: true}) - return schema - } - - static permissions (context) { - return { - read: false, - new: false, - change: false - } - } -} - - -module.exports = VenueProject \ No newline at end of file diff --git a/src/venue/schema/venue_project.js b/src/venue/schema/venue_project.js new file mode 100644 index 0000000..f96d08e --- /dev/null +++ b/src/venue/schema/venue_project.js @@ -0,0 +1,99 @@ +'use strict' + +const debug = require('debug')('venue.venue_project') + +const ISchema = require('../../bouncer/ischema') + +const Utils = ISchema.Utils + + +class VenueProject extends ISchema { + + static get Type () { return 'venue_project' } + + static get Schema(){ + return { + owner: {type: String, required: true}, + created: {type: Number, required: true}, + workspace: {type: String, required: true}, + hash: {type: String, required: true, index: true}, + + enabled: {type: Boolean}, + + + project: { + owner: {type: String, required: true, index: true}, //public_key.key.hash + created: {type: Number, required: true}, + changed: {type: Number}, + + name: {type: String, required: true}, + version: {type: String, required: true}, + venue: {type: String}, + domain: {type: String, index: true, unique: true}, + + i2p: { + address: String, + public: String, + securePrivate: String + }, + + party: [{ + name: String, + type: {type: String, enum: ['tingo', 'loki', 'peer', 'mongo']}, + tingo: { + path: String + }, + loki: { + path: String + }, + peer: { + venue: String, + remoteIdentity: String + }, + key: { + hash: String, + securePrivate: String + }, + settings: { + noCache: Boolean, + }, + defaultConfig: String + }], + routes: [{ + prefix: String, + party: String, + package: { + owner: String, + name: {type: String, required: true}, + version: String, + branch: String, + hash: String, + }, + settings: { + sendFullErrors: {type: Boolean, required: true}, + useNative: {type: Boolean, required: true}, + } + }], + files: Object + }, + signatures: {type: Object, required: true} + + } + } + + static setupSchema(schema){ + //schema.index({ 'package.name': 1 }, {unique: true}) + return schema + } + + static permissions (context) { + return { + read: false, + new: false, + change: false + } + } +} + + +module.exports = VenueProject \ No newline at end of file diff --git a/src/venue/venue-service.js b/src/venue/venue-service.js index a304e47..815ea87 100644 --- a/src/venue/venue-service.js +++ b/src/venue/venue-service.js @@ -15,6 +15,7 @@ class VenueService extends DatapartySrv.IService { //builder.addSchema(Path.join(__dirname, './schema/public-key.js')) //builder.addSchema(Path.join(__dirname, './schema/session-key.js')) builder.addSchema(Path.join(__dirname, './schema/venue_package.js')) + builder.addSchema(Path.join(__dirname, './schema/venue_project.js')) builder.addSchema(DatapartySrv.schema_paths.public_key) builder.addSchema(DatapartySrv.schema_paths.session_key) @@ -35,6 +36,7 @@ class VenueService extends DatapartySrv.IService { //builder.addEndpoint(Path.join(__dirname, './endpoints/key-announce.js')) builder.addEndpoint(Path.join(__dirname, './endpoints/create-package.js')) + builder.addEndpoint(Path.join(__dirname, './endpoints/create-project.js')) builder.addTask(Path.join(__dirname,'./tasks/cleanup-ephemeral-sessions.js')) From 45cfb172f5e5c92a4c0c6be4672e941e0e7c6b29 Mon Sep 17 00:00:00 2001 From: sevenbitbyte Date: Sat, 13 Jun 2026 02:18:48 +0000 Subject: [PATCH 20/49] install i2p script ubuntu --- scripts/install-i2p-ubuntu-24.sh | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100755 scripts/install-i2p-ubuntu-24.sh diff --git a/scripts/install-i2p-ubuntu-24.sh b/scripts/install-i2p-ubuntu-24.sh new file mode 100755 index 0000000..55019f1 --- /dev/null +++ b/scripts/install-i2p-ubuntu-24.sh @@ -0,0 +1,6 @@ +#!/bin/bash + +sudo add-apt-repository ppa:purplei2p/i2pd +sudo apt update +sudo apt install --yes i2pd + From 2fa54e801678eabfeb59ceb14585772c640288f7 Mon Sep 17 00:00:00 2001 From: sevenbitbyte Date: Sat, 13 Jun 2026 02:46:27 +0000 Subject: [PATCH 21/49] clean up regression in schema generation --- src/service/service-builder.js | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/service/service-builder.js b/src/service/service-builder.js index 8e5e2bb..0598e4c 100644 --- a/src/service/service-builder.js +++ b/src/service/service-builder.js @@ -197,8 +197,8 @@ module.exports = class ServiceBuilder { resultType: 'pointer' }).map(p=>{ - debug('\t\t','p',p) - return p + debug('\t\t','indexed p',p) + return p.replace('/options/index', '').replace('/','') }) //return p.split('.')[1]}) @@ -209,9 +209,10 @@ module.exports = class ServiceBuilder { json: safePaths, resultType: 'pointer' }).map(p=>{ - debug(typeof p) + debug(typeof p, 'unique', p) if(typeof p == 'string'){ - return p.split('.')[1] + let filteredP = p.replace('/options/unique', '').replace('/','') + return filteredP } return p From b68cfb700433ce54b434913f0ecc72ce2e14762f Mon Sep 17 00:00:00 2001 From: sevenbitbyte Date: Sun, 14 Jun 2026 01:58:22 +0000 Subject: [PATCH 22/49] move sig --- src/venue/endpoints/create-project.js | 15 +++++++-------- src/venue/schema/venue_project.js | 6 +++--- 2 files changed, 10 insertions(+), 11 deletions(-) diff --git a/src/venue/endpoints/create-project.js b/src/venue/endpoints/create-project.js index ab0d229..a6b0739 100644 --- a/src/venue/endpoints/create-project.js +++ b/src/venue/endpoints/create-project.js @@ -36,12 +36,12 @@ module.exports = class CreateProjectEndpoint extends IEndpoint { ephemeral_session: true, validate: Joi.object().keys({ project: Joi.object().keys({ - owner: Joi.string(), + owner: Joi.string().required(), created: Joi.number(), changed: Joi.number(), - name: Joi.string(), - venue: Joi.string(), + name: Joi.string().required(), + venue: Joi.string().required(), domain: Joi.string(), i2p: Joi.object().keys({ @@ -51,7 +51,7 @@ module.exports = class CreateProjectEndpoint extends IEndpoint { }), party: Joi.array().items(Joi.object().keys({ name: Joi.string(), - type: Joi.string(), + type: Joi.string().required(), tingo: { path: Joi.string() }, loki: { path: Joi.string() }, peer: { @@ -88,9 +88,9 @@ module.exports = class CreateProjectEndpoint extends IEndpoint { hash: Joi.string().required(), size: Joi.number().required() })) - })) + })), + signatures: Joi.object().pattern(Joi.string(), Joi.string()).required() }).required(), - signatures: Joi.object().pattern(Joi.string(), Joi.string()).required(), staticTar: Joi.any().custom(typedArraySchema) }) }, @@ -213,7 +213,6 @@ module.exports = class CreateProjectEndpoint extends IEndpoint { hash: projectHash, workspace: workspacePath, project: ctx.input.project, - signatures: ctx.input.signatures }) debug('project created') @@ -233,7 +232,7 @@ module.exports = class CreateProjectEndpoint extends IEndpoint { fs.writeFileSync( Path.join(workspacePath, safeFileName+'.project.venue.json'), - JSON.stringify(ctx.input.build, null, 2) + JSON.stringify(ctx.input.project, null, 2) ) // verify build signature diff --git a/src/venue/schema/venue_project.js b/src/venue/schema/venue_project.js index f96d08e..fd610c8 100644 --- a/src/venue/schema/venue_project.js +++ b/src/venue/schema/venue_project.js @@ -74,9 +74,9 @@ class VenueProject extends ISchema { useNative: {type: Boolean, required: true}, } }], - files: Object - }, - signatures: {type: Object, required: true} + files: Object, + signatures: {type: Object, required: true} + } } } From 2159b3b4b0543607218d5f962a97e7e5478ac379 Mon Sep 17 00:00:00 2001 From: sevenbitbyte Date: Sun, 14 Jun 2026 04:06:04 +0000 Subject: [PATCH 23/49] origin router fork --- package.json | 2 +- src/venue/endpoints/create-project.js | 1 + src/venue/schema/venue_project.js | 2 +- 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/package.json b/package.json index 88e0447..671ac7f 100644 --- a/package.json +++ b/package.json @@ -108,7 +108,7 @@ "node-mocks-http": "^1.12.1", "node-object-hash": "^3.0.0", "node-persist": "^3.0.1", - "origin-router": "^1.6.4", + "origin-router": "https://github.com/datapartyjs/origin-router.git", "parse-url": "^5.0.1", "promisfy": "^1.2.0", "roslib": "^1.3.0", diff --git a/src/venue/endpoints/create-project.js b/src/venue/endpoints/create-project.js index a6b0739..2216f9d 100644 --- a/src/venue/endpoints/create-project.js +++ b/src/venue/endpoints/create-project.js @@ -41,6 +41,7 @@ module.exports = class CreateProjectEndpoint extends IEndpoint { changed: Joi.number(), name: Joi.string().required(), + version: Joi.string().required(), venue: Joi.string().required(), domain: Joi.string(), diff --git a/src/venue/schema/venue_project.js b/src/venue/schema/venue_project.js index fd610c8..e7e4f30 100644 --- a/src/venue/schema/venue_project.js +++ b/src/venue/schema/venue_project.js @@ -26,7 +26,7 @@ class VenueProject extends ISchema { created: {type: Number, required: true}, changed: {type: Number}, - name: {type: String, required: true}, + name: {type: String, required: true, index: true}, version: {type: String, required: true}, venue: {type: String}, domain: {type: String, index: true, unique: true}, From 1ee631920cf9b3be2be75ed90a79f7a569fcd083 Mon Sep 17 00:00:00 2001 From: sevenbitbyte Date: Sun, 14 Jun 2026 05:11:47 +0000 Subject: [PATCH 24/49] ephemeral-client --- src/party/index-browser.js | 3 +- src/party/index-embedded.js | 3 +- src/party/index.js | 3 +- src/party/peer/ephemeral-client.js | 223 ++++++++++++++++++++++++ src/venue/bin/commands/project-build.js | 141 +++++++++++++++ src/venue/endpoints/create-project.js | 1 - src/venue/schema/venue_project.js | 4 +- 7 files changed, 372 insertions(+), 6 deletions(-) create mode 100644 src/party/peer/ephemeral-client.js create mode 100644 src/venue/bin/commands/project-build.js diff --git a/src/party/index-browser.js b/src/party/index-browser.js index cf40f10..b084914 100644 --- a/src/party/index-browser.js +++ b/src/party/index-browser.js @@ -7,6 +7,7 @@ const ZangoParty = require('./local/zango-party') const IDocument = require('./idocument') const DocumentFactory = require('./document-factory') const CloudDocument = require('./cloud/cloud-document') +const EphemeralClient = require('./peer/ephemeral-client') const MatchMakerClient = require('./peer/match-maker-client') const LokiDb = require('../bouncer/db/loki-db') @@ -15,5 +16,5 @@ module.exports = { IDocument, IParty, DocumentFactory, CloudDocument, CloudParty, LokiParty, ZangoParty, PeerParty, - LokiDb, MatchMakerClient + LokiDb, EphemeralClient, MatchMakerClient } diff --git a/src/party/index-embedded.js b/src/party/index-embedded.js index 422f737..ba36e58 100644 --- a/src/party/index-embedded.js +++ b/src/party/index-embedded.js @@ -7,11 +7,12 @@ const TingoParty = require('./local/tingo-party') const IDocument = require('./idocument') const DocumentFactory = require('./document-factory') const CloudDocument = require('./cloud/cloud-document') +const EphemeralClient = require('./peer/ephemeral-client') const MatchMakerClient = require('./peer/match-maker-client') module.exports = { IDocument, IParty, DocumentFactory, CloudDocument, CloudParty, LokiParty, PeerParty, - TingoParty, MatchMakerClient + TingoParty, EphemeralClient, MatchMakerClient } \ No newline at end of file diff --git a/src/party/index.js b/src/party/index.js index e362eff..af6f1cc 100644 --- a/src/party/index.js +++ b/src/party/index.js @@ -9,4 +9,5 @@ exports.MongoParty = require('./mongo/mongo-party') exports.IDocument = require('./idocument') exports.DocumentFactory = require('./document-factory') exports.CloudDocument = require('./cloud/cloud-document') -exports.MatchMakerClient = require('./peer/match-maker-client') \ No newline at end of file +exports.EphemeralClient = require('./peer/ephemeral-client') +exports.MatchMakerClient = require('./peer/match-maker-client') diff --git a/src/party/peer/ephemeral-client.js b/src/party/peer/ephemeral-client.js new file mode 100644 index 0000000..9cf9be6 --- /dev/null +++ b/src/party/peer/ephemeral-client.js @@ -0,0 +1,223 @@ +const EventEmitter = require('eventemitter3') + +const debug = require('debug')('dataparty.ephemeral-client') + +const dataparty_crypto = require('@dataparty/crypto') +const LokiParty = require('../local/loki-party') +const PeerParty = require('./peer-party') +const MemoryConfig = require('../../config/memory') +const RestComms = require('../../comms/rest-comms') +const WebsocketComms = require('../../comms/websocket-comms') + +class EphemeralClient extends EventEmitter { + constructor({identity, contacts, urlOrParty = 'https://api.dataparty.xyz/api', wsUrlOrParty = 'wss://api.dataparty.xyz/ws'}){ + + super() + + this.contacts = contacts + this.sessionKey = null + this.identity = identity + this.wsParty = null + this.restParty = null + + if(typeof urlOrParty == 'string'){ + this.restUrl = urlOrParty + this.restParty = null + } else { + this.restParty = urlOrParty + } + + if(typeof wsUrlOrParty == 'string'){ + this.wsUrl = wsUrlOrParty + this.wsParty = null + } else { + this.wsParty = wsUrlOrParty + } + } + + + + async start(){ + this.sessionKey = await dataparty_crypto.Identity.fromRandomSeed({id:'ephemeral-session-key'}) + + if(!this.restParty){ + let config = new MemoryConfig({ + basePath:'ephemeral-client', + cloud: { + uri: this.restUrl + } + }) + + this.restParty = new LokiParty({ + path: 'ephemeral-client', + dbAdapter: new LokiParty.Loki.LokiMemoryAdapter(), + config + }) + + await this.restParty.setIdentity(this.sessionKey) + + debug('starting restParty') + await this.restParty.start() + + if(!this.restParty.comms){ + this.restParty.comms = new RestComms({ + party:this.restParty, + config: this.restParty.config + }) + + this.restParty.comms.sessionId = this.sessionKey.key.hash + } + + await this.announcePublicKeys() + } + + if(!this.wsParty && this.wsUrl){ + this.wsParty = new PeerParty({ + comms: new WebsocketComms({ + uri: this.wsUrl, + discoverRemoteIdentity: false, + remoteIdentity: await this.restParty.comms.getServiceIdentity(), + session: this.sessionKey.key.hash + }), + config: this.restParty.config + }) + + await this.wsParty.start() + + debug('starting wsParty') + await this.wsParty.start() + debug('waiting for websocket authorization') + await this.wsParty.comms.authorized() + } + + } + + + + async announcePublicKeys(type='guest', callPath='key/announce'){ + + let currentActor = this.identity + + const announceData = { + annoucement: { + type, + created: Date.now(), + expiry: Date.now() + 24*60*60*1000, //! Set session expiry to 24hr from now + sessionKey: { + type: this.sessionKey.key.type, + hash: this.sessionKey.key.hash, + public: this.sessionKey.key.public + }, + actorKey: { + type: currentActor.key.type, + hash: currentActor.key.hash, + public: currentActor.key.public + } + }, + trust: { + actorSig: null, + sessionSig: null + } + } + + + const actorSigMsg = await currentActor.sign(announceData.annoucement, true) + const sessionSigMsg = await this.sessionKey.sign(announceData.annoucement, true) + + debug('actorSigMsg', actorSigMsg) + debug('sessionSigMsg', sessionSigMsg) + + announceData.trust.actorSig = dataparty_crypto.Routines.Utils.base64.encode( actorSigMsg.sig ) + announceData.trust.sessionSig = dataparty_crypto.Routines.Utils.base64.encode( sessionSigMsg.sig ) + + debug('announcePublicKeys', announceData) + + const announceResult = await this.restParty.comms.call(callPath, announceData, { + expectClearTextReply: false, + sendClearTextRequest: false, + useSessions: false + }) + + if(announceResult.done != true){ + throw new Error('annoucement request failed - '+callPath) + } + } + + + async lookupPublicKey(hash){ + debug('lookupPublicKey - hash:', hash) + + if(hash == this.identity.key.hash){ + return this.identity + } + + if(this.contacts){ + return await this.contacts.lookupPublicKey(hash) + } + + const lookupData = { hash } + + const lookupResult = await this.wsParty.comms.call('key/lookup', lookupData, { + expectClearTextReply: false, + sendClearTextRequest: false, + useSessions: true + }) + + if(!lookupResult.done){ + return null + } + + debug('lookup result -', lookupResult) + + const identity = new dataparty_crypto.Identity({ + key: lookupResult.public_key + }) + + return identity + } + + async createShortCode(use_limit=3, expiry){ + debug('createShortCode') + + const request = { + use_limit, + expiry: !expiry ? Date.now()+24*60*60*3 : expiry + } + + const result = await this.wsParty.comms.call('short-code/create', request, { + expectClearTextReply: false, + sendClearTextRequest: false, + useSessions: true + }) + + console.log('createShortCode result', result) + + if(!result.done){ + return null + } + + return result.short_code + } + + async lookupPublicKeyByShortCode( code ){ + debug('lookupPublicKeyByShortCode') + + const request = { code } + + const result = await this.wsParty.comms.call('short-code/lookup', request, { + expectClearTextReply: false, + sendClearTextRequest: false, + useSessions: true + }) + + console.log('lookupPublicKeyByShortCode result', result) + + if(!result.done){ + return null + } + + return result.short_code + } +} + +module.exports = EphemeralClient diff --git a/src/venue/bin/commands/project-build.js b/src/venue/bin/commands/project-build.js new file mode 100644 index 0000000..038f218 --- /dev/null +++ b/src/venue/bin/commands/project-build.js @@ -0,0 +1,141 @@ +const CmdTree = require('command-tree') +const Hoek = require('@hapi/hoek') +const debug = require('debug')('venue.project-build') +const Path = require('path') +const OS = require('os') +const fs = require('fs') +const mkdirp = require('mkdirp') + +const prompt = require('prompt') +const argon2 = require('argon2') + +const { execSync } = require('child_process') +const findUp = require('find-up-json').default + + +const Dataparty = require('../../../../') +const dataparty_crypto = require('@dataparty/crypto') +const Joi = require('joi') + +const DEFINITION = { + h: { + description: 'Show help', + alias: 'help', + type: 'help' + }, + o: { + alias: 'output', + type: 'string', + default: Path.join(process.cwd(), 'dist/') + }, + nopassword: { + type: 'boolean', + default: false + }, + identity: { + type: 'string', + description: 'developer release identity', + require: true + }, + name: { + description: 'project name' + }, + version: { + description: 'project version' + }, + remote: { + type: 'string', + description: 'name of remote to build project for' + } +} + + +class VenueProjectBuild extends CmdTree.Command { + constructor(context){ + super({...VenueProjectBuild.Definition, context}) + debug('constructor') + } + + static get Command(){ + return 'project build' + } + + static get Definition(){ + return { + usage: `venue project build [project.json]`, + description: 'Build a project', + definition: DEFINITION + } + } + + async run({parsed}){ + //debug('context -', this.context) + //console.log('parsed -', parsed) + + if (parsed.h) { + throw new CmdTree.Error.HelpRequest('help request') + } + + if (parsed._.length != 3){ + throw new CmdTree.Error.UsageError('You must supply project json') + } + + const keyName = parsed.identity + + const phrase = await this.context.secureConfig.read('identity.'+keyName+'.phrase') + + if(!phrase){ + throw new CmdTree.Error.UsageError("Key doesn't exist!") + } + + const {password} = parsed.nopassword ? {password:null} : await prompt.get({ + properties: { + password: { + message: 'Enter password for identity['+keyName+']', + hidden: true + } + }}) + + let key = await dataparty_crypto.Identity.fromMnemonic(phrase, password, argon2) + + key.id = keyName + + const projectJsonPath = Path.resolve(parsed._[2]) + + let foundUp = findUp('package.json', Path.dirname(serviceClassPath)) + + let pkgJson = foundUp.content + + let projectJson = require( projectJsonPath ) + + + const remoteName = parsed.remote || projectJson.venue + const remote = await this.context.secureConfig.read('remote.'+remoteName) + + if(!remote){ + throw new CmdTree.Error.UsageError('Invalid remote ['+remoteName+']') + } + + const project = { + name: parsed.name ? parsed.name : projectJson.name, + version: parsed.version ? parsed.version : projectJson.version, + owner: key.id, + created: Date.now(), + + venue: remote.identity.key.hash, + domain: projectJson.domain, + + } + + await mkdirp(parsed.output) + + const builder = new Dataparty.ServiceBuilder(service) + const build = await builder.compile(parsed.output, true, key) + + return {files: build.files} + } +} + +module.exports = VenueProjectBuild + + diff --git a/src/venue/endpoints/create-project.js b/src/venue/endpoints/create-project.js index 2216f9d..c763206 100644 --- a/src/venue/endpoints/create-project.js +++ b/src/venue/endpoints/create-project.js @@ -38,7 +38,6 @@ module.exports = class CreateProjectEndpoint extends IEndpoint { project: Joi.object().keys({ owner: Joi.string().required(), created: Joi.number(), - changed: Joi.number(), name: Joi.string().required(), version: Joi.string().required(), diff --git a/src/venue/schema/venue_project.js b/src/venue/schema/venue_project.js index e7e4f30..0dfa83e 100644 --- a/src/venue/schema/venue_project.js +++ b/src/venue/schema/venue_project.js @@ -15,6 +15,7 @@ class VenueProject extends ISchema { return { owner: {type: String, required: true}, created: {type: Number, required: true}, + changed: {type: Number, required: true}, workspace: {type: String, required: true}, hash: {type: String, required: true, index: true}, @@ -24,10 +25,9 @@ class VenueProject extends ISchema { project: { owner: {type: String, required: true, index: true}, //public_key.key.hash created: {type: Number, required: true}, - changed: {type: Number}, name: {type: String, required: true, index: true}, - version: {type: String, required: true}, + version: {type: String, required: true, index: true}, venue: {type: String}, domain: {type: String, index: true, unique: true}, From 9ec28a0bfe7bbc5b5fe415dc7f0e4593c1dec3d3 Mon Sep 17 00:00:00 2001 From: sevenbitbyte Date: Sun, 14 Jun 2026 15:45:16 +0000 Subject: [PATCH 25/49] autoreconnect --- src/comms/isocket-comms.js | 1 + src/comms/peer-comms.js | 10 +- src/comms/rest-comms.js | 2 +- src/comms/websocket-comms.js | 36 ++++++- src/comms/websocket-shim.js | 3 +- src/party/peer/ephemeral-client.js | 77 ++++++++++++++- src/service/endpoints/key-announce.js | 3 +- src/venue/bin/commands/venue-remote-check.js | 99 ++++++++++++++++++++ src/venue/bin/venue.js | 11 ++- 9 files changed, 229 insertions(+), 13 deletions(-) create mode 100644 src/venue/bin/commands/venue-remote-check.js diff --git a/src/comms/isocket-comms.js b/src/comms/isocket-comms.js index 94070ae..cfe90fc 100644 --- a/src/comms/isocket-comms.js +++ b/src/comms/isocket-comms.js @@ -71,6 +71,7 @@ class ISocketComms extends EventEmitter { this.connected = false debug('Server closed connection') this.emit('close') + this.emit('server-close') } onopen(){ diff --git a/src/comms/peer-comms.js b/src/comms/peer-comms.js index b20fd16..65b9438 100644 --- a/src/comms/peer-comms.js +++ b/src/comms/peer-comms.js @@ -54,6 +54,8 @@ class PeerComms extends ISocketComms { this.uuid = uuidv4() this.socket = socket || null + this.stopped = false + //this.auto_reconnect = !socket && !host this.host = host //! Is comms host\ this.oncall = null @@ -207,7 +209,7 @@ class PeerComms extends ISocketComms { await this.socketInit() } - this.socket.on('close', this.stop.bind(this)) + this.socket.on('close', this.socketStop.bind(this)) if(this.host){ debug('host mode comms') @@ -226,7 +228,13 @@ class PeerComms extends ISocketComms { } } + socketStop(){ + debug('socket stop') + this.close() + } + async stop(){ + this.stopped = true debug('stop') this.close() } diff --git a/src/comms/rest-comms.js b/src/comms/rest-comms.js index cac3a67..485e345 100644 --- a/src/comms/rest-comms.js +++ b/src/comms/rest-comms.js @@ -207,7 +207,7 @@ class RestComms extends EventEmitter { const serverIdentity = await RestComms.HttpGet(this.uri + `${this.uriPrefix}identity`) debug('server identity - ', serverIdentity) - this.remoteIdentity = new dataparty_crypto.Identity(serverIdentity) + this.remoteIdentity = dataparty_crypto.Identity.fromJSON(serverIdentity) } return this.remoteIdentity diff --git a/src/comms/websocket-comms.js b/src/comms/websocket-comms.js index d91d690..d606cc2 100644 --- a/src/comms/websocket-comms.js +++ b/src/comms/websocket-comms.js @@ -14,11 +14,12 @@ const WebsocketShim = require('./websocket-shim') * @see https://en.wikipedia.org/wiki/WebSocket */ class WebsocketComms extends PeerComms { - constructor({uri, connection, remoteIdentity, host, party, ...options}){ + constructor({uri, connection, timeout=10000, remoteIdentity, host, party, ...options}){ super({remoteIdentity, host, party, ...options}) this.uri = uri this.connection = connection + this.timeout = timeout debug('starting host=',host, ' uuid=', this.uuid, ' uri=', this.uri) @@ -34,13 +35,44 @@ class WebsocketComms extends PeerComms { async socketInit(){ debug('init') - + let isNewConnection = false + if(!this.host && !this.connection){ debug('opening client connection to',this.uri) this.connection = new WebSocket(this.uri) + + isNewConnection = true } this.socket = new WebsocketShim(this.connection) + + if(isNewConnection){ + + //await new Promise((resolve,reject)=>{ + const timer = setTimeout(() => { + debug('websocket timeout') + this.connection.close() + this.emit('timeout') + //reject(new Error("WebSocket connection timeout")); + }, this.timeout); + + this.socket.once('connect', () => { + debug('websocket opened') + clearTimeout(timer); + //resolve(); + }) + + this.socket.once('error',(error) => { + debug('websocket error', error) + clearTimeout(timer) + this.emit('error', error) + //this.connection.close() + //reject(error); + }) + //}) + } + + } } diff --git a/src/comms/websocket-shim.js b/src/comms/websocket-shim.js index 1f8b2ec..1a6e72a 100644 --- a/src/comms/websocket-shim.js +++ b/src/comms/websocket-shim.js @@ -19,7 +19,7 @@ class WebsocketShim extends EventEmitter { } this.conn.onclose = (event) => { - debug('onclose', event) + debug('onclose', event, event.code, event.reason) this.emit('close', event) } @@ -39,6 +39,7 @@ class WebsocketShim extends EventEmitter { } destroy(){ + if(this.conn && this.conn.terminate) this.conn.terminate() } diff --git a/src/party/peer/ephemeral-client.js b/src/party/peer/ephemeral-client.js index 9cf9be6..c2fd90c 100644 --- a/src/party/peer/ephemeral-client.js +++ b/src/party/peer/ephemeral-client.js @@ -10,15 +10,19 @@ const RestComms = require('../../comms/rest-comms') const WebsocketComms = require('../../comms/websocket-comms') class EphemeralClient extends EventEmitter { - constructor({identity, contacts, urlOrParty = 'https://api.dataparty.xyz/api', wsUrlOrParty = 'wss://api.dataparty.xyz/ws'}){ + constructor({identity, role='guest', autoreconnect=true, contacts, urlOrParty = 'https://api.dataparty.xyz/api', wsUrlOrParty = 'wss://api.dataparty.xyz/ws'}){ super() this.contacts = contacts this.sessionKey = null this.identity = identity + this.role = role || 'guest' this.wsParty = null this.restParty = null + this.autoreconnect = autoreconnect + + this.reconnectTimer = null if(typeof urlOrParty == 'string'){ this.restUrl = urlOrParty @@ -33,6 +37,9 @@ class EphemeralClient extends EventEmitter { } else { this.wsParty = wsUrlOrParty } + + this.reconnect_last_attempt = null + this.reconnect_tries = 0 } @@ -72,6 +79,7 @@ class EphemeralClient extends EventEmitter { } if(!this.wsParty && this.wsUrl){ + this.wsParty = new PeerParty({ comms: new WebsocketComms({ uri: this.wsUrl, @@ -82,6 +90,10 @@ class EphemeralClient extends EventEmitter { config: this.restParty.config }) + this.wsParty.comms.on('server-close', this.handleWsClose.bind(this)) + this.wsParty.comms.on('timeout', this.handleWsClose.bind(this)) + this.wsParty.comms.on('error', this.handleWsClose.bind(this)) + await this.wsParty.start() debug('starting wsParty') @@ -92,15 +104,74 @@ class EphemeralClient extends EventEmitter { } + async handleWsClose(){ + + let stopped = this.wsParty.comms.stopped + + const sleepTime = Math.min(9000*this.reconnect_tries, 20*1000) + debug('ws closed with stopped=',stopped, ' waiting ', sleepTime/1000,'sec') + + if(!this.reconnectTimer){ + this.reconnectTimer = setTimeout( + this.doReconnect.bind(this), + sleepTime + ) + } + } + + async doReconnect(){ + let stopped = this.wsParty.comms.stopped + + this.reconnectTimer = null + + if(stopped || !this.wsParty || !this.autoreconnect){ debug('skip reconnect'); return } + + debug('doing ws reconnect...') + + this.reconnect_last_attempt = Date.now() + this.reconnect_tries++ + + try{ + this.wsParty.comms = new WebsocketComms({ + uri: this.wsUrl, + discoverRemoteIdentity: false, + remoteIdentity: await this.restParty.comms.getServiceIdentity(), + session: this.sessionKey.key.hash + }) + + this.wsParty.comms.party = this.wsParty + + this.wsParty.comms.on('server-close', this.handleWsClose.bind(this)) + this.wsParty.comms.on('timeout', this.handleWsClose.bind(this)) + this.wsParty.comms.on('error', this.handleWsClose.bind(this)) + + debug('restarting websocket') + await this.wsParty.comms.start() + debug('waiting for websocket authorization') + await this.wsParty.comms.authorized() + + + debug('connected and authorized') + + this.reconnect_last_attempt = null + this.reconnect_tries = 0 + + } catch(err){ + debug('reconnect error', err) + + + this.handleWsClose() + } + } - async announcePublicKeys(type='guest', callPath='key/announce'){ + async announcePublicKeys(callPath='key/announce'){ let currentActor = this.identity const announceData = { annoucement: { - type, + role: this.role, created: Date.now(), expiry: Date.now() + 24*60*60*1000, //! Set session expiry to 24hr from now sessionKey: { diff --git a/src/service/endpoints/key-announce.js b/src/service/endpoints/key-announce.js index b2768f5..60e8ffe 100644 --- a/src/service/endpoints/key-announce.js +++ b/src/service/endpoints/key-announce.js @@ -37,6 +37,7 @@ module.exports = class KeyAnnounceEndpoint extends IEndpoint { validate: Joi.object().keys({ annoucement: { + role: Joi.string().valid('guest', 'billing').required(), created: Joi.number().required(), expiry: Joi.number().required(), actorKey: KeyVerifier.required(), @@ -190,7 +191,7 @@ module.exports = class KeyAnnounceEndpoint extends IEndpoint { let keyDoc = await ctx.party.createDocument('public_key', { created: Date.now(), - role: 'guest', + role: ctx.input.annoucement.role || 'guest', owner: computedActorHash, ...inputActorKey }) diff --git a/src/venue/bin/commands/venue-remote-check.js b/src/venue/bin/commands/venue-remote-check.js new file mode 100644 index 0000000..a178bfc --- /dev/null +++ b/src/venue/bin/commands/venue-remote-check.js @@ -0,0 +1,99 @@ +const CmdTree = require('command-tree') +const Hoek = require('@hapi/hoek') +const debug = require('debug')('venue.remote-check') +const Path = require('path') +const OS = require('os') +const fs = require('fs') + +const prompt = require('prompt') +const argon2 = require('argon2') + +const { execSync } = require('child_process') + +const Dataparty = require('../../../../') +const dataparty_crypto = require('@dataparty/crypto') + +const DEFINITION = { + h: { + description: 'Show help', + alias: 'help', + type: 'help' + }, + remote: { + type: 'string', + require: true + }, + identity: { + type: 'string', + description: 'developer release identity', + require: true + } +} + + +class VenueRemoteCheck extends CmdTree.Command { + constructor(context){ + super({...VenueRemoteCheck.Definition, context}) + debug('constructor') + } + + static get Command(){ + return 'remote check' + } + + static get Definition(){ + return { + usage: `venue remote check`, + description: 'Check a remote party', + definition: DEFINITION + } + } + + async run({parsed}){ + //debug('context -', this.context) + //console.log('parsed -', parsed) + + if (parsed.h) { + throw new CmdTree.Error.HelpRequest('help request') + } + + const keyName = parsed.identity + + const phrase = await this.context.secureConfig.read('identity.'+keyName+'.phrase') + + if(!phrase){ + throw new CmdTree.Error.UsageError("Key doesn't exist!") + } + + const {password} = parsed.nopassword ? {password:null} : await prompt.get({ + properties: { + password: { + message: 'Enter password for identity['+keyName+']', + hidden: true + } + }}) + + let key = await dataparty_crypto.Identity.fromMnemonic(phrase, password, argon2) + + key.id = keyName + + const remote = await this.context.secureConfig.read('remote.'+parsed.remote) + + const client = new Dataparty.EphemeralClient({ + identity: key, + urlOrParty: remote.url, + wsUrlOrParty: remote.ws + }) + + await client.start() + console.log('client started') + + this.context.exiting = false + + return {remote, client} + } +} + +module.exports = VenueRemoteCheck + + diff --git a/src/venue/bin/venue.js b/src/venue/bin/venue.js index fc715db..3b4a312 100755 --- a/src/venue/bin/venue.js +++ b/src/venue/bin/venue.js @@ -21,6 +21,7 @@ commandTree.addCommand(require('./commands/venue-identity-show')) commandTree.addCommand(require('./commands/venue-remote-add')) commandTree.addCommand(require('./commands/venue-remote-list')) commandTree.addCommand(require('./commands/venue-remote-show')) +commandTree.addCommand(require('./commands/venue-remote-check')) commandTree.addCommand(require('./commands/pkg-build')) @@ -118,11 +119,13 @@ async function main(){ await secureConfig.waitForUnlocked('startup') - - const output = await commandTree.run({context: { + + context = { secureConfig, collectPassword, - ...context - }}) + exiting: false + } + + const output = await commandTree.run({context}) if(output){ console.log(output) From 4f2f2c80f7e6f49cd1f5111505f8266341f64ddd Mon Sep 17 00:00:00 2001 From: sevenbitbyte Date: Sun, 14 Jun 2026 15:59:31 +0000 Subject: [PATCH 26/49] improve reconnect timer math --- src/comms/websocket-comms.js | 2 +- src/party/peer/ephemeral-client.js | 26 +++++++++++++++++++++++++- 2 files changed, 26 insertions(+), 2 deletions(-) diff --git a/src/comms/websocket-comms.js b/src/comms/websocket-comms.js index d606cc2..8eb5831 100644 --- a/src/comms/websocket-comms.js +++ b/src/comms/websocket-comms.js @@ -14,7 +14,7 @@ const WebsocketShim = require('./websocket-shim') * @see https://en.wikipedia.org/wiki/WebSocket */ class WebsocketComms extends PeerComms { - constructor({uri, connection, timeout=10000, remoteIdentity, host, party, ...options}){ + constructor({uri, connection, timeout=20000, remoteIdentity, host, party, ...options}){ super({remoteIdentity, host, party, ...options}) this.uri = uri diff --git a/src/party/peer/ephemeral-client.js b/src/party/peer/ephemeral-client.js index c2fd90c..7348eb4 100644 --- a/src/party/peer/ephemeral-client.js +++ b/src/party/peer/ephemeral-client.js @@ -9,6 +9,20 @@ const MemoryConfig = require('../../config/memory') const RestComms = require('../../comms/rest-comms') const WebsocketComms = require('../../comms/websocket-comms') +const MAX_RECONNECT_INTERVAL = 120*1000 +const MIN_RECONNECT_INTERVAL = 5*1000 +const MIN_BACKOFF = 3*1000 + +function getReconnectInterval(count, backoff=9000){ + return Math.max( + MIN_RECONNECT_INTERVAL, + Math.min( + MAX_RECONNECT_INTERVAL, + count * Math.max(backoff, MIN_BACKOFF) + ) + ) +} + class EphemeralClient extends EventEmitter { constructor({identity, role='guest', autoreconnect=true, contacts, urlOrParty = 'https://api.dataparty.xyz/api', wsUrlOrParty = 'wss://api.dataparty.xyz/ws'}){ @@ -21,6 +35,7 @@ class EphemeralClient extends EventEmitter { this.wsParty = null this.restParty = null this.autoreconnect = autoreconnect + this.backoff = 9000 this.reconnectTimer = null @@ -100,18 +115,24 @@ class EphemeralClient extends EventEmitter { await this.wsParty.start() debug('waiting for websocket authorization') await this.wsParty.comms.authorized() + this.emit('connected') } } async handleWsClose(){ + this.emit('disconnected') + let stopped = this.wsParty.comms.stopped - const sleepTime = Math.min(9000*this.reconnect_tries, 20*1000) + const sleepTime = getReconnectInterval(this.reconnect_tries, this.backoff) debug('ws closed with stopped=',stopped, ' waiting ', sleepTime/1000,'sec') if(!this.reconnectTimer){ + + this.emit('reconnecting', {sleepTime, wakeTime: Date.now()+sleepTime}) + this.reconnectTimer = setTimeout( this.doReconnect.bind(this), sleepTime @@ -156,6 +177,9 @@ class EphemeralClient extends EventEmitter { this.reconnect_last_attempt = null this.reconnect_tries = 0 + this.emit('connected') + this.emit('reconnected') + } catch(err){ debug('reconnect error', err) From 9fc0dc2f4377954aea5b8ce3d5e814cd0af49599 Mon Sep 17 00:00:00 2001 From: sevenbitbyte Date: Sun, 14 Jun 2026 16:08:53 +0000 Subject: [PATCH 27/49] clean top level signals --- src/party/peer/ephemeral-client.js | 3 ++- src/venue/bin/commands/venue-remote-check.js | 20 ++++++++++++++++++++ 2 files changed, 22 insertions(+), 1 deletion(-) diff --git a/src/party/peer/ephemeral-client.js b/src/party/peer/ephemeral-client.js index 7348eb4..df9ea1e 100644 --- a/src/party/peer/ephemeral-client.js +++ b/src/party/peer/ephemeral-client.js @@ -94,7 +94,7 @@ class EphemeralClient extends EventEmitter { } if(!this.wsParty && this.wsUrl){ - + this.emit('connecting') this.wsParty = new PeerParty({ comms: new WebsocketComms({ uri: this.wsUrl, @@ -153,6 +153,7 @@ class EphemeralClient extends EventEmitter { this.reconnect_tries++ try{ + this.emit('connecting') this.wsParty.comms = new WebsocketComms({ uri: this.wsUrl, discoverRemoteIdentity: false, diff --git a/src/venue/bin/commands/venue-remote-check.js b/src/venue/bin/commands/venue-remote-check.js index a178bfc..edbf0b1 100644 --- a/src/venue/bin/commands/venue-remote-check.js +++ b/src/venue/bin/commands/venue-remote-check.js @@ -85,6 +85,26 @@ class VenueRemoteCheck extends CmdTree.Command { wsUrlOrParty: remote.ws }) + client.on('connecting',()=>{ + console.log('connecting') + }) + + client.on('connected',()=>{ + console.log('connected') + }) + + client.on('disconnected',()=>{ + console.log('disconnected') + }) + + client.on('reconnected',()=>{ + console.log('reconnected') + }) + + client.on('reconnecting',(info)=>{ + console.log('reconnecting',info) + }) + await client.start() console.log('client started') From 68d480075f112e5519baa6bf5cd9376d9fdd0cd2 Mon Sep 17 00:00:00 2001 From: sevenbitbyte Date: Sun, 14 Jun 2026 17:09:09 +0000 Subject: [PATCH 28/49] sane defaults --- src/comms/peer-comms.js | 7 +++ src/comms/websocket-comms.js | 7 +-- src/party/peer/ephemeral-client.js | 47 ++++++++++++++++++-- src/party/peer/peer-party.js | 6 +++ src/venue/bin/commands/venue-remote-check.js | 8 ++++ 5 files changed, 69 insertions(+), 6 deletions(-) diff --git a/src/comms/peer-comms.js b/src/comms/peer-comms.js index 65b9438..4402f66 100644 --- a/src/comms/peer-comms.js +++ b/src/comms/peer-comms.js @@ -55,6 +55,7 @@ class PeerComms extends ISocketComms { this.uuid = uuidv4() this.socket = socket || null this.stopped = false + this.started = false //this.auto_reconnect = !socket && !host this.host = host //! Is comms host\ @@ -205,6 +206,11 @@ class PeerComms extends ISocketComms { async start(){ debug('start') + + if(this.started){ return } + + this.started = true + if(this.socketInit){ await this.socketInit() } @@ -235,6 +241,7 @@ class PeerComms extends ISocketComms { async stop(){ this.stopped = true + this.started = false debug('stop') this.close() } diff --git a/src/comms/websocket-comms.js b/src/comms/websocket-comms.js index 8eb5831..fbfce40 100644 --- a/src/comms/websocket-comms.js +++ b/src/comms/websocket-comms.js @@ -20,6 +20,7 @@ class WebsocketComms extends PeerComms { this.uri = uri this.connection = connection this.timeout = timeout + this.timer = null debug('starting host=',host, ' uuid=', this.uuid, ' uri=', this.uri) @@ -49,7 +50,7 @@ class WebsocketComms extends PeerComms { if(isNewConnection){ //await new Promise((resolve,reject)=>{ - const timer = setTimeout(() => { + this.timer = setTimeout(() => { debug('websocket timeout') this.connection.close() this.emit('timeout') @@ -58,13 +59,13 @@ class WebsocketComms extends PeerComms { this.socket.once('connect', () => { debug('websocket opened') - clearTimeout(timer); + clearTimeout(this.timer); //resolve(); }) this.socket.once('error',(error) => { debug('websocket error', error) - clearTimeout(timer) + clearTimeout(this.timer) this.emit('error', error) //this.connection.close() //reject(error); diff --git a/src/party/peer/ephemeral-client.js b/src/party/peer/ephemeral-client.js index df9ea1e..0c43c5a 100644 --- a/src/party/peer/ephemeral-client.js +++ b/src/party/peer/ephemeral-client.js @@ -13,6 +13,9 @@ const MAX_RECONNECT_INTERVAL = 120*1000 const MIN_RECONNECT_INTERVAL = 5*1000 const MIN_BACKOFF = 3*1000 +const MAX_SESSION_AGE = 24*60*60*1000 //! Set session expiry to 24hr from now +const SESSION_ROLL_AGE = Math.round(MAX_SESSION_AGE * 0.75) + function getReconnectInterval(count, backoff=9000){ return Math.max( MIN_RECONNECT_INTERVAL, @@ -30,6 +33,8 @@ class EphemeralClient extends EventEmitter { this.contacts = contacts this.sessionKey = null + this.sessionExpiry = null + this.sessionTimer = null this.identity = identity this.role = role || 'guest' this.wsParty = null @@ -109,8 +114,6 @@ class EphemeralClient extends EventEmitter { this.wsParty.comms.on('timeout', this.handleWsClose.bind(this)) this.wsParty.comms.on('error', this.handleWsClose.bind(this)) - await this.wsParty.start() - debug('starting wsParty') await this.wsParty.start() debug('waiting for websocket authorization') @@ -120,12 +123,40 @@ class EphemeralClient extends EventEmitter { } + async checkSessionExpiry(){ + if(!this.sessionKey){ return } + + const now = Date.now() + + if(this.sessionExpiry <= now){ + await this.rollSessionKey() + } + } + + async rollSessionKey(){ + + debug('rollSessionKey') + this.emit('session-end', this.sessionKey.key.hash) + + if(this.wsParty){ + await this.wsParty.stop() + } + + this.sessionKey = null + this.restParty = null + this.wsParty = null + + await this.start() + } + async handleWsClose(){ this.emit('disconnected') let stopped = this.wsParty.comms.stopped + if(stopped){ return } + const sleepTime = getReconnectInterval(this.reconnect_tries, this.backoff) debug('ws closed with stopped=',stopped, ' waiting ', sleepTime/1000,'sec') @@ -193,12 +224,20 @@ class EphemeralClient extends EventEmitter { async announcePublicKeys(callPath='key/announce'){ let currentActor = this.identity + + const now = Date.now() + this.sessionExpiry = now + MAX_SESSION_AGE + this.sessionTimer = setTimeout( + this.rollSessionKey.bind(this), + SESSION_ROLL_AGE + ) + const announceData = { annoucement: { role: this.role, created: Date.now(), - expiry: Date.now() + 24*60*60*1000, //! Set session expiry to 24hr from now + expiry: this.sessionExpiry, sessionKey: { type: this.sessionKey.key.type, hash: this.sessionKey.key.hash, @@ -237,6 +276,8 @@ class EphemeralClient extends EventEmitter { if(announceResult.done != true){ throw new Error('annoucement request failed - '+callPath) } + + this.emit('session', this.sessionKey.key.hash) } diff --git a/src/party/peer/peer-party.js b/src/party/peer/peer-party.js index 889286d..0502d61 100644 --- a/src/party/peer/peer-party.js +++ b/src/party/peer/peer-party.js @@ -62,6 +62,12 @@ class PeerParty extends IParty { await this.comms.start() } + async stop(){ + if(this.comms){ + await this.comms.stop() + } + } + async handleCall(ask){ debug('handleCall') diff --git a/src/venue/bin/commands/venue-remote-check.js b/src/venue/bin/commands/venue-remote-check.js index edbf0b1..37b38aa 100644 --- a/src/venue/bin/commands/venue-remote-check.js +++ b/src/venue/bin/commands/venue-remote-check.js @@ -85,6 +85,14 @@ class VenueRemoteCheck extends CmdTree.Command { wsUrlOrParty: remote.ws }) + client.on('session',(id)=>{ + console.log('session', id) + }) + + client.on('session-end',(id)=>{ + console.log('session-end', id) + }) + client.on('connecting',()=>{ console.log('connecting') }) From affd9c891de931a11ea83557a1221b89d24c4ee3 Mon Sep 17 00:00:00 2001 From: sevenbitbyte Date: Sun, 14 Jun 2026 17:18:52 +0000 Subject: [PATCH 29/49] randomize reconnect timing --- src/party/peer/ephemeral-client.js | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/party/peer/ephemeral-client.js b/src/party/peer/ephemeral-client.js index 0c43c5a..38283df 100644 --- a/src/party/peer/ephemeral-client.js +++ b/src/party/peer/ephemeral-client.js @@ -10,8 +10,8 @@ const RestComms = require('../../comms/rest-comms') const WebsocketComms = require('../../comms/websocket-comms') const MAX_RECONNECT_INTERVAL = 120*1000 -const MIN_RECONNECT_INTERVAL = 5*1000 -const MIN_BACKOFF = 3*1000 +const MIN_RECONNECT_INTERVAL = 9*1000 +const MIN_BACKOFF = 9*1000 const MAX_SESSION_AGE = 24*60*60*1000 //! Set session expiry to 24hr from now const SESSION_ROLL_AGE = Math.round(MAX_SESSION_AGE * 0.75) @@ -21,7 +21,7 @@ function getReconnectInterval(count, backoff=9000){ MIN_RECONNECT_INTERVAL, Math.min( MAX_RECONNECT_INTERVAL, - count * Math.max(backoff, MIN_BACKOFF) + Math.round(MIN_BACKOFF + (count * Math.max(backoff, MIN_BACKOFF) * Math.random())) ) ) } @@ -40,7 +40,7 @@ class EphemeralClient extends EventEmitter { this.wsParty = null this.restParty = null this.autoreconnect = autoreconnect - this.backoff = 9000 + this.backoff = MIN_BACKOFF this.reconnectTimer = null From 90b3d9dba57e74124b9e1688982d06ebf652a37c Mon Sep 17 00:00:00 2001 From: sevenbitbyte Date: Sun, 14 Jun 2026 17:50:46 +0000 Subject: [PATCH 30/49] add timing info --- src/party/peer/ephemeral-client.js | 16 ++++++++-------- src/venue/bin/commands/venue-remote-check.js | 16 ++++++++-------- 2 files changed, 16 insertions(+), 16 deletions(-) diff --git a/src/party/peer/ephemeral-client.js b/src/party/peer/ephemeral-client.js index 38283df..6c0c3f5 100644 --- a/src/party/peer/ephemeral-client.js +++ b/src/party/peer/ephemeral-client.js @@ -99,7 +99,7 @@ class EphemeralClient extends EventEmitter { } if(!this.wsParty && this.wsUrl){ - this.emit('connecting') + this.emit('connecting', {time: Date.now()}) this.wsParty = new PeerParty({ comms: new WebsocketComms({ uri: this.wsUrl, @@ -118,7 +118,7 @@ class EphemeralClient extends EventEmitter { await this.wsParty.start() debug('waiting for websocket authorization') await this.wsParty.comms.authorized() - this.emit('connected') + this.emit('connected', {time: Date.now()}) } } @@ -136,7 +136,7 @@ class EphemeralClient extends EventEmitter { async rollSessionKey(){ debug('rollSessionKey') - this.emit('session-end', this.sessionKey.key.hash) + this.emit('session-end', {time: Date.now(), session: this.sessionKey.key.hash}) if(this.wsParty){ await this.wsParty.stop() @@ -151,7 +151,7 @@ class EphemeralClient extends EventEmitter { async handleWsClose(){ - this.emit('disconnected') + this.emit('disconnected', {time: Date.now()}) let stopped = this.wsParty.comms.stopped @@ -184,7 +184,7 @@ class EphemeralClient extends EventEmitter { this.reconnect_tries++ try{ - this.emit('connecting') + this.emit('connecting', {time:Date.now()}) this.wsParty.comms = new WebsocketComms({ uri: this.wsUrl, discoverRemoteIdentity: false, @@ -209,8 +209,8 @@ class EphemeralClient extends EventEmitter { this.reconnect_last_attempt = null this.reconnect_tries = 0 - this.emit('connected') - this.emit('reconnected') + this.emit('connected', {time:Date.now()}) + this.emit('reconnected', {time:Date.now()}) } catch(err){ debug('reconnect error', err) @@ -277,7 +277,7 @@ class EphemeralClient extends EventEmitter { throw new Error('annoucement request failed - '+callPath) } - this.emit('session', this.sessionKey.key.hash) + this.emit('session', {time: Date.now(), session: this.sessionKey.key.hash}) } diff --git a/src/venue/bin/commands/venue-remote-check.js b/src/venue/bin/commands/venue-remote-check.js index 37b38aa..bae2595 100644 --- a/src/venue/bin/commands/venue-remote-check.js +++ b/src/venue/bin/commands/venue-remote-check.js @@ -93,20 +93,20 @@ class VenueRemoteCheck extends CmdTree.Command { console.log('session-end', id) }) - client.on('connecting',()=>{ - console.log('connecting') + client.on('connecting',(info)=>{ + console.log('connecting', info) }) - client.on('connected',()=>{ - console.log('connected') + client.on('connected',(info)=>{ + console.log('connected', info) }) - client.on('disconnected',()=>{ - console.log('disconnected') + client.on('disconnected',(info)=>{ + console.log('disconnected', info) }) - client.on('reconnected',()=>{ - console.log('reconnected') + client.on('reconnected',(info)=>{ + console.log('reconnected', info) }) client.on('reconnecting',(info)=>{ From 68f5446508bcb2bad23a6d11a4e29e214caea0e6 Mon Sep 17 00:00:00 2001 From: sevenbitbyte Date: Sun, 14 Jun 2026 20:39:23 +0000 Subject: [PATCH 31/49] project build --- src/venue/bin/commands/project-build.js | 127 ++++++++++++++++++++++-- src/venue/bin/venue.js | 1 + src/venue/example-project.js | 25 +++++ 3 files changed, 144 insertions(+), 9 deletions(-) create mode 100644 src/venue/example-project.js diff --git a/src/venue/bin/commands/project-build.js b/src/venue/bin/commands/project-build.js index 038f218..0b2aad6 100644 --- a/src/venue/bin/commands/project-build.js +++ b/src/venue/bin/commands/project-build.js @@ -10,13 +10,19 @@ const prompt = require('prompt') const argon2 = require('argon2') const { execSync } = require('child_process') -const findUp = require('find-up-json').default + +const { + globSync +} = require('glob') +const tar = require('tar') const Dataparty = require('../../../../') const dataparty_crypto = require('@dataparty/crypto') const Joi = require('joi') +const {Routines} = dataparty_crypto + const DEFINITION = { h: { description: 'Show help', @@ -50,10 +56,60 @@ const DEFINITION = { } +async function compressFiles(projectName, root, fileList, outputPath, writeFile){ + + if(!fileList){ return } + + let fileMap={} + + let files = fileList.map(file=>{ + // + const content = fs.readFileSync(file) + const hash = dataparty_crypto.Routines.Utils.base64.encode( + dataparty_crypto.Routines.Utils.hash(content) + ) + + fileMap[file] = { hash, size: content.length } + + return hash + }) + + if(!files || files.length < 1){ return } + + const tarFileName = projectName.replace('/', '-')+'.project.files.venue.tgz' + const tarPath = Path.join(outputPath, tarFileName) + + await tar.create({ + cwd: root, + gzip: true, + file: tarPath + }, fileList) + + const staticTar = fs.readFileSync(tarPath) + + let tarHash = dataparty_crypto.Routines.Utils.hash( staticTar ) + let tarHash64 = dataparty_crypto.Routines.Utils.base64.encode(tarHash) + + const fileInfo = { + [tarFileName]: { + tar: tarFileName, + hash:tarHash64, + size: staticTar.length, + files: fileMap + } + } + + return {tarPath, files: fileInfo} +} + + class VenueProjectBuild extends CmdTree.Command { constructor(context){ super({...VenueProjectBuild.Definition, context}) debug('constructor') + + this.project = {} + this.project_sources = [] } static get Command(){ @@ -77,7 +133,7 @@ class VenueProjectBuild extends CmdTree.Command { } if (parsed._.length != 3){ - throw new CmdTree.Error.UsageError('You must supply project json') + throw new CmdTree.Error.UsageError('You must supply project json/js') } const keyName = parsed.identity @@ -102,9 +158,9 @@ class VenueProjectBuild extends CmdTree.Command { const projectJsonPath = Path.resolve(parsed._[2]) - let foundUp = findUp('package.json', Path.dirname(serviceClassPath)) + /*let foundUp = findUp('package.json', Path.dirname(serviceClassPath)) - let pkgJson = foundUp.content + let pkgJson = foundUp.content*/ let projectJson = require( projectJsonPath ) @@ -117,22 +173,75 @@ class VenueProjectBuild extends CmdTree.Command { } const project = { + owner: key.key.hash, + created: Date.now(), + name: parsed.name ? parsed.name : projectJson.name, version: parsed.version ? parsed.version : projectJson.version, - owner: key.id, - created: Date.now(), venue: remote.identity.key.hash, domain: projectJson.domain, + i2p: projectJson.i2p, + party: projectJson.party, + routes: projectJson.routes, + files: projectJson.files + } await mkdirp(parsed.output) - const builder = new Dataparty.ServiceBuilder(service) - const build = await builder.compile(parsed.output, true, key) + const buildOutput = parsed.output+'/'+ project.name.replace('/', '-') +'.project.venue.json' + + let prjFiles = [] + prjFiles.push(buildOutput) + + if(project.files){ + this.addProjectFiles( + Path.dirname(projectJsonPath), + projectJson.files, + { nodir: true, follow: true } + ) + + const {tarPath, files} = await compressFiles(project.name,Path.dirname(projectJsonPath), this.project_sources.files, parsed.output, true) + + project.files = files + + if(tarPath){prjFiles.push(tarPath)} + + + } + + const ownerSig = await key.sign( project, true ) + + project.signatures = { + [key.key.hash]: dataparty_crypto.Routines.Utils.base64.encode(ownerSig.sig) + } + + fs.writeFileSync(buildOutput, JSON.stringify(project, null,2)) + + return {files: prjFiles, project} + } + + + addProjectFiles(root, pattern, options){ + + let result = globSync(pattern, { + dotRelative: true, + cwd:root, + ...options + }) + + if(!this.project_sources.files){ + this.project_sources.files = result + } else { + this.project_sources.files = this.project_sources.files.concat(result) + } + + this.project_sources.files_root = root + + debug('addFiles',result) - return {files: build.files} } } diff --git a/src/venue/bin/venue.js b/src/venue/bin/venue.js index 3b4a312..b405c7a 100755 --- a/src/venue/bin/venue.js +++ b/src/venue/bin/venue.js @@ -24,6 +24,7 @@ commandTree.addCommand(require('./commands/venue-remote-show')) commandTree.addCommand(require('./commands/venue-remote-check')) commandTree.addCommand(require('./commands/pkg-build')) +commandTree.addCommand(require('./commands/project-build')) const HOMEDIR = OS.homedir() const DEFAULT_FOLDER = '.venue' diff --git a/src/venue/example-project.js b/src/venue/example-project.js new file mode 100644 index 0000000..44b93e9 --- /dev/null +++ b/src/venue/example-project.js @@ -0,0 +1,25 @@ +module.exports = { + owner: 'null', + name: 'example-project', + version: '1.0', + + venue: 'dataparty-venue', + domain: 'api.dataparty.xyz', + + routes: [{ + prefix: '/api', + party:'SYSTEM', + package: { + name: '@dataparty/venue' + }, + settings: { + sendFullErrors: false, + useNative: false + } + }], + files: [ + 'public/*', + 'public/dist/dataparty-browser.*', + 'public/node_modules/argon2-browser/dist/*' + ] + } \ No newline at end of file From f3d0e3be025f62637a92dae91026727f548245f1 Mon Sep 17 00:00:00 2001 From: sevenbitbyte Date: Sun, 14 Jun 2026 21:38:00 +0000 Subject: [PATCH 32/49] suport deploy --- src/venue/bin/commands/pkg-build.js | 42 +++++++++++++++++++++++++ src/venue/bin/commands/project-build.js | 32 +++++++++++++++++++ 2 files changed, 74 insertions(+) diff --git a/src/venue/bin/commands/pkg-build.js b/src/venue/bin/commands/pkg-build.js index 11a9db7..5bcaca5 100644 --- a/src/venue/bin/commands/pkg-build.js +++ b/src/venue/bin/commands/pkg-build.js @@ -41,6 +41,14 @@ const DEFINITION = { }, version: { description: 'package version' + }, + remote: { + type: 'string', + description: 'name of remote to build project for' + }, + deploy: { + type: 'boolean', + default: false } } @@ -113,8 +121,42 @@ class VenuePackageBuild extends CmdTree.Command { const builder = new Dataparty.ServiceBuilder(service) const build = await builder.compile(parsed.output, true, key) + + if(parsed.deploy && parsed.remote){ + console.log('uploading...') + const remote = await this.context.secureConfig.read('remote.'+parsed.remote) + let staticTar = undefined + + if(build.files.length == 3){ + staticTar = fs.readFileSync(build.files[ build.files.length - 1 ]) + } + + await this.pushPackage(key, remote, build.build, staticTar) + } + return {files: build.files} } + + + async pushPackage(devId, remote, build, staticTar){ + + let client = new Dataparty.EphemeralClient({ + identity: devId, + urlOrParty: remote.url, + wsUrlOrParty: remote.ws + }) + + await client.start() + + + let uploadResult = await client.restParty.comms.call('create-package', {build, staticTar}, { + expectClearTextReply: false, + sendClearTextRequest: false, + useSessions: true + }) + + console.log('result', uploadResult) + } } module.exports = VenuePackageBuild diff --git a/src/venue/bin/commands/project-build.js b/src/venue/bin/commands/project-build.js index 0b2aad6..3708280 100644 --- a/src/venue/bin/commands/project-build.js +++ b/src/venue/bin/commands/project-build.js @@ -52,6 +52,10 @@ const DEFINITION = { remote: { type: 'string', description: 'name of remote to build project for' + }, + deploy: { + type: 'boolean', + default: false } } @@ -220,6 +224,14 @@ class VenueProjectBuild extends CmdTree.Command { fs.writeFileSync(buildOutput, JSON.stringify(project, null,2)) + let staticTar = undefined + + if(prjFiles.length == 3){ + staticTar = fs.readFileSync(prjFiles[ prjFiles.length - 1 ]) + } + + await this.pushProject(key, remote, project, staticTar) + return {files: prjFiles, project} } @@ -243,6 +255,26 @@ class VenueProjectBuild extends CmdTree.Command { debug('addFiles',result) } + + async pushProject(devId, remote, build, staticTar){ + + let client = new Dataparty.EphemeralClient({ + identity: devId, + urlOrParty: remote.url, + wsUrlOrParty: remote.ws + }) + + await client.start() + + + let uploadResult = await client.restParty.comms.call('create-project', {project:build, staticTar}, { + expectClearTextReply: false, + sendClearTextRequest: false, + useSessions: true + }) + + console.log('result', uploadResult) + } } module.exports = VenueProjectBuild From a393621afbf7464e5d56b81209d503a867ce52a2 Mon Sep 17 00:00:00 2001 From: sevenbitbyte Date: Tue, 16 Jun 2026 22:12:55 +0000 Subject: [PATCH 33/49] clients --- src/party/peer/ephemeral-client.js | 5 + src/party/peer/match-maker-client.js | 283 +++++---------------------- src/party/peer/peer-client.js | 64 ++++++ src/party/peer/peer-invite.js | 41 +++- 4 files changed, 157 insertions(+), 236 deletions(-) create mode 100644 src/party/peer/peer-client.js diff --git a/src/party/peer/ephemeral-client.js b/src/party/peer/ephemeral-client.js index 6c0c3f5..04321c5 100644 --- a/src/party/peer/ephemeral-client.js +++ b/src/party/peer/ephemeral-client.js @@ -65,6 +65,7 @@ class EphemeralClient extends EventEmitter { async start(){ + if(this.sessionKey){ return } this.sessionKey = await dataparty_crypto.Identity.fromRandomSeed({id:'ephemeral-session-key'}) if(!this.restParty){ @@ -123,6 +124,10 @@ class EphemeralClient extends EventEmitter { } + get socketPeerParty(){ + return this.wsParty + } + async checkSessionExpiry(){ if(!this.sessionKey){ return } diff --git a/src/party/peer/match-maker-client.js b/src/party/peer/match-maker-client.js index 7c9b9ff..cc4dee7 100644 --- a/src/party/peer/match-maker-client.js +++ b/src/party/peer/match-maker-client.js @@ -12,30 +12,11 @@ const WebsocketComms = require('../../comms/websocket-comms') const PeerInvite = require('./peer-invite') class MatchMakerClient extends EventEmitter { - constructor(identity, contacts, urlOrParty = 'https://api.dataparty.xyz/api', wsUrlOrParty = 'wss://api.dataparty.xyz/ws', billingIdentity=null){ + constructor(client){ super() - - this.contacts = contacts - this.sessionKey = null - this.identity = identity - this.wsParty = null - this.restParty = null - this.billingIdentity = null - - if(typeof urlOrParty == 'string'){ - this.restUrl = urlOrParty - this.restParty = null - } else { - this.restParty = urlOrParty - } - if(typeof wsUrlOrParty == 'string'){ - this.wsUrl = wsUrlOrParty - this.wsParty = null - } else { - this.wsParty = wsUrlOrParty - } + this.client = client this.invitesTx = null this.invitesRx = null @@ -44,80 +25,55 @@ class MatchMakerClient extends EventEmitter { tx: {}, rx: {} } + + this.started = false + + this.client.on('connected', this.handleConnect.bind(this)) + this.client.on('disconnected', this.handleDisconnect.bind(this)) } + async handleConnect(){ + if(!this.started){ return } + debug('handleConnect') + await this.start() + } - async start(){ - this.sessionKey = await dataparty_crypto.Identity.fromRandomSeed({id:'ephemeral-session-key'}) - - if(!this.restParty){ - let config = new MemoryConfig({ - basePath:'match-maker-client', - cloud: { - uri: this.restUrl - } - }) - - this.restParty = new LokiParty({ - path: 'match-maker-client', - dbAdapter: new LokiParty.Loki.LokiMemoryAdapter(), - config - }) - - await this.restParty.setIdentity(this.sessionKey) - - debug('starting restParty') - await this.restParty.start() - - if(!this.restParty.comms){ - this.restParty.comms = new RestComms({ - party:this.restParty, - config: this.restParty.config - }) - - this.restParty.comms.sessionId = this.sessionKey.key.hash - } + async handleDisconnect(){ + if(!this.started){ return } - await this.announcePublicKeys() - } + debug('handleDisconnect') + this.invitesRx.unsubscribe( this.handleInviteRxMsg.bind(this) ) + this.invitesTx.unsubscribe( this.handleInviteTxMsg.bind(this) ) - if(!this.wsParty && this.wsUrl){ - this.wsParty = new PeerParty({ - comms: new WebsocketComms({ - uri: this.wsUrl, - discoverRemoteIdentity: false, - remoteIdentity: await this.restParty.comms.getServiceIdentity(), - session: this.sessionKey.key.hash - }), - config: this.restParty.config - }) + this.emit('disconnected') + } - await this.wsParty.start() + async start(){ - debug('starting wsParty') - await this.wsParty.start() - debug('waiting for websocket authorization') - await this.wsParty.comms.authorized() + this.started = true + await this.client.start() - this.invitesRx = new this.wsParty.ROSLIB.Topic({ - ros : this.wsParty.comms.ros, - name : '/invites/' + encodeURIComponent(this.identity.key.hash) + '/rx', - messageType: 'Object' - }) + const party = this.client.socketPeerParty - this.invitesRx.subscribe( this.handleInviteRxMsg.bind(this) ) + this.invitesRx = new party.ROSLIB.Topic({ + ros : party.comms.ros, + name : '/invites/' + encodeURIComponent(this.client.identity.key.hash) + '/rx', + messageType: 'Object' + }) + this.invitesRx.subscribe( this.handleInviteRxMsg.bind(this) ) - this.invitesTx = new this.wsParty.ROSLIB.Topic({ - ros : this.wsParty.comms.ros, - name : '/invites/' + encodeURIComponent(this.identity.key.hash) + '/tx', - messageType: 'Object' - }) - this.invitesTx.subscribe( this.handleInviteTxMsg.bind(this) ) - } - + this.invitesTx = new party.ROSLIB.Topic({ + ros : party.comms.ros, + name : '/invites/' + encodeURIComponent(this.client.identity.key.hash) + '/tx', + messageType: 'Object' + }) + + this.invitesTx.subscribe( this.handleInviteTxMsg.bind(this) ) + + this.emit('connected') } async handleInviteRxMsg( msg ){ @@ -127,8 +83,8 @@ class MatchMakerClient extends EventEmitter { if(!this.pendingInvites.rx[inviteId] && msg.invite.state == 'invited'){ - const from = await this.lookupPublicKey(msg.invite.fromHash) - const to = await this.lookupPublicKey(msg.invite.toHash) + const from = await this.client.lookupPublicKey(msg.invite.fromHash) + const to = await this.client.lookupPublicKey(msg.invite.toHash) let invite = new PeerInvite(msg.invite, to, this, from) @@ -161,95 +117,6 @@ class MatchMakerClient extends EventEmitter { } } - async announceBillingKey({stripeCheckoutSession}={}){ - this.announcePublicKeys(true, { - stripe: stripeCheckoutSession - }) - } - - async announcePublicKeys(useBillingKeyAsActor=false, billingMethodDetails=null){ - - let currentActor = useBillingKeyAsActor == true ? this.billingIdentity : this.identity - - const announceData = { - annoucement: { - //type: 'guest',//useBillingKeyAsActor ? 'billing_identity' : 'user_identity', - created: Date.now(), - expiry: Date.now() + 24*60*60*1000, //! Set session expiry to 24hr from now - sessionKey: { - type: this.sessionKey.key.type, - hash: this.sessionKey.key.hash, - public: this.sessionKey.key.public - }, - actorKey: { - type: currentActor.key.type, - hash: currentActor.key.hash, - public: currentActor.key.public - } - }, - trust: { - actorSig: null, - sessionSig: null - } - } - - - const actorSigMsg = await currentActor.sign(announceData.annoucement, true) - const sessionSigMsg = await this.sessionKey.sign(announceData.annoucement, true) - - debug('actorSigMsg', actorSigMsg) - debug('sessionSigMsg', sessionSigMsg) - - announceData.trust.actorSig = dataparty_crypto.Routines.Utils.base64.encode( actorSigMsg.sig ) - announceData.trust.sessionSig = dataparty_crypto.Routines.Utils.base64.encode( sessionSigMsg.sig ) - - debug('announcePublicKeys', announceData) - - let callPath = useBillingKeyAsActor ? 'billing/key/announce' : 'key/announce' - - const announceResult = await this.restParty.comms.call(callPath, announceData, { - expectClearTextReply: false, - sendClearTextRequest: false, - useSessions: false - }) - - if(announceResult.done != true){ - throw new Error('annoucement request failed - '+callPath) - } - } - - - async lookupPublicKey(hash){ - debug('lookupPublicKey - hash:', hash) - - if(hash == this.identity.key.hash){ - return this.identity - } - - if(this.contacts){ - return await this.contacts.lookupPublicKey(hash) - } - - const lookupData = { hash } - - const lookupResult = await this.restParty.comms.call('key/lookup', lookupData, { - expectClearTextReply: false, - sendClearTextRequest: false, - useSessions: true - }) - - if(!lookupResult.done){ - return null - } - - debug('lookup result -', lookupResult) - - const identity = new dataparty_crypto.Identity({ - key: lookupResult.public_key - }) - - return identity - } async createInvite(toHashOrIdentity, {type, service, role, session}, info){ @@ -257,7 +124,7 @@ class MatchMakerClient extends EventEmitter { let toIdentity = null if(typeof toHashOrIdentity == 'string'){ - toIdentity = await this.lookupPublicKey(toHashOrIdentity) + toIdentity = await this.client.lookupPublicKey(toHashOrIdentity) } else { toIdentity = toHashOrIdentity } @@ -269,7 +136,7 @@ class MatchMakerClient extends EventEmitter { service: service ? service : '@dataparty/video-chat', role: role ? role : 'client', timestamp: (new Date()).getTime(), - from: this.identity.key.hash, + from: this.client.identity.key.hash, to: toIdentity.key.hash, session: session ? session : Math.random().toString(36).slice(2), info: info ? info : { @@ -278,17 +145,17 @@ class MatchMakerClient extends EventEmitter { } } - const secureInvite = await this.identity.encrypt(invitePayload, toIdentity) + const secureInvite = await this.client.identity.encrypt(invitePayload, toIdentity) debug('secure-invite', secureInvite) const invitePostData = { to: toIdentity.key.hash, - from: this.identity.key.hash, + from: this.client.identity.key.hash, payload: JSON.stringify(secureInvite.toJSON()) } - const inviteResult = await this.restParty.comms.call('invite/create', invitePostData, { + const inviteResult = await this.client.socketPeerParty.comms.call('invite/create', invitePostData, { expectClearTextReply: false, sendClearTextRequest: false, useSessions: true @@ -298,9 +165,9 @@ class MatchMakerClient extends EventEmitter { if(!inviteDoc){ return } - let invite = new PeerInvite(inviteResult.invite, toIdentity, this, this.identity) + let invite = new PeerInvite(inviteResult.invite, toIdentity, this, this.client.identity, invitePayload) - invite.payload = invitePayload + //invite.payload = invitePayload this.pendingInvites.tx[inviteDoc.$meta.id] = invite @@ -310,16 +177,16 @@ class MatchMakerClient extends EventEmitter { } async lookupInvites({createdAfter, type='to', id, actorHash }){ - let actor = this.identity.key.hash + let actor = this.client.identity.key.hash const lookup = { invite: id, - actor: actorHash ? actorHash : this.identity.key.hash, + actor: actorHash ? actorHash : this.client.identity.key.hash, createdAfter, type: !type ? 'to' : type } - const lookupResult = await this.restParty.comms.call('invite/lookup', lookup, { + const lookupResult = await this.client.socketPeerParty.comms.call('invite/lookup', lookup, { expectClearTextReply: false, sendClearTextRequest: false, useSessions: true @@ -355,8 +222,8 @@ class MatchMakerClient extends EventEmitter { for(let i=0; i < invites.length; i++){ const invite = invites[i] - let to = await this.lookupPublicKey( invite.toHash ) - let from = await this.lookupPublicKey( invite.fromHash ) + let to = await this.client.lookupPublicKey( invite.toHash ) + let from = await this.client.lookupPublicKey( invite.fromHash ) let peerInvite = new PeerInvite( invites[i], to, this, from) @@ -377,14 +244,14 @@ class MatchMakerClient extends EventEmitter { async setInviteState(invite, newState){ debug('setInviteState') - let actor = this.identity.key.hash + let actor = this.client.identity.key.hash const inviteState = { invite: invite.inviteDoc.$meta.id, state: newState } - const inviteStateResult = await this.restParty.comms.call('invite/set-state', inviteState, { + const inviteStateResult = await this.client.socketPeerParty.comms.call('invite/set-state', inviteState, { expectClearTextReply: false, sendClearTextRequest: false, useSessions: true @@ -399,48 +266,6 @@ class MatchMakerClient extends EventEmitter { return inviteStateResult.invite } - async createShortCode(use_limit=3, expiry){ - debug('createShortCode') - - const request = { - use_limit, - expiry: !expiry ? Date.now()+24*60*60*3 : expiry - } - - const result = await this.restParty.comms.call('short-code/create', request, { - expectClearTextReply: false, - sendClearTextRequest: false, - useSessions: true - }) - - console.log('createShortCode result', result) - - if(!result.done){ - return null - } - - return result.short_code - } - - async lookupPublicKeyByShortCode( code ){ - debug('lookupPublicKeyByShortCode') - - const request = { code } - - const result = await this.restParty.comms.call('short-code/lookup', request, { - expectClearTextReply: false, - sendClearTextRequest: false, - useSessions: true - }) - - console.log('lookupPublicKeyByShortCode result', result) - - if(!result.done){ - return null - } - - return result.short_code - } } module.exports = MatchMakerClient diff --git a/src/party/peer/peer-client.js b/src/party/peer/peer-client.js new file mode 100644 index 0000000..d614126 --- /dev/null +++ b/src/party/peer/peer-client.js @@ -0,0 +1,64 @@ +const EphemeralClient = require("./ephemeral-client") + + +class PeerClient extends EphemeralClient { + constructor({config, hostParty, contacts, identity, remoteIdentityHash, matchMaker, service, role='client', rtcSettings}){ + + this.config = config + this.hostParty = hostParty + this.matchMaker = matchMaker + + this.remoteIdentityHash = remoteIdentityHash + + this.inviteSettings = { + type: 'webrtc', + service: service, + role: role ? role : 'client', + session: null + } + + this.rtcSettings = this.rtcSettings + + this.peerParty = null + } + + async start(mediaSrc){ + // + + if(this.sessionKey){ return } + this.sessionKey = await dataparty_crypto.Identity.fromRandomSeed({id:'ephemeral-session-key'}) + + this.inviteSettings.session = this.sessionKey.key.hash + + this.emit('connecting', {time: Date.now()}) + let invite = await this.matchMaker.createInvite(this.remoteIdentityHash, this.inviteSettings) + + await invite.waitForAccepted() + + this.peerParty = await invite.establish({ + mediaSrc, + hostParty: this.hostParty, + config: this.config, + rtcSettings: this.rtcSettings + }) + + this.emit('connected', {time: Date.now()}) + + return this.peerParty + } + + async rollSessionKey(){ + // + } + + async handleClose(){ + // + } + + async doReconnect(){ + // + } + +} + +module.exports = PeerClient diff --git a/src/party/peer/peer-invite.js b/src/party/peer/peer-invite.js index dbd3f8b..f2cb05b 100644 --- a/src/party/peer/peer-invite.js +++ b/src/party/peer/peer-invite.js @@ -8,6 +8,8 @@ const dataparty_crypto = require('@dataparty/crypto') const PeerParty = require('./peer-party') const RTCSocketComms = require('../../comms/rtc-socket-comms') +const DEFAULT_EXPIRY = 5*60*1000 + const END_STATES = [ 'cancelled', 'rejected', 'expired', 'completed' ] @@ -33,7 +35,7 @@ async function delay(ms){ } class PeerInvite extends EventEmitter { - constructor(inviteDoc, toIdentity, matchMakerClient, fromIdentity){ + constructor(inviteDoc, toIdentity, matchMakerClient, fromIdentity, payload=null){ super() this.peerParty = null @@ -42,7 +44,7 @@ class PeerInvite extends EventEmitter { this.matchMaker = matchMakerClient this.inviteDoc = inviteDoc this.inviteMsg = null //this.latestDoc = null - this.payload = null + this.payload = payload this.topicSub = null this.topicPub = null @@ -54,6 +56,18 @@ class PeerInvite extends EventEmitter { this.incomingStream = null + this.timeoutTimer = null + + if(this.payload){ + const expiry = this.payload.timestamp + DEFAULT_EXPIRY + const now = Date.now() + + const delta = expiry - now + if(delta > 0){ + this.timeoutTimer = setTimeout(this.handleTimeout.bind(this)) + } + } + /*if(!this.isSender()){ this.inviteDoc. }*/ @@ -89,7 +103,7 @@ class PeerInvite extends EventEmitter { async accept(mediaSrc, config){ debug('accepting invite') - /*if(this.inviteDoc.toHash == matchMaker.wsParty.identity.key.hash){ + /*if(this.inviteDoc.toHash == this.matchMaker.client.socketPeerParty.identity.key.hash){ otherIdentity = await this.matchMaker.lookupPublicKey(this.inviteDoc.fromHash) } else { otherIdentity = await this.matchMaker.lookupPublicKey(this.inviteDoc.toHash) @@ -102,12 +116,20 @@ class PeerInvite extends EventEmitter { let msgWorkAround = new dataparty_crypto.Message({}) msgWorkAround.fromJSON(JSON.parse(changedInvite.payload)) - let payload = await this.matchMaker.identity.decrypt( + let payload = await this.matchMaker.client.identity.decrypt( msgWorkAround ) this.payload = payload.msg + /*const expiry = this.payload.timestamp + DEFAULT_EXPIRY + const now = Date.now() + + const delta = expiry - now + if(delta > 0 && this.timeoutTimer != null){ + this.timeoutTimer = setTimeout(this.handleTimeout.bind(this)) + }*/ + return await this.establish({mediaSrc, config}) } @@ -120,6 +142,10 @@ class PeerInvite extends EventEmitter { return (this.inviteMsg || this.inviteDoc).state } + async handleTimeout(){ + await this.matchMaker.setInviteState(this, 'expired') + } + async onInviteMsg(inviteMsg){ debug('onInviteMsg', inviteMsg) @@ -168,7 +194,7 @@ class PeerInvite extends EventEmitter { let actorField = this.isSender() ? 'from' : 'to' let otherIdentity = this.isSender() ? this.to : this.from - let party = this.matchMaker.wsParty + let party = this.this.matchMaker.client.socketPeerParty this.topicSub = new party.ROSLIB.Topic({ ros : party.comms.ros, @@ -190,7 +216,7 @@ class PeerInvite extends EventEmitter { let msgWorkAround = new dataparty_crypto.Message({}) msgWorkAround.fromJSON(msg.offers[i]) - let offer = await this.matchMaker.identity.decrypt(msgWorkAround) + let offer = await this.matchMaker.client.identity.decrypt(msgWorkAround) if(offer.from.hash != otherIdentity.key.hash){ debug('BAD IDENTITY') @@ -268,7 +294,7 @@ class PeerInvite extends EventEmitter { debug(' >> offer signal trickle', data) - const secureOffer = await this.matchMaker.identity.encrypt(data, otherIdentity) + const secureOffer = await this.matchMaker.client.identity.encrypt(data, otherIdentity) if(host && !sendFreely){ //console.log('am host') @@ -312,6 +338,7 @@ class PeerInvite extends EventEmitter { } catch (err){ console.log(err) + throw err } } From 270c5e859c20e6efd4174b51f246099a57a94898 Mon Sep 17 00:00:00 2001 From: sevenbitbyte Date: Fri, 19 Jun 2026 20:35:26 +0000 Subject: [PATCH 34/49] cleanup duplicate code --- src/party/peer/ephemeral-client.js | 154 ++---------------- src/party/peer/match-maker-client.js | 15 +- src/party/peer/peer-client.js | 24 ++- src/party/peer/peer-invite.js | 4 +- src/service/index-browser.js | 8 + src/service/index.js | 11 +- src/venue/tasks/cleanup-ephemeral-sessions.js | 86 ---------- src/venue/venue-service.js | 2 +- 8 files changed, 64 insertions(+), 240 deletions(-) delete mode 100644 src/venue/tasks/cleanup-ephemeral-sessions.js diff --git a/src/party/peer/ephemeral-client.js b/src/party/peer/ephemeral-client.js index 04321c5..ce36b7c 100644 --- a/src/party/peer/ephemeral-client.js +++ b/src/party/peer/ephemeral-client.js @@ -9,40 +9,17 @@ const MemoryConfig = require('../../config/memory') const RestComms = require('../../comms/rest-comms') const WebsocketComms = require('../../comms/websocket-comms') -const MAX_RECONNECT_INTERVAL = 120*1000 -const MIN_RECONNECT_INTERVAL = 9*1000 -const MIN_BACKOFF = 9*1000 - -const MAX_SESSION_AGE = 24*60*60*1000 //! Set session expiry to 24hr from now -const SESSION_ROLL_AGE = Math.round(MAX_SESSION_AGE * 0.75) - -function getReconnectInterval(count, backoff=9000){ - return Math.max( - MIN_RECONNECT_INTERVAL, - Math.min( - MAX_RECONNECT_INTERVAL, - Math.round(MIN_BACKOFF + (count * Math.max(backoff, MIN_BACKOFF) * Math.random())) - ) - ) -} - class EphemeralClient extends EventEmitter { - constructor({identity, role='guest', autoreconnect=true, contacts, urlOrParty = 'https://api.dataparty.xyz/api', wsUrlOrParty = 'wss://api.dataparty.xyz/ws'}){ + constructor({identity, role='guest', contacts, urlOrParty = 'https://api.dataparty.xyz/api', wsUrlOrParty = 'wss://api.dataparty.xyz/ws'}){ super() this.contacts = contacts this.sessionKey = null - this.sessionExpiry = null - this.sessionTimer = null this.identity = identity this.role = role || 'guest' this.wsParty = null this.restParty = null - this.autoreconnect = autoreconnect - this.backoff = MIN_BACKOFF - - this.reconnectTimer = null if(typeof urlOrParty == 'string'){ this.restUrl = urlOrParty @@ -57,15 +34,11 @@ class EphemeralClient extends EventEmitter { } else { this.wsParty = wsUrlOrParty } - - this.reconnect_last_attempt = null - this.reconnect_tries = 0 } async start(){ - if(this.sessionKey){ return } this.sessionKey = await dataparty_crypto.Identity.fromRandomSeed({id:'ephemeral-session-key'}) if(!this.restParty){ @@ -100,7 +73,7 @@ class EphemeralClient extends EventEmitter { } if(!this.wsParty && this.wsUrl){ - this.emit('connecting', {time: Date.now()}) + this.wsParty = new PeerParty({ comms: new WebsocketComms({ uri: this.wsUrl, @@ -111,138 +84,33 @@ class EphemeralClient extends EventEmitter { config: this.restParty.config }) - this.wsParty.comms.on('server-close', this.handleWsClose.bind(this)) - this.wsParty.comms.on('timeout', this.handleWsClose.bind(this)) - this.wsParty.comms.on('error', this.handleWsClose.bind(this)) - - debug('starting wsParty') - await this.wsParty.start() - debug('waiting for websocket authorization') - await this.wsParty.comms.authorized() - this.emit('connected', {time: Date.now()}) - } - - } - - get socketPeerParty(){ - return this.wsParty - } + this.wsParty.comms.on('close', ()=>{ - async checkSessionExpiry(){ - if(!this.sessionKey){ return } - - const now = Date.now() - - if(this.sessionExpiry <= now){ - await this.rollSessionKey() - } - } - - async rollSessionKey(){ - - debug('rollSessionKey') - this.emit('session-end', {time: Date.now(), session: this.sessionKey.key.hash}) - - if(this.wsParty){ - await this.wsParty.stop() - } - - this.sessionKey = null - this.restParty = null - this.wsParty = null - - await this.start() - } + let stopped = this.wsParty.comms.stopped + console.log('hey the ws closed - stopped=', stopped) - async handleWsClose(){ - - this.emit('disconnected', {time: Date.now()}) - - let stopped = this.wsParty.comms.stopped - - if(stopped){ return } - - const sleepTime = getReconnectInterval(this.reconnect_tries, this.backoff) - debug('ws closed with stopped=',stopped, ' waiting ', sleepTime/1000,'sec') - - if(!this.reconnectTimer){ - - this.emit('reconnecting', {sleepTime, wakeTime: Date.now()+sleepTime}) - - this.reconnectTimer = setTimeout( - this.doReconnect.bind(this), - sleepTime - ) - } - } - - async doReconnect(){ - let stopped = this.wsParty.comms.stopped - - this.reconnectTimer = null - - if(stopped || !this.wsParty || !this.autoreconnect){ debug('skip reconnect'); return } - - debug('doing ws reconnect...') - - this.reconnect_last_attempt = Date.now() - this.reconnect_tries++ - - try{ - this.emit('connecting', {time:Date.now()}) - this.wsParty.comms = new WebsocketComms({ - uri: this.wsUrl, - discoverRemoteIdentity: false, - remoteIdentity: await this.restParty.comms.getServiceIdentity(), - session: this.sessionKey.key.hash }) - this.wsParty.comms.party = this.wsParty - - this.wsParty.comms.on('server-close', this.handleWsClose.bind(this)) - this.wsParty.comms.on('timeout', this.handleWsClose.bind(this)) - this.wsParty.comms.on('error', this.handleWsClose.bind(this)) + await this.wsParty.start() - debug('restarting websocket') - await this.wsParty.comms.start() + debug('starting wsParty') + await this.wsParty.start() debug('waiting for websocket authorization') await this.wsParty.comms.authorized() - - - debug('connected and authorized') - - this.reconnect_last_attempt = null - this.reconnect_tries = 0 - - this.emit('connected', {time:Date.now()}) - this.emit('reconnected', {time:Date.now()}) - - } catch(err){ - debug('reconnect error', err) - - - this.handleWsClose() } + } async announcePublicKeys(callPath='key/announce'){ let currentActor = this.identity - - const now = Date.now() - this.sessionExpiry = now + MAX_SESSION_AGE - this.sessionTimer = setTimeout( - this.rollSessionKey.bind(this), - SESSION_ROLL_AGE - ) - const announceData = { annoucement: { role: this.role, created: Date.now(), - expiry: this.sessionExpiry, + expiry: Date.now() + 24*60*60*1000, //! Set session expiry to 24hr from now sessionKey: { type: this.sessionKey.key.type, hash: this.sessionKey.key.hash, @@ -281,8 +149,6 @@ class EphemeralClient extends EventEmitter { if(announceResult.done != true){ throw new Error('annoucement request failed - '+callPath) } - - this.emit('session', {time: Date.now(), session: this.sessionKey.key.hash}) } diff --git a/src/party/peer/match-maker-client.js b/src/party/peer/match-maker-client.js index cc4dee7..14dbb99 100644 --- a/src/party/peer/match-maker-client.js +++ b/src/party/peer/match-maker-client.js @@ -28,29 +28,36 @@ class MatchMakerClient extends EventEmitter { this.started = false - this.client.on('connected', this.handleConnect.bind(this)) + this.client.on('reconnected', this.handleReconnect.bind(this)) this.client.on('disconnected', this.handleDisconnect.bind(this)) } - async handleConnect(){ - if(!this.started){ return } + async handleReconnect(){ + //if(!this.started){ return } - debug('handleConnect') + debug('handleReconnect') await this.start() } async handleDisconnect(){ if(!this.started){ return } + thi.started = false + debug('handleDisconnect') this.invitesRx.unsubscribe( this.handleInviteRxMsg.bind(this) ) this.invitesTx.unsubscribe( this.handleInviteTxMsg.bind(this) ) + this.invitesRx = null + this.invitesTx = null + this.emit('disconnected') } async start(){ + if(this.started){ return } + this.started = true await this.client.start() diff --git a/src/party/peer/peer-client.js b/src/party/peer/peer-client.js index d614126..4292ceb 100644 --- a/src/party/peer/peer-client.js +++ b/src/party/peer/peer-client.js @@ -31,7 +31,7 @@ class PeerClient extends EphemeralClient { this.inviteSettings.session = this.sessionKey.key.hash this.emit('connecting', {time: Date.now()}) - let invite = await this.matchMaker.createInvite(this.remoteIdentityHash, this.inviteSettings) + const invite = await this.announcePublicKeys() await invite.waitForAccepted() @@ -48,7 +48,17 @@ class PeerClient extends EphemeralClient { } async rollSessionKey(){ - // + debug('rollSessionKey') + this.emit('session-end', {time: Date.now(), session: this.sessionKey.key.hash}) + + if(this.peerParty){ + await this.peerParty.stop() + } + + this.sessionKey = null + this.peerParty = null + + await this.start() } async handleClose(){ @@ -58,6 +68,16 @@ class PeerClient extends EphemeralClient { async doReconnect(){ // } + + async announcePublicKeys(){ + const announceData = await this.createSessionAnnoucement() + + let invite = await this.matchMaker.createInvite(this.remoteIdentityHash, this.inviteSettings, announceData) + + this.emit('session', {time: Date.now(), session: this.sessionKey.key.hash}) + + return invite + } } diff --git a/src/party/peer/peer-invite.js b/src/party/peer/peer-invite.js index f2cb05b..eeb7ec0 100644 --- a/src/party/peer/peer-invite.js +++ b/src/party/peer/peer-invite.js @@ -100,7 +100,7 @@ class PeerInvite extends EventEmitter { this.emit('done', this) } - async accept(mediaSrc, config){ + async accept(mediaSrc, config, hostParty){ debug('accepting invite') /*if(this.inviteDoc.toHash == this.matchMaker.client.socketPeerParty.identity.key.hash){ @@ -130,7 +130,7 @@ class PeerInvite extends EventEmitter { this.timeoutTimer = setTimeout(this.handleTimeout.bind(this)) }*/ - return await this.establish({mediaSrc, config}) + return await this.establish({mediaSrc, config, hostParty}) } async reject(){ diff --git a/src/service/index-browser.js b/src/service/index-browser.js index e026bf2..d33a83a 100644 --- a/src/service/index-browser.js +++ b/src/service/index-browser.js @@ -50,4 +50,12 @@ exports.endpoint_paths = { secureecho: Path.join(__dirname, './endpoints/secure-echo.js'), identity: Path.join(__dirname, './endpoints/service-identity.js'), version: Path.join(__dirname, './endpoints/service-version.js') +} + +exports.task = { + cleanup_ephemeral_sessions: require('./tasks/cleanup-ephemeral-sessions.js') +} + +exports.task_paths = { + cleanup_ephemeral_sessions: Path.join(__dirname, './tasks/cleanup-ephemeral-sessions.js') } \ No newline at end of file diff --git a/src/service/index.js b/src/service/index.js index 9f00a91..0472ce0 100644 --- a/src/service/index.js +++ b/src/service/index.js @@ -50,7 +50,7 @@ exports.endpoint = { secureecho: require('./endpoints/secure-echo'), identity: require('./endpoints/service-identity'), version: require('./endpoints/service-version'), - key_announce: require('./endpoints/key-announce'), + key_announce: require('./endpoints/key-announce') } exports.endpoint_paths = { @@ -69,4 +69,13 @@ exports.schema = { exports.schema_paths = { public_key: Path.join(__dirname, './schema/public-key.js'), session_key: Path.join(__dirname, './schema/session-key.js') +} + + +exports.task = { + cleanup_ephemeral_sessions: require('./tasks/cleanup-ephemeral-sessions.js') +} + +exports.task_paths = { + cleanup_ephemeral_sessions: Path.join(__dirname, './tasks/cleanup-ephemeral-sessions.js') } \ No newline at end of file diff --git a/src/venue/tasks/cleanup-ephemeral-sessions.js b/src/venue/tasks/cleanup-ephemeral-sessions.js deleted file mode 100644 index 435a48b..0000000 --- a/src/venue/tasks/cleanup-ephemeral-sessions.js +++ /dev/null @@ -1,86 +0,0 @@ -const debug = require('debug')('dataparty.task.cleanup-ephemeral-sessions') - -//const ITask = require('@dataparty/api/src/service/itask') -const ITask = require('../../service/itask') - -class CleanupEphemeralSessionsTask extends ITask { - - constructor(options){ - super({ - name: CleanupEphemeralSessionsTask.name, - background: CleanupEphemeralSessionsTask.Config.background, - ...options - }) - - debug('new') - - this.duration = Math.round(1000*60*15) - this.timeout = null - } - - static get Config(){ - return { - background: true, - autostart: true - } - } - - async exec(){ - - this.setTimer() - - return this.detach() - } - - - async lookupSessions(){ - - let now = Date.now() - - return (await this.context.party.find() - .type('session_key') - .where('expiry').lt(now) - .exec()) - } - - setTimer(){ - this.timeout = setTimeout(this.onTimeout.bind(this), this.duration) - } - - async onTimeout(){ - this.timeout = null - - debug('cleanup ephemeral sessions task') - - try{ - let sessions = await this.lookupSessions() - - if(sessions && sessions.length > 0){ - debug('expired sessions ', sessions.length) - let list = sessions.map(i=>{return i.data}) - await this.context.party.remove(...list) - } - } catch (err){ - debug(err) - } - - this.setTimer() - } - - stop(){ - if(this.timeout !== null){ - clearTimeout(this.timeout) - this.timeout = null - } - } - - static get Name(){ - return 'cleanup-ephemeral-sessions' - } - - static get Description(){ - return 'Cleanup Ephemeral sessions' - } -} - -module.exports = CleanupEphemeralSessionsTask \ No newline at end of file diff --git a/src/venue/venue-service.js b/src/venue/venue-service.js index 815ea87..444c358 100644 --- a/src/venue/venue-service.js +++ b/src/venue/venue-service.js @@ -39,7 +39,7 @@ class VenueService extends DatapartySrv.IService { builder.addEndpoint(Path.join(__dirname, './endpoints/create-project.js')) - builder.addTask(Path.join(__dirname,'./tasks/cleanup-ephemeral-sessions.js')) + builder.addTask(DatapartySrv.task_paths.cleanup_ephemeral_sessions) builder.addAuth(Path.join(__dirname, './auth.js')) From 3ec1a77e7a22417d8361df3d2bcf338aeb098d12 Mon Sep 17 00:00:00 2001 From: sevenbitbyte Date: Fri, 19 Jun 2026 20:40:27 +0000 Subject: [PATCH 35/49] missed cleanup task move and add start of peer client --- src/service/service-host-peer.js | 32 +++++++ .../tasks/cleanup-ephemeral-sessions.js | 86 +++++++++++++++++++ 2 files changed, 118 insertions(+) create mode 100644 src/service/service-host-peer.js create mode 100644 src/service/tasks/cleanup-ephemeral-sessions.js diff --git a/src/service/service-host-peer.js b/src/service/service-host-peer.js new file mode 100644 index 0000000..bebaded --- /dev/null +++ b/src/service/service-host-peer.js @@ -0,0 +1,32 @@ +const debug = require('debug')('dataparty.service.host-peer') + +class ServiceHostPeer { + + constructor({ + runner, + matchMaker, + mediaSrc + + }){ + this.runner = runner + this.matchMaker = matchMaker + + this.mediaSrc = mediaSrc + } + + async start(){ + // + + this.matchMaker.on('invited', this.onInvite.bind(this)) + } + + async onInvite(invite){ + // + + //! check if service wants to allow user + + const peerParty = await invite.accept(this.mediaSrc,this.config) + } +} + +module.exports = ServiceHostPeer \ No newline at end of file diff --git a/src/service/tasks/cleanup-ephemeral-sessions.js b/src/service/tasks/cleanup-ephemeral-sessions.js new file mode 100644 index 0000000..18cb7d0 --- /dev/null +++ b/src/service/tasks/cleanup-ephemeral-sessions.js @@ -0,0 +1,86 @@ +const debug = require('debug')('dataparty.task.cleanup-ephemeral-sessions') + +//const ITask = require('@dataparty/api/src/service/itask') +const ITask = require('../../service/itask') + +class CleanupEphemeralSessionsTask extends ITask { + + constructor(options){ + super({ + name: CleanupEphemeralSessionsTask.name, + background: CleanupEphemeralSessionsTask.Config.background, + ...options + }) + + debug('new') + + this.duration = Math.round(1000*60*15) + this.timeout = null + } + + static get Config(){ + return { + background: true, + autostart: true + } + } + + async exec(){ + + this.setTimer() + + return this.detach() + } + + + async lookupSessions(){ + + let now = Date.now() + + return (await this.context.party.find() + .type('session_key') + .where('expiry').lt(now) + .exec()) + } + + setTimer(){ + this.timeout = setTimeout(this.onTimeout.bind(this), this.duration) + } + + async onTimeout(){ + this.timeout = null + + debug('cleanup ephemeral sessions task') + + try{ + let sessions = await this.lookupSessions() + + if(sessions && sessions.length > 0){ + debug('expired sessions ', sessions.length) + let list = sessions.map(i=>{return i.data}) + await this.context.party.remove(...list) + } + } catch (err){ + debug(err) + } + + this.setTimer() + } + + stop(){ + if(this.timeout !== null){ + clearTimeout(this.timeout) + this.timeout = null + } + } + + static get Name(){ + return 'cleanup-ephemeral-sessions' + } + + static get Description(){ + return 'Cleanup Ephemeral sessions' + } +} + +module.exports = CleanupEphemeralSessionsTask From 8151261fb494e94041a6215fdd22229c789a5b6f Mon Sep 17 00:00:00 2001 From: sevenbitbyte Date: Sat, 20 Jun 2026 16:44:13 +0000 Subject: [PATCH 36/49] improve role logic and access in PeerInvite --- src/party/peer/match-maker-client.js | 6 +++++ src/party/peer/peer-client.js | 7 ++++-- src/party/peer/peer-invite.js | 32 ++++++++++++++++++++------ src/service/runner-router.js | 4 ++-- src/service/service-host-peer.js | 34 +++++++++++++++++++++++++--- src/venue/venue-service.js | 5 ---- 6 files changed, 69 insertions(+), 19 deletions(-) diff --git a/src/party/peer/match-maker-client.js b/src/party/peer/match-maker-client.js index 14dbb99..7847d4e 100644 --- a/src/party/peer/match-maker-client.js +++ b/src/party/peer/match-maker-client.js @@ -127,6 +127,12 @@ class MatchMakerClient extends EventEmitter { async createInvite(toHashOrIdentity, {type, service, role, session}, info){ + const roles = ['client', 'host'] + + if(roles.indexOf(role) == -1){ + throw new Error("Invalid requested role [" + role + "]") + } + debug('createInvite') let toIdentity = null diff --git a/src/party/peer/peer-client.js b/src/party/peer/peer-client.js index 4292ceb..c14bc18 100644 --- a/src/party/peer/peer-client.js +++ b/src/party/peer/peer-client.js @@ -30,6 +30,8 @@ class PeerClient extends EphemeralClient { this.inviteSettings.session = this.sessionKey.key.hash + const role = this.inviteSettings.role + this.emit('connecting', {time: Date.now()}) const invite = await this.announcePublicKeys() @@ -37,8 +39,9 @@ class PeerClient extends EphemeralClient { this.peerParty = await invite.establish({ mediaSrc, - hostParty: this.hostParty, - config: this.config, + role, + hostParty: role == 'host' ? this. this.hostParty : undefined, + model: role == 'client' ? this.hostParty.factory.model : undefined, rtcSettings: this.rtcSettings }) diff --git a/src/party/peer/peer-invite.js b/src/party/peer/peer-invite.js index eeb7ec0..feee206 100644 --- a/src/party/peer/peer-invite.js +++ b/src/party/peer/peer-invite.js @@ -58,7 +58,10 @@ class PeerInvite extends EventEmitter { this.timeoutTimer = null + this.role = null + if(this.payload){ + this._updateRole() const expiry = this.payload.timestamp + DEFAULT_EXPIRY const now = Date.now() @@ -80,6 +83,18 @@ class PeerInvite extends EventEmitter { get to(){ return this.toIdentity } get from(){ return this.fromIdentity } + _updateRole(){ + + if(this.isSender()){ + + this.role = this.payload.role + return + + } + + this.role = this.payload.role == 'client' ? 'host' : 'client' + } + isSender(doc){ if(doc){ @@ -100,7 +115,7 @@ class PeerInvite extends EventEmitter { this.emit('done', this) } - async accept(mediaSrc, config, hostParty){ + async accept({mediaSrc, model, hostParty, hostRunner}){ debug('accepting invite') /*if(this.inviteDoc.toHash == this.matchMaker.client.socketPeerParty.identity.key.hash){ @@ -121,6 +136,7 @@ class PeerInvite extends EventEmitter { ) this.payload = payload.msg + this._updateRole() /*const expiry = this.payload.timestamp + DEFAULT_EXPIRY const now = Date.now() @@ -130,7 +146,7 @@ class PeerInvite extends EventEmitter { this.timeoutTimer = setTimeout(this.handleTimeout.bind(this)) }*/ - return await this.establish({mediaSrc, config, hostParty}) + return await this.establish({mediaSrc, model, hostParty, hostRunner}) } async reject(){ @@ -184,13 +200,13 @@ class PeerInvite extends EventEmitter { }) } - async establish({mediaSrc, hostParty, config, rtcSettings}){ + async establish({mediaSrc, model, hostParty, hostRunner, rtcSettings}){ if(!rtcSettings){ rtcSettings = {} } - let host = this.isSender() + let host = (!this.isSender() && this.role == 'client') || (this.isSender() && this.role == 'host') let actorField = this.isSender() ? 'from' : 'to' let otherIdentity = this.isSender() ? this.to : this.from @@ -242,6 +258,10 @@ class PeerInvite extends EventEmitter { }*/ this.peerParty = new PeerParty({ + hostParty, + hostRunner, + model: hostParty.factory.model, + config: hostParty.config, comms: new RTCSocketComms({ host: this.isSender(), session: this.payload.session, @@ -258,9 +278,7 @@ class PeerInvite extends EventEmitter { trickle: rtcSettings.trickle? rtcSettings.trickle : true, discoverRemoteIdentity: false, remoteIdentity: otherIdentity - }), - hostParty: this.isSender() ? hostParty : undefined, - config: config ? config : hostParty.config + }) }) diff --git a/src/service/runner-router.js b/src/service/runner-router.js index c6897b8..24ae4dc 100644 --- a/src/service/runner-router.js +++ b/src/service/runner-router.js @@ -69,7 +69,7 @@ class RunnerRouter { * @returns {module:Service.ServiceRunner} */ getRunnerByHostIdentity(identity){ - const partyId = identity.toString() + const partyId = typeof identity !== 'string' ? identity.key.hash : identity debug('getRunnerByHostIdentity -', partyId) const runner = this.runnersByHost.get(partyId) @@ -83,7 +83,7 @@ class RunnerRouter { */ addRunner({domain, runner}){ - const partyId = runner.party.identity.toString() + const partyId = runner.party.identity.key.hash debug('addRunner - ', partyId, domain) if(!this.runnersByHost.has(partyId)){ diff --git a/src/service/service-host-peer.js b/src/service/service-host-peer.js index bebaded..01758ab 100644 --- a/src/service/service-host-peer.js +++ b/src/service/service-host-peer.js @@ -21,11 +21,39 @@ class ServiceHostPeer { } async onInvite(invite){ - // - //! check if service wants to allow user - const peerParty = await invite.accept(this.mediaSrc,this.config) + // Filter out none host mode requests + if(invite.role !== 'host'){ + debug('FAIL - unexpected role[', invite.role, '] we expecte to be host') + await invite.reject() + return + } + + let hostRunner = this.runner.party ? this.runner : this.runner.getRunnerByHostIdentity(invite.to) + + // Make sure we know the requested party & runner + if(!hostRunner){ + debug('FAIL - requested party not available', invite.to) + await invite.reject() + return + } + + //! Check if party wants to allow user + if(!(await hostRunner.auth.isSocketConnectionAllowed(invite.from))){ + debug('NOT ALLOWED - user is not allowed', invite.from) + await invite.reject() + return + } + + + let hostParty = hostRunner.party + + const peerParty = await invite.accept({ + media: this.mediaSrc, + hostParty, + hostRunner, + }) } } diff --git a/src/venue/venue-service.js b/src/venue/venue-service.js index 444c358..e98511e 100644 --- a/src/venue/venue-service.js +++ b/src/venue/venue-service.js @@ -12,8 +12,6 @@ class VenueService extends DatapartySrv.IService { let builder = new DatapartySrv.ServiceBuilder(this) - //builder.addSchema(Path.join(__dirname, './schema/public-key.js')) - //builder.addSchema(Path.join(__dirname, './schema/session-key.js')) builder.addSchema(Path.join(__dirname, './schema/venue_package.js')) builder.addSchema(Path.join(__dirname, './schema/venue_project.js')) @@ -24,7 +22,6 @@ class VenueService extends DatapartySrv.IService { builder.addMiddleware(DatapartySrv.middleware_paths.pre.validate) builder.addMiddleware(DatapartySrv.middleware_paths.pre.ephemeral_session) - //builder.addMiddleware(Path.join(__dirname, './middleware/pre/ephemeral-session.js')) builder.addMiddleware(DatapartySrv.middleware_paths.post.validate) builder.addMiddleware(DatapartySrv.middleware_paths.post.encrypt) @@ -33,8 +30,6 @@ class VenueService extends DatapartySrv.IService { builder.addEndpoint(DatapartySrv.endpoint_paths.version) builder.addEndpoint(DatapartySrv.endpoint_paths.key_announce) - //builder.addEndpoint(Path.join(__dirname, './endpoints/key-announce.js')) - builder.addEndpoint(Path.join(__dirname, './endpoints/create-package.js')) builder.addEndpoint(Path.join(__dirname, './endpoints/create-project.js')) From ca50083e1f8529713820a27f0be26dee95049ea3 Mon Sep 17 00:00:00 2001 From: sevenbitbyte Date: Sat, 20 Jun 2026 16:45:15 +0000 Subject: [PATCH 37/49] matchmaker client identity --- src/party/peer/peer-invite.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/party/peer/peer-invite.js b/src/party/peer/peer-invite.js index feee206..1ebe97b 100644 --- a/src/party/peer/peer-invite.js +++ b/src/party/peer/peer-invite.js @@ -98,11 +98,11 @@ class PeerInvite extends EventEmitter { isSender(doc){ if(doc){ - if(doc.toHash == matchMaker.identity.key.hash){return false } + if(doc.toHash == matchMaker.client.identity.key.hash){return false } else { return true } } - if(this.inviteDoc.toHash == matchMaker.identity.key.hash){return false } + if(this.inviteDoc.toHash == matchMaker.client.identity.key.hash){return false } else { return true } } From 32f600582e1faf3a77c0262625a9b35db416b3ae Mon Sep 17 00:00:00 2001 From: sevenbitbyte Date: Sat, 20 Jun 2026 17:13:20 +0000 Subject: [PATCH 38/49] plumbing model --- src/party/peer/ephemeral-client.js | 2 +- src/party/peer/peer-client.js | 12 +++++++++--- src/party/peer/peer-invite.js | 12 ++++++------ src/service/service-host-peer.js | 7 ++++--- 4 files changed, 20 insertions(+), 13 deletions(-) diff --git a/src/party/peer/ephemeral-client.js b/src/party/peer/ephemeral-client.js index ce36b7c..128a4ec 100644 --- a/src/party/peer/ephemeral-client.js +++ b/src/party/peer/ephemeral-client.js @@ -17,7 +17,7 @@ class EphemeralClient extends EventEmitter { this.contacts = contacts this.sessionKey = null this.identity = identity - this.role = role || 'guest' + this.role = role || 'guest' //! todo/note - these are different from invite roles this.wsParty = null this.restParty = null diff --git a/src/party/peer/peer-client.js b/src/party/peer/peer-client.js index c14bc18..fb29fb6 100644 --- a/src/party/peer/peer-client.js +++ b/src/party/peer/peer-client.js @@ -2,9 +2,11 @@ const EphemeralClient = require("./ephemeral-client") class PeerClient extends EphemeralClient { - constructor({config, hostParty, contacts, identity, remoteIdentityHash, matchMaker, service, role='client', rtcSettings}){ + constructor({model=null, /*hostParty=null*/, contacts, identity, remoteIdentityHash, matchMaker, service, role='client', rtcSettings}){ - this.config = config + super({identity, role}) + + this.model = model this.hostParty = hostParty this.matchMaker = matchMaker @@ -20,6 +22,10 @@ class PeerClient extends EphemeralClient { this.rtcSettings = this.rtcSettings this.peerParty = null + + /*if(role == 'host' && hostParty==null){ + throw + }*/ } async start(mediaSrc){ @@ -41,7 +47,7 @@ class PeerClient extends EphemeralClient { mediaSrc, role, hostParty: role == 'host' ? this. this.hostParty : undefined, - model: role == 'client' ? this.hostParty.factory.model : undefined, + model: role == 'host' ? this.hostParty.factory.model : this.model, rtcSettings: this.rtcSettings }) diff --git a/src/party/peer/peer-invite.js b/src/party/peer/peer-invite.js index 1ebe97b..f51c834 100644 --- a/src/party/peer/peer-invite.js +++ b/src/party/peer/peer-invite.js @@ -115,7 +115,7 @@ class PeerInvite extends EventEmitter { this.emit('done', this) } - async accept({mediaSrc, model, hostParty, hostRunner}){ + async accept({mediaSrc, model, hostParty, hostRunner, discoverRemoteIdentity=false}){ debug('accepting invite') /*if(this.inviteDoc.toHash == this.matchMaker.client.socketPeerParty.identity.key.hash){ @@ -146,7 +146,7 @@ class PeerInvite extends EventEmitter { this.timeoutTimer = setTimeout(this.handleTimeout.bind(this)) }*/ - return await this.establish({mediaSrc, model, hostParty, hostRunner}) + return await this.establish({mediaSrc, model, hostParty, hostRunner, discoverRemoteIdentity}) } async reject(){ @@ -200,13 +200,13 @@ class PeerInvite extends EventEmitter { }) } - async establish({mediaSrc, model, hostParty, hostRunner, rtcSettings}){ + async establish({mediaSrc, model, hostParty, hostRunner, rtcSettings, discoverRemoteIdentity=false}){ if(!rtcSettings){ rtcSettings = {} } - let host = (!this.isSender() && this.role == 'client') || (this.isSender() && this.role == 'host') + let host = (this.role == 'host') let actorField = this.isSender() ? 'from' : 'to' let otherIdentity = this.isSender() ? this.to : this.from @@ -276,8 +276,8 @@ class PeerInvite extends EventEmitter { config: DEFAULT_ICE_SERVERS }, trickle: rtcSettings.trickle? rtcSettings.trickle : true, - discoverRemoteIdentity: false, - remoteIdentity: otherIdentity + discoverRemoteIdentity: discoverRemoteIdentity ? discoverRemoteIdentity : false, + remoteIdentity: discoverRemoteIdentity ? undefined: otherIdentity }) }) diff --git a/src/service/service-host-peer.js b/src/service/service-host-peer.js index 01758ab..f455bfe 100644 --- a/src/service/service-host-peer.js +++ b/src/service/service-host-peer.js @@ -5,13 +5,13 @@ class ServiceHostPeer { constructor({ runner, matchMaker, - mediaSrc - + mediaSrc, + discoverRemoteIdentity = false }){ this.runner = runner this.matchMaker = matchMaker - this.mediaSrc = mediaSrc + this.discoverRemoteIdentity = discoverRemoteIdentity } async start(){ @@ -53,6 +53,7 @@ class ServiceHostPeer { media: this.mediaSrc, hostParty, hostRunner, + discoverRemoteIdentity: this.discoverRemoteIdentity }) } } From c116a773824accfd3edfc793764e255b2f7b26ce Mon Sep 17 00:00:00 2001 From: sevenbitbyte Date: Sat, 20 Jun 2026 18:02:24 +0000 Subject: [PATCH 39/49] model, socket, internal calls plumbing --- src/party/document-factory.js | 2 +- src/party/peer/ephemeral-client.js | 51 +++++++++++++++++++++++++-- src/party/peer/peer-client.js | 15 ++++++-- src/party/peer/peer-invite.js | 2 +- src/service/iauth.js | 4 +++ src/service/middleware/pre/decrypt.js | 5 ++- src/service/service-host-peer.js | 12 ++++--- src/service/service-runner-node.js | 33 +++++++++++++++++ 8 files changed, 111 insertions(+), 13 deletions(-) diff --git a/src/party/document-factory.js b/src/party/document-factory.js index ba5c85f..6f7adf8 100644 --- a/src/party/document-factory.js +++ b/src/party/document-factory.js @@ -19,7 +19,7 @@ class DocumentFactory { this.factories = factories || {} this.party = party || null this.ajv = new Ajv() - //this.model = model + this.model = model this.documentClass = documentClass || IDocument this.validators = {} diff --git a/src/party/peer/ephemeral-client.js b/src/party/peer/ephemeral-client.js index 128a4ec..a60f7f3 100644 --- a/src/party/peer/ephemeral-client.js +++ b/src/party/peer/ephemeral-client.js @@ -37,6 +37,14 @@ class EphemeralClient extends EventEmitter { } + get restParty(){ + return this.restParty + } + + get socketParty(){ + return this.wsParty + } + async start(){ this.sessionKey = await dataparty_crypto.Identity.fromRandomSeed({id:'ephemeral-session-key'}) @@ -101,6 +109,43 @@ class EphemeralClient extends EventEmitter { } + async createSessionAnnoucement(){ + let currentActor = this.identity + + const announceData = { + annoucement: { + role: this.role, + created: Date.now(), + expiry: Date.now() + 24*60*60*1000, //! Set session expiry to 24hr from now + sessionKey: { + type: this.sessionKey.key.type, + hash: this.sessionKey.key.hash, + public: this.sessionKey.key.public + }, + actorKey: { + type: currentActor.key.type, + hash: currentActor.key.hash, + public: currentActor.key.public + } + }, + trust: { + actorSig: null, + sessionSig: null + } + } + + + const actorSigMsg = await currentActor.sign(announceData.annoucement, true) + const sessionSigMsg = await this.sessionKey.sign(announceData.annoucement, true) + + debug('actorSigMsg', actorSigMsg) + debug('sessionSigMsg', sessionSigMsg) + + announceData.trust.actorSig = dataparty_crypto.Routines.Utils.base64.encode( actorSigMsg.sig ) + announceData.trust.sessionSig = dataparty_crypto.Routines.Utils.base64.encode( sessionSigMsg.sig ) + + return announceData + } async announcePublicKeys(callPath='key/announce'){ @@ -165,7 +210,7 @@ class EphemeralClient extends EventEmitter { const lookupData = { hash } - const lookupResult = await this.wsParty.comms.call('key/lookup', lookupData, { + const lookupResult = await this.socketParty.comms.call('key/lookup', lookupData, { expectClearTextReply: false, sendClearTextRequest: false, useSessions: true @@ -192,7 +237,7 @@ class EphemeralClient extends EventEmitter { expiry: !expiry ? Date.now()+24*60*60*3 : expiry } - const result = await this.wsParty.comms.call('short-code/create', request, { + const result = await this.socketParty.comms.call('short-code/create', request, { expectClearTextReply: false, sendClearTextRequest: false, useSessions: true @@ -212,7 +257,7 @@ class EphemeralClient extends EventEmitter { const request = { code } - const result = await this.wsParty.comms.call('short-code/lookup', request, { + const result = await this.socketParty.comms.call('short-code/lookup', request, { expectClearTextReply: false, sendClearTextRequest: false, useSessions: true diff --git a/src/party/peer/peer-client.js b/src/party/peer/peer-client.js index fb29fb6..704f6ab 100644 --- a/src/party/peer/peer-client.js +++ b/src/party/peer/peer-client.js @@ -2,9 +2,9 @@ const EphemeralClient = require("./ephemeral-client") class PeerClient extends EphemeralClient { - constructor({model=null, /*hostParty=null*/, contacts, identity, remoteIdentityHash, matchMaker, service, role='client', rtcSettings}){ + constructor({model=null, /*hostParty=null, */ contacts, identity, remoteIdentityHash, matchMaker, service, role='client', rtcSettings}){ - super({identity, role}) + super({identity, contacts, role}) this.model = model this.hostParty = hostParty @@ -12,9 +12,10 @@ class PeerClient extends EphemeralClient { this.remoteIdentityHash = remoteIdentityHash + this.inviteSettings = { type: 'webrtc', - service: service, + service: model ? model.package.name : service, role: role ? role : 'client', session: null } @@ -28,6 +29,14 @@ class PeerClient extends EphemeralClient { }*/ } + get restParty(){ + return this.peerParty + } + + get socketParty(){ + return this.peerParty + } + async start(mediaSrc){ // diff --git a/src/party/peer/peer-invite.js b/src/party/peer/peer-invite.js index f51c834..59bfa9f 100644 --- a/src/party/peer/peer-invite.js +++ b/src/party/peer/peer-invite.js @@ -210,7 +210,7 @@ class PeerInvite extends EventEmitter { let actorField = this.isSender() ? 'from' : 'to' let otherIdentity = this.isSender() ? this.to : this.from - let party = this.this.matchMaker.client.socketPeerParty + let party = this.this.matchMaker.client.socketParty this.topicSub = new party.ROSLIB.Topic({ ros : party.comms.ros, diff --git a/src/service/iauth.js b/src/service/iauth.js index c83ef0e..7201d3f 100644 --- a/src/service/iauth.js +++ b/src/service/iauth.js @@ -37,6 +37,10 @@ module.exports = class IAuth { return identity } + async isPeerConnectionAllowed(identity){ + return true + } + async isSocketConnectionAllowed(identity){ //throw new Error('not implemented') return true diff --git a/src/service/middleware/pre/decrypt.js b/src/service/middleware/pre/decrypt.js index 048dd52..195b521 100644 --- a/src/service/middleware/pre/decrypt.js +++ b/src/service/middleware/pre/decrypt.js @@ -35,7 +35,10 @@ module.exports = class Decrypt extends IMiddleware { if(!context.input || !context.input.enc){ - if(!context.req.source || context.req.source != 'PeerComms'){ + if(!context.req.source || + ( context.req.source != 'PeerComms' && + context.req.source != 'INTERNAL' ) + ){ throw new Error('insecure message -' + context.req.source) } diff --git a/src/service/service-host-peer.js b/src/service/service-host-peer.js index f455bfe..ae67f5d 100644 --- a/src/service/service-host-peer.js +++ b/src/service/service-host-peer.js @@ -39,8 +39,12 @@ class ServiceHostPeer { return } - //! Check if party wants to allow user - if(!(await hostRunner.auth.isSocketConnectionAllowed(invite.from))){ + //! Validate the session annoucement payload + + invite.payload.info + + //! Check if party wants to allow / deny invite sender + if(!(await hostRunner.auth.isPeerConnectionAllowed(invite.from))){ debug('NOT ALLOWED - user is not allowed', invite.from) await invite.reject() return @@ -50,10 +54,10 @@ class ServiceHostPeer { let hostParty = hostRunner.party const peerParty = await invite.accept({ - media: this.mediaSrc, + mediaSrc: this.mediaSrc, hostParty, hostRunner, - discoverRemoteIdentity: this.discoverRemoteIdentity + discoverRemoteIdentity: this.discoverRemoteIdentity //! todo - this is probably something a party config could reasonably over ride }) } } diff --git a/src/service/service-runner-node.js b/src/service/service-runner-node.js index b251c2e..cb15e11 100644 --- a/src/service/service-runner-node.js +++ b/src/service/service-runner-node.js @@ -5,6 +5,7 @@ const Debug = require('debug') const debug = Debug('dataparty.service.runner-node') const EndpointContext = require('./endpoint-context') const DeltaTime = require('../utils/delta-time') +const HttpMocks = require('node-mocks-http') const Router = require('origin-router').Router const Runner = require('@dataparty/tasker').Runner @@ -446,6 +447,38 @@ class ServiceRunnerNode { } } + async internalRequest(endpoint, data){ + let bodyValue = data + + const req = HttpMocks.createRequest({ + method: 'GET', + url: '/'+endpoint, + body: bodyValue + }) + + const res = HttpMocks.createResponse() + + debug('\tthe request', req) + + debug('req ip type', typeof req.ip) + + const route = this.router.get(endpoint) + + debug('route',route) + + req.runner = this + req.source = 'INTERNAL' + + debug('call route', await route._events.route({ + method: req.method, + pathname: req.url, + request: req, + response: res + })) + + return {result: res._getData() } + } + async getTopic(path){ debug('looking up topic', path) From fc84be8f11db359cbf41b908e6e50f74ec63a7f9 Mon Sep 17 00:00:00 2001 From: sevenbitbyte Date: Sat, 20 Jun 2026 19:07:52 +0000 Subject: [PATCH 40/49] plumb access controls better --- src/service/endpoints/key-announce.js | 21 ++++++++++++++------- src/service/iauth.js | 1 + src/service/middleware/post/encrypt.js | 5 ++++- src/service/service-host-peer.js | 25 ++++++++++++++++++++----- src/venue/endpoints/create-project.js | 15 ++++++++------- 5 files changed, 47 insertions(+), 20 deletions(-) diff --git a/src/service/endpoints/key-announce.js b/src/service/endpoints/key-announce.js index 60e8ffe..ed89db9 100644 --- a/src/service/endpoints/key-announce.js +++ b/src/service/endpoints/key-announce.js @@ -95,10 +95,16 @@ module.exports = class KeyAnnounceEndpoint extends IEndpoint { } - // ensure sender is connected using session key mentioned in annoucement + // ensure sender is connected using session key mentioned in annoucement OR this is an internal call if(computedSessionHash == inputSessionKey.hash && - inputSessionKey.public.sign == ctx.senderKey.public.sign && - inputSessionKey.public.box == ctx.senderKey.public.box + ( + ctx.req.source == 'INTERNAL' || + //ctx.req.source == 'PeerComms' || + ( + inputSessionKey.public.sign == ctx.senderKey.public.sign && + inputSessionKey.public.box == ctx.senderKey.public.box + ) + ) ){ const actorSigBson = Routines.Utils.base64.decode( ctx.input.trust.actorSig ) @@ -124,10 +130,11 @@ module.exports = class KeyAnnounceEndpoint extends IEndpoint { await actorSigMsg.assertVerified( actorIdentity, true ) await sessionSigMsg.assertVerified( sessionIdentity, true ) - // verify actor is an admin - const isAdmin = await ctx.runner.auth.isAdmin(actorIdentity) - if(!isAdmin){ - ctx.debug('non-admin user') + // verify actor is allowed + const isAllowed = (await ctx.runner.auth.isAdmin(actorIdentity)) || + (await ctx.runner.auth.isSocketConnectionAllowed(actrIdentity)) + if(!isAllowed){ + ctx.debug('non-allowed user') return {done: false} } diff --git a/src/service/iauth.js b/src/service/iauth.js index 7201d3f..5db6e17 100644 --- a/src/service/iauth.js +++ b/src/service/iauth.js @@ -46,6 +46,7 @@ module.exports = class IAuth { return true } + async isInternal(identity){ return false } diff --git a/src/service/middleware/post/encrypt.js b/src/service/middleware/post/encrypt.js index 8645087..2487722 100644 --- a/src/service/middleware/post/encrypt.js +++ b/src/service/middleware/post/encrypt.js @@ -31,7 +31,10 @@ module.exports = class Encrypt extends IMiddleware { if (!Config){ return } - if(ctx.req.source && ctx.req.source == 'PeerComms'){ + if(!context.req.source && + ( context.req.source == 'PeerComms' || + context.req.source == 'INTERNAL' ) + ){ ctx.setOutput(ctx.output) return } diff --git a/src/service/service-host-peer.js b/src/service/service-host-peer.js index ae67f5d..4aa5f9c 100644 --- a/src/service/service-host-peer.js +++ b/src/service/service-host-peer.js @@ -39,18 +39,33 @@ class ServiceHostPeer { return } - //! Validate the session annoucement payload - - invite.payload.info - - //! Check if party wants to allow / deny invite sender + //! Check if party wants to allow/deny invite sender if(!(await hostRunner.auth.isPeerConnectionAllowed(invite.from))){ + + // verify actor is allowed + const isAllowed = (await ctx.runner.auth.isAdmin(invite.from)) || + (await hostRunner.auth.isPeerConnectionAllowed(invite.from)) || + (await ctx.runner.auth.isSocketConnectionAllowed(invite.from)) + if(!isAllowed){ debug('NOT ALLOWED - user is not allowed', invite.from) await invite.reject() return } + //! Announce the session key + const annoucement = invite.payload.info + const result = await hostRunner.internalRequest('key/announce', annoucement) + + debug('annoucement result', result) + + if(!result || !result.done){ + + debug('user session not allowed') + + return + } + let hostParty = hostRunner.party const peerParty = await invite.accept({ diff --git a/src/venue/endpoints/create-project.js b/src/venue/endpoints/create-project.js index c763206..c4185be 100644 --- a/src/venue/endpoints/create-project.js +++ b/src/venue/endpoints/create-project.js @@ -1,5 +1,6 @@ const fs = require('fs') const Joi = require('joi') +const Path = require('path') const Hoek = require('@hapi/hoek') const {Message, Routines, Identity} = require('@dataparty/crypto') const debug = require('debug')('dataparty.endpoint.create-project') @@ -103,7 +104,7 @@ module.exports = class CreateProjectEndpoint extends IEndpoint { static async run(ctx){ - if(ctx.party.identity.key.hash != ctx.input.project.hash){ + if(ctx.party.identity.key.hash != ctx.input.project.venue){ throw new Error('project venue does not match this host') } @@ -176,7 +177,7 @@ module.exports = class CreateProjectEndpoint extends IEndpoint { debug('\t'+'hash', projectHash) - const projectWorkspace = 'projects'+safeFileName+'/'+ctx.input.project.version+'/'+safeProjectHash + const projectWorkspace = Path.join('projects/',safeFileName, ctx.input.project.version, safeProjectHash) const config = ctx.party.config @@ -195,11 +196,11 @@ module.exports = class CreateProjectEndpoint extends IEndpoint { debug('addProject', projectId) let projectDoc = (await ctx.party.find() - .type('venue_project') - .where('project.name').equals(project.name) - .where('project.version').equals(project.version) - .where('hash').equals(projectHash) - .exec())[0] + .type('venue_project') + .where('project.name').equals(project.name) + .where('project.version').equals(project.version) + .where('hash').equals(projectHash) + .exec())[0] if(!projectDoc){ From eea837515b93f294d1bad12d40d72b712f387b52 Mon Sep 17 00:00:00 2001 From: sevenbitbyte Date: Sat, 20 Jun 2026 20:45:40 +0000 Subject: [PATCH 41/49] project staticTar handling --- src/comms/rest-comms.js | 14 ++++++++++- src/party/peer/ephemeral-client.js | 4 ++-- src/service/middleware/post/encrypt.js | 6 ++--- src/venue/bin/commands/project-build.js | 5 ++-- src/venue/bin/venue.js | 2 +- src/venue/endpoints/create-project.js | 32 ++++++++++++++++--------- src/venue/schema/venue_project.js | 2 +- 7 files changed, 44 insertions(+), 21 deletions(-) diff --git a/src/comms/rest-comms.js b/src/comms/rest-comms.js index 485e345..be11241 100644 --- a/src/comms/rest-comms.js +++ b/src/comms/rest-comms.js @@ -142,7 +142,19 @@ class RestComms extends EventEmitter { // debug('raw reply ->', reply) } catch (error) { debug('rest', fullPath, ' call fail ->', error.message) - throw new Error('RestCommsError') + + console.log(Object.keys(error), Object.keys(error.response)) + + const simpleError = { + name: error.name, + code: error.code, + //message: error.message, + statusCode: error.response.statusCode, + statusMessage: error.response.statusMessage, + data: error.response.data + } + + throw simpleError } const msg = await this.party.decrypt( diff --git a/src/party/peer/ephemeral-client.js b/src/party/peer/ephemeral-client.js index a60f7f3..8babd30 100644 --- a/src/party/peer/ephemeral-client.js +++ b/src/party/peer/ephemeral-client.js @@ -37,9 +37,9 @@ class EphemeralClient extends EventEmitter { } - get restParty(){ + /*get restParty(){ return this.restParty - } + }*/ get socketParty(){ return this.wsParty diff --git a/src/service/middleware/post/encrypt.js b/src/service/middleware/post/encrypt.js index 2487722..09dbe3d 100644 --- a/src/service/middleware/post/encrypt.js +++ b/src/service/middleware/post/encrypt.js @@ -31,9 +31,9 @@ module.exports = class Encrypt extends IMiddleware { if (!Config){ return } - if(!context.req.source && - ( context.req.source == 'PeerComms' || - context.req.source == 'INTERNAL' ) + if(!ctx.req.source && + ( ctx.req.source == 'PeerComms' || + ctx.req.source == 'INTERNAL' ) ){ ctx.setOutput(ctx.output) return diff --git a/src/venue/bin/commands/project-build.js b/src/venue/bin/commands/project-build.js index 3708280..5d62c74 100644 --- a/src/venue/bin/commands/project-build.js +++ b/src/venue/bin/commands/project-build.js @@ -178,7 +178,7 @@ class VenueProjectBuild extends CmdTree.Command { const project = { owner: key.key.hash, - created: Date.now(), + //created: Date.now(), name: parsed.name ? parsed.name : projectJson.name, version: parsed.version ? parsed.version : projectJson.version, @@ -197,6 +197,7 @@ class VenueProjectBuild extends CmdTree.Command { const buildOutput = parsed.output+'/'+ project.name.replace('/', '-') +'.project.venue.json' + let prjFiles = [] prjFiles.push(buildOutput) @@ -226,7 +227,7 @@ class VenueProjectBuild extends CmdTree.Command { let staticTar = undefined - if(prjFiles.length == 3){ + if(prjFiles.length == 2){ staticTar = fs.readFileSync(prjFiles[ prjFiles.length - 1 ]) } diff --git a/src/venue/bin/venue.js b/src/venue/bin/venue.js index b405c7a..c83b6c0 100755 --- a/src/venue/bin/venue.js +++ b/src/venue/bin/venue.js @@ -123,7 +123,7 @@ async function main(){ context = { secureConfig, collectPassword, - exiting: false + exiting: true } const output = await commandTree.run({context}) diff --git a/src/venue/endpoints/create-project.js b/src/venue/endpoints/create-project.js index c4185be..c81d3f5 100644 --- a/src/venue/endpoints/create-project.js +++ b/src/venue/endpoints/create-project.js @@ -38,7 +38,7 @@ module.exports = class CreateProjectEndpoint extends IEndpoint { validate: Joi.object().keys({ project: Joi.object().keys({ owner: Joi.string().required(), - created: Joi.number(), + //created: Joi.number(), name: Joi.string().required(), version: Joi.string().required(), @@ -149,19 +149,27 @@ module.exports = class CreateProjectEndpoint extends IEndpoint { debug('verified package signature') - const tarHash = Routines.Utils.hash(ctx.input.staticTar) - const tarHash64 = Routines.Utils.base64.encode( tarHash ) + const safeFileName = ctx.input.project.name.replace('/', '-') const tarFileName = safeFileName+'.files.venue.tgz' const projectFiles = ctx.input.project.files[tarFileName] - if(projectFiles && projectFiles.hash != tarHash64){ - throw new Error("staticTar hash doesn't match project definition") + if(projectFiles && !ctx.input.staticTar){ + throw new Error('project definition lists a static tar but none was uploaded') } - debug('verified staticTar') + if(ctx.input.staticTar){ + const tarHash = Routines.Utils.hash(ctx.input.staticTar) + const tarHash64 = Routines.Utils.base64.encode( tarHash ) + + if(projectFiles && projectFiles.hash != tarHash64){ + throw new Error("staticTar hash doesn't match project definition") + } + + debug('verified staticTar') + } debug('verified project - '+ctx.input.project.name+'@'+ctx.input.project.version) @@ -206,7 +214,7 @@ module.exports = class CreateProjectEndpoint extends IEndpoint { if(!projectDoc){ debug('creating project') - const {owner, ...pkgWithoutOwner} = project.package + const {owner, ...pkgWithoutOwner} = project projectDoc = await ctx.party.createDocument('venue_pkg', { owner: project.owner, @@ -221,10 +229,12 @@ module.exports = class CreateProjectEndpoint extends IEndpoint { debug('need to update project?') } - fs.writeFileSync( - Path.join(workspacePath, tarFileName), - ctx.input.staticTar - ) + if(ctx.input.staticTar){ + fs.writeFileSync( + Path.join(workspacePath, tarFileName), + ctx.input.staticTar + ) + } /*fs.writeFileSync( Path.join(workspacePath, safeFileName+'.service.venue.bson'), diff --git a/src/venue/schema/venue_project.js b/src/venue/schema/venue_project.js index 0dfa83e..03583c4 100644 --- a/src/venue/schema/venue_project.js +++ b/src/venue/schema/venue_project.js @@ -24,7 +24,7 @@ class VenueProject extends ISchema { project: { owner: {type: String, required: true, index: true}, //public_key.key.hash - created: {type: Number, required: true}, + //created: {type: Number, required: true}, name: {type: String, required: true, index: true}, version: {type: String, required: true, index: true}, From 53d043708af988a2e28d534deeeb7ca063c361f4 Mon Sep 17 00:00:00 2001 From: sevenbitbyte Date: Sun, 21 Jun 2026 00:16:23 +0000 Subject: [PATCH 42/49] tar handling in package --- src/venue/endpoints/create-package.js | 35 ++++++++++++++++----------- 1 file changed, 21 insertions(+), 14 deletions(-) diff --git a/src/venue/endpoints/create-package.js b/src/venue/endpoints/create-package.js index 4eedd2d..90b2da1 100644 --- a/src/venue/endpoints/create-package.js +++ b/src/venue/endpoints/create-package.js @@ -1,5 +1,6 @@ const fs = require('fs') const Joi = require('joi') +const Path = require('path') const Hoek = require('@hapi/hoek') const {Message, Routines, Identity} = require('@dataparty/crypto') const debug = require('debug')('dataparty.endpoint.create-package') @@ -184,19 +185,23 @@ module.exports = class CreatePkgEndpoint extends IEndpoint { debug('verified package signature') - const tarHash = Routines.Utils.hash(ctx.input.staticTar) - const tarHash64 = Routines.Utils.base64.encode( tarHash ) - const safeFileName = ctx.input.build.package.name.replace('/', '-') - const tarFileName = safeFileName+'.files.venue.tgz' - const buildFiles = ctx.input.build.files[tarFileName] + if(ctx.input.staticTar){ - if(buildFiles && buildFiles.hash != tarHash64){ - throw new Error("staticTar hash doesn't match package definition") - } + const tarHash = Routines.Utils.hash(ctx.input.staticTar) + const tarHash64 = Routines.Utils.base64.encode( tarHash ) + + const tarFileName = safeFileName+'.files.venue.tgz' - debug('verified staticTar') + const buildFiles = ctx.input.build.files[tarFileName] + + if(buildFiles && buildFiles.hash != tarHash64){ + throw new Error("staticTar hash doesn't match package definition") + } + + debug('verified staticTar') + } debug('verified package - '+ctx.input.build.package.name+'@'+ctx.input.build.package.version) @@ -212,7 +217,7 @@ module.exports = class CreatePkgEndpoint extends IEndpoint { debug('\t'+'hash', buildHash) - const buildWorkspace = 'packages/'+safeFileName+'/'+ctx.input.build.package.version+'/'+safeBuildHash + const buildWorkspace = Path.join('packages', safeFileName, ctx.input.build.package.version, safeBuildHash) const config = ctx.party.config @@ -259,10 +264,12 @@ module.exports = class CreatePkgEndpoint extends IEndpoint { debug('need to update service?') } - fs.writeFileSync( - Path.join(workspacePath, tarFileName), - ctx.input.staticTar - ) + if(ctx.input.staticTar){ + fs.writeFileSync( + Path.join(workspacePath, tarFileName), + ctx.input.staticTar + ) + } /*fs.writeFileSync( Path.join(workspacePath, safeFileName+'.service.venue.bson'), From fde4fb3eb705f632675c220fd8fe97e4c619e686 Mon Sep 17 00:00:00 2001 From: sevenbitbyte Date: Sun, 21 Jun 2026 01:36:58 +0000 Subject: [PATCH 43/49] project flow fixes --- src/service/endpoints/key-announce.js | 2 +- src/venue/bin/commands/venued-host.js | 36 ++++++++++++++++++++++++++- src/venue/endpoints/create-project.js | 7 ++++-- src/venue/schema/venue_project.js | 2 +- 4 files changed, 42 insertions(+), 5 deletions(-) diff --git a/src/service/endpoints/key-announce.js b/src/service/endpoints/key-announce.js index ed89db9..26afb71 100644 --- a/src/service/endpoints/key-announce.js +++ b/src/service/endpoints/key-announce.js @@ -132,7 +132,7 @@ module.exports = class KeyAnnounceEndpoint extends IEndpoint { // verify actor is allowed const isAllowed = (await ctx.runner.auth.isAdmin(actorIdentity)) || - (await ctx.runner.auth.isSocketConnectionAllowed(actrIdentity)) + (await ctx.runner.auth.isSocketConnectionAllowed(actorIdentity)) if(!isAllowed){ ctx.debug('non-allowed user') return {done: false} diff --git a/src/venue/bin/commands/venued-host.js b/src/venue/bin/commands/venued-host.js index 2ef584c..4785d1c 100644 --- a/src/venue/bin/commands/venued-host.js +++ b/src/venue/bin/commands/venued-host.js @@ -274,7 +274,41 @@ class VenuedHost extends CmdTree.Command { console.log('\t', i2pAddress) } - //console.log(Path.join(__dirname,'../public')) + /** + * config section + * + * projects: { + * [name]: latest-hash + * } + */ + + const projects = await config.read('projects') + + if(projects){ + for(let name in projects){ + + const hash = projects[name] + console.log('\tloading project', name, hash) + + + + const project = (await party.find() + .type('venue_project') + .where('hash').equals(hash).exec())[0] + + + + //console.log(project.data) + + /** + * 1. load project doc from db + * 2. validate files exist + * 3. launch project + */ + } + } + + this.context.exiting = false diff --git a/src/venue/endpoints/create-project.js b/src/venue/endpoints/create-project.js index c81d3f5..94f3699 100644 --- a/src/venue/endpoints/create-project.js +++ b/src/venue/endpoints/create-project.js @@ -216,9 +216,10 @@ module.exports = class CreateProjectEndpoint extends IEndpoint { const {owner, ...pkgWithoutOwner} = project - projectDoc = await ctx.party.createDocument('venue_pkg', { + projectDoc = await ctx.party.createDocument('venue_project', { owner: project.owner, created: Date.now(), + changed: Date.now(), hash: projectHash, workspace: workspacePath, project: ctx.input.project, @@ -245,6 +246,8 @@ module.exports = class CreateProjectEndpoint extends IEndpoint { Path.join(workspacePath, safeFileName+'.project.venue.json'), JSON.stringify(ctx.input.project, null, 2) ) + + await ctx.party.config.write('projects.'+ctx.input.project.name, projectHash) // verify build signature @@ -311,7 +314,7 @@ module.exports = class CreateProjectEndpoint extends IEndpoint { debug('updated service') }*/ - return {done: true} + return {done: true, project: projectDoc.data} //return {srv:projectDoc.data} } diff --git a/src/venue/schema/venue_project.js b/src/venue/schema/venue_project.js index 03583c4..77117c4 100644 --- a/src/venue/schema/venue_project.js +++ b/src/venue/schema/venue_project.js @@ -29,7 +29,7 @@ class VenueProject extends ISchema { name: {type: String, required: true, index: true}, version: {type: String, required: true, index: true}, venue: {type: String}, - domain: {type: String, index: true, unique: true}, + domain: {type: String, index: true/*, unique: true*/}, i2p: { address: String, From 44b8210d95d7fffe2b60d0dc457faaad09a15fb7 Mon Sep 17 00:00:00 2001 From: sevenbitbyte Date: Sun, 21 Jun 2026 15:34:30 +0000 Subject: [PATCH 44/49] cleanup package and project creation --- src/bouncer/db/tingo-db.js | 7 ++- src/bouncer/mongo-query.js | 6 +- src/service/service-host-peer.js | 7 +-- src/venue/bin/commands/pkg-build.js | 8 ++- src/venue/bin/commands/project-build.js | 2 +- src/venue/endpoints/create-package.js | 6 +- src/venue/endpoints/create-project.js | 83 ++++++++++++++++++++++++- src/venue/example-project.js | 2 +- src/venue/schema/venue_package.js | 1 + src/venue/schema/venue_project.js | 1 + 10 files changed, 106 insertions(+), 17 deletions(-) diff --git a/src/bouncer/db/tingo-db.js b/src/bouncer/db/tingo-db.js index 69dcb4c..9ae83a7 100644 --- a/src/bouncer/db/tingo-db.js +++ b/src/bouncer/db/tingo-db.js @@ -164,10 +164,13 @@ module.exports = class TingoDb extends IDb { debug('find collection=', collectionName, ' query=', JSON.stringify(query,null,2)) let collection = await this.getCollection(collectionName) let cursor = await promisfy(collection.find.bind(collection))( - query, - mongoQuery.hasSort() ? mongoQuery.getSort() : undefined + query ) + if(mongoQuery.hasSort()){ + cursor = cursor.sort(mongoQuery.getSort()) + } + if(mongoQuery.hasLimit()){ cursor = cursor.limit(mongoQuery.getLimit()) } diff --git a/src/bouncer/mongo-query.js b/src/bouncer/mongo-query.js index ba09b97..1ad8111 100644 --- a/src/bouncer/mongo-query.js +++ b/src/bouncer/mongo-query.js @@ -44,7 +44,11 @@ class MongoQuery { } getSort () { - return { [this.spec.sort.param.join('.')]: this.spec.sort.direction } + if(Array.isArray(this.spec.sort.param)){ + return { [this.spec.sort.param.join('.')]: this.spec.sort.direction } + } + + return { [this.spec.sort.param]: this.spec.sort.direction } } /** diff --git a/src/service/service-host-peer.js b/src/service/service-host-peer.js index 4aa5f9c..03c37a1 100644 --- a/src/service/service-host-peer.js +++ b/src/service/service-host-peer.js @@ -40,12 +40,9 @@ class ServiceHostPeer { } //! Check if party wants to allow/deny invite sender - if(!(await hostRunner.auth.isPeerConnectionAllowed(invite.from))){ - - // verify actor is allowed - const isAllowed = (await ctx.runner.auth.isAdmin(invite.from)) || + const isAllowed = (await hostRunner.auth.isAdmin(invite.from)) || (await hostRunner.auth.isPeerConnectionAllowed(invite.from)) || - (await ctx.runner.auth.isSocketConnectionAllowed(invite.from)) + (await hostRunner.auth.isSocketConnectionAllowed(invite.from)) if(!isAllowed){ debug('NOT ALLOWED - user is not allowed', invite.from) await invite.reject() diff --git a/src/venue/bin/commands/pkg-build.js b/src/venue/bin/commands/pkg-build.js index 5bcaca5..ee297ba 100644 --- a/src/venue/bin/commands/pkg-build.js +++ b/src/venue/bin/commands/pkg-build.js @@ -124,7 +124,13 @@ class VenuePackageBuild extends CmdTree.Command { if(parsed.deploy && parsed.remote){ console.log('uploading...') + const remote = await this.context.secureConfig.read('remote.'+parsed.remote) + + if(!remote){ + throw 'invalid remote ['+parsed.remote+']' + } + let staticTar = undefined if(build.files.length == 3){ @@ -138,7 +144,7 @@ class VenuePackageBuild extends CmdTree.Command { } - async pushPackage(devId, remote, build, staticTar){ + async pushPackage(devId, remote, build, staticTar=null){ let client = new Dataparty.EphemeralClient({ identity: devId, diff --git a/src/venue/bin/commands/project-build.js b/src/venue/bin/commands/project-build.js index 5d62c74..0a9d8eb 100644 --- a/src/venue/bin/commands/project-build.js +++ b/src/venue/bin/commands/project-build.js @@ -257,7 +257,7 @@ class VenueProjectBuild extends CmdTree.Command { } - async pushProject(devId, remote, build, staticTar){ + async pushProject(devId, remote, build, staticTar=null){ let client = new Dataparty.EphemeralClient({ identity: devId, diff --git a/src/venue/endpoints/create-package.js b/src/venue/endpoints/create-package.js index 90b2da1..18d05d9 100644 --- a/src/venue/endpoints/create-package.js +++ b/src/venue/endpoints/create-package.js @@ -186,14 +186,13 @@ module.exports = class CreatePkgEndpoint extends IEndpoint { debug('verified package signature') const safeFileName = ctx.input.build.package.name.replace('/', '-') + const tarFileName = safeFileName+'.files.venue.tgz' if(ctx.input.staticTar){ const tarHash = Routines.Utils.hash(ctx.input.staticTar) const tarHash64 = Routines.Utils.base64.encode( tarHash ) - const tarFileName = safeFileName+'.files.venue.tgz' - const buildFiles = ctx.input.build.files[tarFileName] if(buildFiles && buildFiles.hash != tarHash64){ @@ -254,6 +253,7 @@ module.exports = class CreatePkgEndpoint extends IEndpoint { venue: ctx.party.identity.key.hash, hash: buildHash, workspace: workspacePath, + tarpath: Path.join(workspacePath, tarFileName), settings: ctx.input.settings, package: pkgWithoutOwner, compressedBuild: Routines.Utils.base64.encode(compressedBrotliBuild) @@ -346,7 +346,7 @@ module.exports = class CreatePkgEndpoint extends IEndpoint { debug('updated service') }*/ - return {done: true} + return {done: true, package: srvDoc.data} //return {srv:srvDoc.data} } diff --git a/src/venue/endpoints/create-project.js b/src/venue/endpoints/create-project.js index 94f3699..493185a 100644 --- a/src/venue/endpoints/create-project.js +++ b/src/venue/endpoints/create-project.js @@ -4,6 +4,9 @@ const Path = require('path') const Hoek = require('@hapi/hoek') const {Message, Routines, Identity} = require('@dataparty/crypto') const debug = require('debug')('dataparty.endpoint.create-project') + +const process = require('process') +const tar = require('tar') const zlib = require('zlib') const IEndpoint = require('../../service/iendpoint') @@ -149,10 +152,8 @@ module.exports = class CreateProjectEndpoint extends IEndpoint { debug('verified package signature') - - const safeFileName = ctx.input.project.name.replace('/', '-') - const tarFileName = safeFileName+'.files.venue.tgz' + const tarFileName = safeFileName+'.project.files.venue.tgz' const projectFiles = ctx.input.project.files[tarFileName] @@ -171,6 +172,39 @@ module.exports = class CreateProjectEndpoint extends IEndpoint { debug('verified staticTar') } + // check route packages are valid + let packages = {} + let tarList = [] + + for(let route of ctx.input.project.routes){ + console.log(route) + let pkgDoc = (await ctx.party.find() + .type('venue_pkg') + .or() + .where('package.name').equals(route.package.name) + .where('package.githash').equals(route.package.githash) + .sort('-created') + .limit(1) + .exec())[0] + + if(!pkgDoc){ + throw new Error(`package ${JSON.stringify(route.package)} not found. required by ${route.prefix}`) + } + + const {compressedBuild, ...printablePkg} = pkgDoc.data + + console.log('found package', printablePkg) + + let pkgTarPath = pkgDoc.data.tarpath + + if(pkgTarPath && pkgTarPath.length > 0){ + tarList.push(pkgTarPath) + } + + packages[route.prefix] = pkgDoc.data + } + + debug('verified project - '+ctx.input.project.name+'@'+ctx.input.project.version) const projectBSON = Routines.BSON.serializeBSONWithoutOptimiser(projectWithoutSig) @@ -222,6 +256,7 @@ module.exports = class CreateProjectEndpoint extends IEndpoint { changed: Date.now(), hash: projectHash, workspace: workspacePath, + tarpath: Path.join(workspacePath, tarFileName), project: ctx.input.project, }) @@ -230,11 +265,53 @@ module.exports = class CreateProjectEndpoint extends IEndpoint { debug('need to update project?') } + debug('extracting other tars', tarList) + + for(let tarPath of tarList){ + debug('extracting', tarPath) + + await tar.extract({ + cwd: workspacePath, + file: tarPath, + newer: true, + unlink: true, + uid: process.getuid(), + gid: process.getgid() + }, tarFileList ) + } + if(ctx.input.staticTar){ + debug('saving tar file', tarFileName) fs.writeFileSync( Path.join(workspacePath, tarFileName), ctx.input.staticTar ) + + debug('extracting contents') + + /*await tar.t({ + cwd: workspacePath, + file: Path.join(workspacePath, tarFileName), + onReadEntry: entry => { console.log('\t\t', entry) } + })*/ + + //const tarFileInfo = projectDoc.data.project.files[ tarFileName ] + + //if(projectFiles){ + const tarFileList = Object.keys(projectFiles.files) + + debug('file list - ', tarFileList) + + await tar.extract({ + cwd: workspacePath, + file: Path.join(workspacePath, tarFileName), + newer: true, + unlink: true, + uid: process.getuid(), + gid: process.getgid() + }, tarFileList ) + + //} } /*fs.writeFileSync( diff --git a/src/venue/example-project.js b/src/venue/example-project.js index 44b93e9..315dd1f 100644 --- a/src/venue/example-project.js +++ b/src/venue/example-project.js @@ -10,7 +10,7 @@ module.exports = { prefix: '/api', party:'SYSTEM', package: { - name: '@dataparty/venue' + name: '@dataparty/api' }, settings: { sendFullErrors: false, diff --git a/src/venue/schema/venue_package.js b/src/venue/schema/venue_package.js index e7802bd..8ce3914 100644 --- a/src/venue/schema/venue_package.js +++ b/src/venue/schema/venue_package.js @@ -18,6 +18,7 @@ class VenuePkg extends ISchema { changed: {type: Number}, venue: {type: String}, workspace: {type: String, required: true}, + tarpath: {type: String}, hash: {type: String, required: true, index: true}, settings: { //enabled: {type: Boolean, required: true}, diff --git a/src/venue/schema/venue_project.js b/src/venue/schema/venue_project.js index 77117c4..2d65721 100644 --- a/src/venue/schema/venue_project.js +++ b/src/venue/schema/venue_project.js @@ -17,6 +17,7 @@ class VenueProject extends ISchema { created: {type: Number, required: true}, changed: {type: Number, required: true}, workspace: {type: String, required: true}, + tarpath: {type: String}, hash: {type: String, required: true, index: true}, enabled: {type: Boolean}, From 3cd823298229ad13d6dafdfb684e978c79352eca Mon Sep 17 00:00:00 2001 From: sevenbitbyte Date: Sun, 21 Jun 2026 18:49:05 +0000 Subject: [PATCH 45/49] dynamic loading --- src/service/service-runner-node.js | 4 +- src/venue/bin/commands/venued-host.js | 112 +++++++++++++++++++++----- 2 files changed, 94 insertions(+), 22 deletions(-) diff --git a/src/service/service-runner-node.js b/src/service/service-runner-node.js index cb15e11..7d93b19 100644 --- a/src/service/service-runner-node.js +++ b/src/service/service-runner-node.js @@ -111,13 +111,13 @@ class ServiceRunnerNode { let AuthClass = null - if(!this.useNative){ + if(!this.useNative && Hoek.reach(this.service, `compiled.auth`)){ var self={} const build = Hoek.reach(this.service, `compiled.auth`) eval(build.code/*, build.map*/) AuthClass = self.Lib } - else{ + else if(this.service.constructors.auth){ AuthClass = this.service.constructors.auth } diff --git a/src/venue/bin/commands/venued-host.js b/src/venue/bin/commands/venued-host.js index 4785d1c..0573a37 100644 --- a/src/venue/bin/commands/venued-host.js +++ b/src/venue/bin/commands/venued-host.js @@ -4,10 +4,12 @@ const debug = require('debug')('venued.host') const Path = require('path') const OS = require('os') const fs = require('fs') - +const zlib = require('zlib') const { execSync } = require('child_process') const Dataparty = require('../../../../') +const {Routines} = require('@dataparty/crypto') +const express = require('express') const HOMEDIR = OS.homedir() const DEFAULT_FOLDER = (HOMEDIR.indexOf('opt')==-1) ? '.venued' : '' @@ -191,7 +193,17 @@ class VenuedHost extends CmdTree.Command { CustomIpFilter.ips.push(ip) } } + + if(!fs.existsSync(parsed['ssl-key'])){ + execSync('openssl req -newkey rsa:2048 -new -nodes -x509 -days 3650 -keyout key.pem -out cert.pem -subj "/C=US/ST=State/L=City/O=Organization/OU=Unit/CN=example.com"', + {cwd: parsed.path} + ) + } + + const ssl_key = fs.readFileSync( parsed['ssl-key'], 'utf8') + const ssl_cert = fs.readFileSync( parsed['ssl-cert'], 'utf8') + const runner = new Dataparty.ServiceRunnerNode({ party, service, @@ -203,14 +215,6 @@ class VenuedHost extends CmdTree.Command { let runnerRouter = new Dataparty.RunnerRouter(runner) - if(!fs.existsSync(parsed['ssl-key'])){ - execSync('openssl req -newkey rsa:2048 -new -nodes -x509 -days 3650 -keyout key.pem -out cert.pem -subj "/C=US/ST=State/L=City/O=Organization/OU=Unit/CN=example.com"', - {cwd: parsed.path} - ) - } - - const ssl_key = fs.readFileSync( parsed['ssl-key'], 'utf8') - const ssl_cert = fs.readFileSync( parsed['ssl-cert'], 'utf8') @@ -287,24 +291,92 @@ class VenuedHost extends CmdTree.Command { if(projects){ for(let name in projects){ + let projectParties = {} + + //! todo - load parties and put them in projectParties map + const hash = projects[name] console.log('\tloading project', name, hash) - - const project = (await party.find() .type('venue_project') .where('hash').equals(hash).exec())[0] - - - //console.log(project.data) - - /** - * 1. load project doc from db - * 2. validate files exist - * 3. launch project - */ + const workspace = project.data.workspace + + for(let route of project.data.project.routes){ + console.log('route', route.package.name) + let pkgDoc = (await party.find() + .type('venue_pkg') + .or() + .where('package.name').equals(route.package.name) + .where('package.githash').equals(route.package.githash) + .sort('-created') + .limit(1) + .exec())[0] + + if(!pkgDoc){ + throw new Error(`package ${JSON.stringify(route.package)} not found. required by ${route.prefix}`) + } + + const {compressedBuild, ...printablePkg} = pkgDoc.data + + console.log('found package', printablePkg) + + const serviceFile = JSON.parse( + zlib.brotliDecompressSync( + Routines.Utils.base64.decode( compressedBuild ) + ) + ) + + console.log('decompressed', serviceFile.package) + + let serviceParty = null; + + + if(route.party == 'SYSTEM'){ + serviceParty = party + } else if( projectParties[route.party] ){ + serviceParty = projectParties[route.party] + } + + serviceParty.topics = new Dataparty.LocalTopicHost() + + debug('loading service') + const service = new Dataparty.IService(serviceFile.package, serviceFile) + debug('loaded service') + + let projectRunner = new Dataparty.ServiceRunnerNode({ + party: serviceParty, service, + sendFullErrors: route.settings.sendFullErrors, + useNative: route.settings.useNative, + prefix: route.prefix + }) + + if(route.party == 'SYSTEM'){ + //projectRunner.router = runner.router + } + + //await serviceParty.start() + await projectRunner.start() + + + console.log('workspace', workspace) + + const projectStaticPath = Path.join(workspace, 'public') + projectRunner.router.add('static-files', '/:path*', (req,res, next)=>{ + + let staticHandler = express.static(projectStaticPath) + + return staticHandler(req.request,req.response) + }) + + + await runnerRouter.addRunner({ + domain: project.data.project.domain, + runner: projectRunner + }) + } } } From c9b81e25ec01bdef09d9fbcb07d0768bd1a1d01b Mon Sep 17 00:00:00 2001 From: sevenbitbyte Date: Sun, 21 Jun 2026 19:37:49 +0000 Subject: [PATCH 46/49] static file hosting working --- src/venue/bin/commands/venued-host.js | 25 ++++++++++++++++++++----- src/venue/endpoints/create-project.js | 8 +++++++- 2 files changed, 27 insertions(+), 6 deletions(-) diff --git a/src/venue/bin/commands/venued-host.js b/src/venue/bin/commands/venued-host.js index 0573a37..81200cb 100644 --- a/src/venue/bin/commands/venued-host.js +++ b/src/venue/bin/commands/venued-host.js @@ -358,18 +358,33 @@ class VenuedHost extends CmdTree.Command { } //await serviceParty.start() - await projectRunner.start() + await projectRunner.start() console.log('workspace', workspace) const projectStaticPath = Path.join(workspace, 'public') - projectRunner.router.add('static-files', '/:path*', (req,res, next)=>{ + + let handler = (req,res)=>{ - let staticHandler = express.static(projectStaticPath) + let staticHandler = express.static(projectStaticPath, { index: ['index.html']}) - return staticHandler(req.request,req.response) - }) + let results = staticHandler(req.request,req.response, req.request.next) + + console.log('returning results', Object.keys(req.request)) + + + + console.log('sent already - headers?', req.response.headersSent) + console.log('sent already - date?', req.response.sendDate) + console.log('sent already - outputSize?', req.response.outputSize) + + } + + projectRunner.router.add('static-files1', '/:path*', handler) + projectRunner.router.add('static-files2', '/', handler) + + await runnerRouter.addRunner({ diff --git a/src/venue/endpoints/create-project.js b/src/venue/endpoints/create-project.js index 493185a..83c1f2a 100644 --- a/src/venue/endpoints/create-project.js +++ b/src/venue/endpoints/create-project.js @@ -277,7 +277,7 @@ module.exports = class CreateProjectEndpoint extends IEndpoint { unlink: true, uid: process.getuid(), gid: process.getgid() - }, tarFileList ) + }, /*tarFileList*/ ) } if(ctx.input.staticTar){ @@ -391,6 +391,12 @@ module.exports = class CreateProjectEndpoint extends IEndpoint { debug('updated service') }*/ + console.log('restarting in 3 seconds...') + setTimeout(()=>{ + process.exit(), + 3000 + }) + return {done: true, project: projectDoc.data} //return {srv:projectDoc.data} From fdad1c5f0f1236f9bfd2e11facffcea863f640d6 Mon Sep 17 00:00:00 2001 From: sevenbitbyte Date: Sun, 28 Jun 2026 00:32:32 +0000 Subject: [PATCH 47/49] venue remote repl --- src/venue/bin/commands/venue-remote-repl.js | 138 ++++++++++++++++++++ src/venue/bin/venue.js | 1 + 2 files changed, 139 insertions(+) create mode 100644 src/venue/bin/commands/venue-remote-repl.js diff --git a/src/venue/bin/commands/venue-remote-repl.js b/src/venue/bin/commands/venue-remote-repl.js new file mode 100644 index 0000000..118fd2d --- /dev/null +++ b/src/venue/bin/commands/venue-remote-repl.js @@ -0,0 +1,138 @@ +const CmdTree = require('command-tree') +const Hoek = require('@hapi/hoek') +const debug = require('debug')('venue.remote-repl') +const Path = require('path') +const OS = require('os') +const fs = require('fs') +const repl = require("repl"); + +const prompt = require('prompt') +const argon2 = require('argon2') + +const { execSync } = require('child_process') + +const Dataparty = require('../../../../') +const dataparty_crypto = require('@dataparty/crypto') + +const DEFINITION = { + h: { + description: 'Show help', + alias: 'help', + type: 'help' + }, + remote: { + type: 'string', + require: true + }, + identity: { + type: 'string', + description: 'admin identity', + require: true + } +} + + +class VenueRemoteRepl extends CmdTree.Command { + constructor(context){ + super({...VenueRemoteRepl.Definition, context}) + debug('constructor') + } + + static get Command(){ + return 'remote repl' + } + + static get Definition(){ + return { + usage: `venue remote repl`, + description: 'Run a repl with access to a remote party', + definition: DEFINITION + } + } + + async run({parsed}){ + //debug('context -', this.context) + //console.log('parsed -', parsed) + + if (parsed.h) { + throw new CmdTree.Error.HelpRequest('help request') + } + + const keyName = parsed.identity + + const phrase = await this.context.secureConfig.read('identity.'+keyName+'.phrase') + + if(!phrase){ + throw new CmdTree.Error.UsageError("Key doesn't exist!") + } + + const {password} = parsed.nopassword ? {password:null} : await prompt.get({ + properties: { + password: { + message: 'Enter password for identity['+keyName+']', + hidden: true + } + }}) + + let key = await dataparty_crypto.Identity.fromMnemonic(phrase, password, argon2) + + key.id = keyName + + const remote = await this.context.secureConfig.read('remote.'+parsed.remote) + + const client = new Dataparty.EphemeralClient({ + identity: key, + urlOrParty: remote.url, + wsUrlOrParty: remote.ws + }) + + client.on('session',(id)=>{ + console.log('session', id) + }) + + client.on('session-end',(id)=>{ + console.log('session-end', id) + }) + + client.on('connecting',(info)=>{ + console.log('connecting', info) + }) + + client.on('connected',(info)=>{ + console.log('connected', info) + }) + + client.on('disconnected',(info)=>{ + console.log('disconnected', info) + }) + + client.on('reconnected',(info)=>{ + console.log('reconnected', info) + }) + + client.on('reconnecting',(info)=>{ + console.log('reconnecting',info) + }) + + await client.start() + console.log('client started') + + + + const replSrv = repl.start({ + prompt: parsed.identity +'@'+ parsed.remote+'> ' + }) + + replSrv.context.remote = remote + replSrv.context.client = client + + + this.context.exiting = false + + return // {remote, client, replSrv} + } +} + +module.exports = VenueRemoteRepl + + diff --git a/src/venue/bin/venue.js b/src/venue/bin/venue.js index c83b6c0..7ce2db9 100755 --- a/src/venue/bin/venue.js +++ b/src/venue/bin/venue.js @@ -22,6 +22,7 @@ commandTree.addCommand(require('./commands/venue-remote-add')) commandTree.addCommand(require('./commands/venue-remote-list')) commandTree.addCommand(require('./commands/venue-remote-show')) commandTree.addCommand(require('./commands/venue-remote-check')) +commandTree.addCommand(require('./commands/venue-remote-repl')) commandTree.addCommand(require('./commands/pkg-build')) commandTree.addCommand(require('./commands/project-build')) From f27761e461d404bf5b7848bd7cad7d82f7820a23 Mon Sep 17 00:00:00 2001 From: sevenbitbyte Date: Sun, 28 Jun 2026 00:38:52 +0000 Subject: [PATCH 48/49] peer query handler --- src/comms/peer-comms.js | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/src/comms/peer-comms.js b/src/comms/peer-comms.js index 4402f66..68a23b9 100644 --- a/src/comms/peer-comms.js +++ b/src/comms/peer-comms.js @@ -92,7 +92,7 @@ class PeerComms extends ISocketComms { let response = null let request = await this.decrypt( {data: message}, this.remoteIdentity ) - debug('handleHostCall', truncateString(request, 1024)) + debug('handleClientCall', truncateString(JSON.stringify(request, null, 2), 1024)) let inputValidated @@ -461,11 +461,15 @@ class PeerComms extends ISocketComms { async handleCallOp(op){ debug('peer-call', op.input.endpoint) + const actor = await this.party.hostRunner.auth.lookupIdentity(this.remoteIdentity) + if(this.party.hostRunner){ debug('calling runner') - if(op.input.endpoint == 'api-v2-peer-bouncer' && await this.party.hostRunner.auth.isAdmin(this.remoteIdentity)){ + + + if(op.input.endpoint == 'api-v2-peer-bouncer' && await this.party.hostRunner.auth.isAdmin(actor)){ debug('ask->', truncateString(op.input.data, 1024)) op.result = {result: await this.party.handleCall(op.input.data) } @@ -514,7 +518,7 @@ class PeerComms extends ISocketComms { op.setState(HostOp.STATES.Finished_Success) return - } else if(op.input.endpoint == 'api-v2-peer-bouncer' && await this.party.hostRunner.auth.isAdmin(this.remoteIdentity)){ + } else if(op.input.endpoint == 'api-v2-peer-bouncer' && await this.party.hostRunner.auth.isAdmin(actor)){ debug('ask->',op.input.data) op.result = {result: await this.party.handleCall(op.input.data) } From 54935f9e7f1f1a87a24f921563e24cceeb0fec22 Mon Sep 17 00:00:00 2001 From: sevenbitbyte Date: Sun, 28 Jun 2026 00:55:45 +0000 Subject: [PATCH 49/49] version bump --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 671ac7f..6ccd4f3 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "name": "@dataparty/api", "private": false, - "version": "1.2.25", + "version": "1.3.0", "main": "dist/dataparty.js", "frontend": "dist/dataparty-browser.js", "backend": "dist/dataparty.js",