diff --git a/examples/secure-config-argon2.js b/examples/secure-config-argon2.js index d4df83f..ce419d4 100644 --- a/examples/secure-config-argon2.js +++ b/examples/secure-config-argon2.js @@ -18,7 +18,6 @@ const HELP_INFO = ` ` async function main(){ - const memoryConfig = new Dataparty.Config.MemoryConfig({foo: 'bar'}) const jsonConfig = new Dataparty.Config.JsonFileConfig({ diff --git a/package.json b/package.json index 5caa37b..6ccd4f3 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "name": "@dataparty/api", "private": false, - "version": "1.2.25", + "version": "1.3.0", "main": "dist/dataparty.js", "frontend": "dist/dataparty-browser.js", "backend": "dist/dataparty.js", @@ -47,6 +47,10 @@ "dist", "src/*" ], + "bin": { + "venue": "./src/venue/bin/venue.js", + "venued": "./src/venue/bin/venued.js" + }, "scripts": { "test": "npx lab", "build": "npx parcel build --no-scope-hoist", @@ -63,10 +67,10 @@ }, "dependencies": { "@babel/runtime": "^7.28.4", - "@dataparty/bouncer-db": "1.0.1", + "@dataparty/bouncer-db": "https://github.com/datapartyjs/bouncer-db#mongoose-latest", "@dataparty/crypto": "github:datapartyjs/dataparty-crypto", "@dataparty/tasker": "^0.0.3", - "@diva.exchange/i2p-sam": "^4.1.8", + "@diva.exchange/i2p-sam": "5.5.2", "@markwylde/liferaft": "^1.3.4", "@sevenbitbyte/ncc": "0.0.2", "ajv": "6.12.5", @@ -77,6 +81,7 @@ "buffer": "^6.0.3", "bufferutil": "^4.0.8", "colors": "1.3.1", + "command-tree": "github:datapartyjs/command-tree", "cors": "^2.8.5", "debug": "^3.1.0", "dom-storage": "^2.1.0", @@ -84,11 +89,14 @@ "express": "^4.17.1", "express-ipfilter": "^1.3.2", "express-list-routes": "^1.1.9", + "fast-safe-stringify": "^2.1.1", + "find-up-json": "^2.0.5", "git-repo-info": "^2.1.1", - "joi": "^17.13.3", + "glob": "^13.0.6", + "joi": "^18.2.1", "joi-objectid": "^4.0.2", "jshashes": "^1.0.8", - "jsonpath-plus": "^0.20.1", + "jsonpath-plus": "10.4.0", "last-eventemitter": "^1.1.1", "lodash": "^4.17.21", "lokijs": "1.5.12", @@ -100,7 +108,7 @@ "node-mocks-http": "^1.12.1", "node-object-hash": "^3.0.0", "node-persist": "^3.0.1", - "origin-router": "^1.6.4", + "origin-router": "https://github.com/datapartyjs/origin-router.git", "parse-url": "^5.0.1", "promisfy": "^1.2.0", "roslib": "^1.3.0", @@ -108,6 +116,7 @@ "simple-peer": "9.11.1", "source-map": "^0.7.3", "store-js": "^2.0.4", + "tar": "^7.5.15", "tingodb": "^0.6.1", "touch": "^3.1.0", "url-parse": "^1.4.7", @@ -118,8 +127,9 @@ "zangodb": "https://github.com/sevenbitbyte/zangodb#hash-patch" }, "devDependencies": { - "@dataparty/bouncer-model": "1.4.3", + "@dataparty/bouncer-model": "https://github.com/datapartyjs/bouncer-model#mongoose-latest", "@hapi/code": "^9.0.1", + "@hapi/joi": "^17.1.1", "@hapi/lab": "^25.0.1", "argon2": "^0.30.3", "argon2-browser": "^1.18.0", diff --git a/scripts/install-i2p-ubuntu-24.sh b/scripts/install-i2p-ubuntu-24.sh new file mode 100755 index 0000000..55019f1 --- /dev/null +++ b/scripts/install-i2p-ubuntu-24.sh @@ -0,0 +1,6 @@ +#!/bin/bash + +sudo add-apt-repository ppa:purplei2p/i2pd +sudo apt update +sudo apt install --yes i2pd + diff --git a/src/bouncer/db/tingo-db.js b/src/bouncer/db/tingo-db.js index 69dcb4c..9ae83a7 100644 --- a/src/bouncer/db/tingo-db.js +++ b/src/bouncer/db/tingo-db.js @@ -164,10 +164,13 @@ module.exports = class TingoDb extends IDb { debug('find collection=', collectionName, ' query=', JSON.stringify(query,null,2)) let collection = await this.getCollection(collectionName) let cursor = await promisfy(collection.find.bind(collection))( - query, - mongoQuery.hasSort() ? mongoQuery.getSort() : undefined + query ) + if(mongoQuery.hasSort()){ + cursor = cursor.sort(mongoQuery.getSort()) + } + if(mongoQuery.hasLimit()){ cursor = cursor.limit(mongoQuery.getLimit()) } diff --git a/src/bouncer/ischema.js b/src/bouncer/ischema.js index 78fd0a8..18c080b 100644 --- a/src/bouncer/ischema.js +++ b/src/bouncer/ischema.js @@ -1,4 +1,4 @@ -const debug = require('debug')('bouncer.ISchema') +const debug = require('debug')('dataparty.bouncer.ISchema') const MgoUtils = require('../utils/mongoose-scheme-utils') module.exports = class ISchema { diff --git a/src/bouncer/mongo-query.js b/src/bouncer/mongo-query.js index ba09b97..1ad8111 100644 --- a/src/bouncer/mongo-query.js +++ b/src/bouncer/mongo-query.js @@ -44,7 +44,11 @@ class MongoQuery { } getSort () { - return { [this.spec.sort.param.join('.')]: this.spec.sort.direction } + if(Array.isArray(this.spec.sort.param)){ + return { [this.spec.sort.param.join('.')]: this.spec.sort.direction } + } + + return { [this.spec.sort.param]: this.spec.sort.direction } } /** diff --git a/src/comms/isocket-comms.js b/src/comms/isocket-comms.js index 94070ae..cfe90fc 100644 --- a/src/comms/isocket-comms.js +++ b/src/comms/isocket-comms.js @@ -71,6 +71,7 @@ class ISocketComms extends EventEmitter { this.connected = false debug('Server closed connection') this.emit('close') + this.emit('server-close') } onopen(){ diff --git a/src/comms/peer-comms.js b/src/comms/peer-comms.js index c19a07c..68a23b9 100644 --- a/src/comms/peer-comms.js +++ b/src/comms/peer-comms.js @@ -54,6 +54,9 @@ class PeerComms extends ISocketComms { this.uuid = uuidv4() this.socket = socket || null + this.stopped = false + this.started = false + //this.auto_reconnect = !socket && !host this.host = host //! Is comms host\ this.oncall = null @@ -89,7 +92,7 @@ class PeerComms extends ISocketComms { let response = null let request = await this.decrypt( {data: message}, this.remoteIdentity ) - debug('handleHostCall', truncateString(request, 1024)) + debug('handleClientCall', truncateString(JSON.stringify(request, null, 2), 1024)) let inputValidated @@ -203,11 +206,16 @@ class PeerComms extends ISocketComms { async start(){ debug('start') + + if(this.started){ return } + + this.started = true + if(this.socketInit){ await this.socketInit() } - this.socket.on('close', this.stop.bind(this)) + this.socket.on('close', this.socketStop.bind(this)) if(this.host){ debug('host mode comms') @@ -226,7 +234,14 @@ class PeerComms extends ISocketComms { } } + socketStop(){ + debug('socket stop') + this.close() + } + async stop(){ + this.stopped = true + this.started = false debug('stop') this.close() } @@ -385,13 +400,13 @@ class PeerComms extends ISocketComms { if(this.party.hostRunner){ const actor = await this.party.hostRunner.auth.lookupIdentity(offer.sender) - const verified = await Routines.verifyDataPQ(actor, signature, offerBSON) + const verified = await Routines.verifyDataPQ(offer.sender, signature, offerBSON) if(!verified){ throw new Error('DENY(hostRunner) - auth op signature is not valid') } - if(this.discoverRemoteIdentity){ this.remoteIdentity = actor } + if(this.discoverRemoteIdentity){ this.remoteIdentity = offer.sender } const authorized = await this.party.hostRunner.auth.isSocketConnectionAllowed(actor) if(!authorized){ @@ -406,6 +421,7 @@ class PeerComms extends ISocketComms { await this.stop() debug('DENY - client not allowed - ', this.remoteIdentity) + throw new Error('DENY - client not allowed') } } else { const actor = offer.sender @@ -420,7 +436,7 @@ class PeerComms extends ISocketComms { } } - debug('clienr auth op offer -', offer) + debug('client auth op offer -', offer) debug('ALLOW - allowing client - ', this.remoteIdentity) this.aesStream = await AESStream.recoverStream( @@ -445,11 +461,15 @@ class PeerComms extends ISocketComms { async handleCallOp(op){ debug('peer-call', op.input.endpoint) + const actor = await this.party.hostRunner.auth.lookupIdentity(this.remoteIdentity) + if(this.party.hostRunner){ debug('calling runner') - if(op.input.endpoint == 'api-v2-peer-bouncer' && await this.party.hostRunner.auth.isAdmin(this.remoteIdentity)){ + + + if(op.input.endpoint == 'api-v2-peer-bouncer' && await this.party.hostRunner.auth.isAdmin(actor)){ debug('ask->', truncateString(op.input.data, 1024)) op.result = {result: await this.party.handleCall(op.input.data) } @@ -457,10 +477,18 @@ class PeerComms extends ISocketComms { return } + debug('input type', typeof op.input.data, Object.keys(op.input.data)) + debug('op.msg type', typeof op.msg, Object.keys(op.msg), Buffer.isBuffer(op.msg)) + + let bodyValue = Buffer.isBuffer(op.msg) ? + op.input.data : + //Routines.BSON.parseObject(new Routines.BSON.BaseParser( op.msg )) : + JSON.parse(op.msg.toString()) + const req = HttpMocks.createRequest({ method: 'GET', url: '/'+op.input.endpoint, - body: (op.input.data) ? JSON.parse(op.msg.toString()) : undefined + body: bodyValue }) const res = HttpMocks.createResponse() @@ -473,6 +501,9 @@ class PeerComms extends ISocketComms { debug('route',route) + req.peer = this + req.source = 'PeerComms' + debug('call route', await route._events.route({ method: req.method, pathname: req.url, @@ -487,7 +518,7 @@ class PeerComms extends ISocketComms { op.setState(HostOp.STATES.Finished_Success) return - } else if(op.input.endpoint == 'api-v2-peer-bouncer' && await this.party.hostRunner.auth.isAdmin(this.remoteIdentity)){ + } else if(op.input.endpoint == 'api-v2-peer-bouncer' && await this.party.hostRunner.auth.isAdmin(actor)){ debug('ask->',op.input.data) op.result = {result: await this.party.handleCall(op.input.data) } diff --git a/src/comms/rest-comms.js b/src/comms/rest-comms.js index cac3a67..be11241 100644 --- a/src/comms/rest-comms.js +++ b/src/comms/rest-comms.js @@ -142,7 +142,19 @@ class RestComms extends EventEmitter { // debug('raw reply ->', reply) } catch (error) { debug('rest', fullPath, ' call fail ->', error.message) - throw new Error('RestCommsError') + + console.log(Object.keys(error), Object.keys(error.response)) + + const simpleError = { + name: error.name, + code: error.code, + //message: error.message, + statusCode: error.response.statusCode, + statusMessage: error.response.statusMessage, + data: error.response.data + } + + throw simpleError } const msg = await this.party.decrypt( @@ -207,7 +219,7 @@ class RestComms extends EventEmitter { const serverIdentity = await RestComms.HttpGet(this.uri + `${this.uriPrefix}identity`) debug('server identity - ', serverIdentity) - this.remoteIdentity = new dataparty_crypto.Identity(serverIdentity) + this.remoteIdentity = dataparty_crypto.Identity.fromJSON(serverIdentity) } return this.remoteIdentity diff --git a/src/comms/websocket-comms.js b/src/comms/websocket-comms.js index d91d690..fbfce40 100644 --- a/src/comms/websocket-comms.js +++ b/src/comms/websocket-comms.js @@ -14,11 +14,13 @@ const WebsocketShim = require('./websocket-shim') * @see https://en.wikipedia.org/wiki/WebSocket */ class WebsocketComms extends PeerComms { - constructor({uri, connection, remoteIdentity, host, party, ...options}){ + constructor({uri, connection, timeout=20000, remoteIdentity, host, party, ...options}){ super({remoteIdentity, host, party, ...options}) this.uri = uri this.connection = connection + this.timeout = timeout + this.timer = null debug('starting host=',host, ' uuid=', this.uuid, ' uri=', this.uri) @@ -34,13 +36,44 @@ class WebsocketComms extends PeerComms { async socketInit(){ debug('init') - + let isNewConnection = false + if(!this.host && !this.connection){ debug('opening client connection to',this.uri) this.connection = new WebSocket(this.uri) + + isNewConnection = true } this.socket = new WebsocketShim(this.connection) + + if(isNewConnection){ + + //await new Promise((resolve,reject)=>{ + this.timer = setTimeout(() => { + debug('websocket timeout') + this.connection.close() + this.emit('timeout') + //reject(new Error("WebSocket connection timeout")); + }, this.timeout); + + this.socket.once('connect', () => { + debug('websocket opened') + clearTimeout(this.timer); + //resolve(); + }) + + this.socket.once('error',(error) => { + debug('websocket error', error) + clearTimeout(this.timer) + this.emit('error', error) + //this.connection.close() + //reject(error); + }) + //}) + } + + } } diff --git a/src/comms/websocket-shim.js b/src/comms/websocket-shim.js index 1f8b2ec..1a6e72a 100644 --- a/src/comms/websocket-shim.js +++ b/src/comms/websocket-shim.js @@ -19,7 +19,7 @@ class WebsocketShim extends EventEmitter { } this.conn.onclose = (event) => { - debug('onclose', event) + debug('onclose', event, event.code, event.reason) this.emit('close', event) } @@ -39,6 +39,7 @@ class WebsocketShim extends EventEmitter { } destroy(){ + if(this.conn && this.conn.terminate) this.conn.terminate() } diff --git a/src/config/json-file.js b/src/config/json-file.js index b5280ab..a8376c8 100644 --- a/src/config/json-file.js +++ b/src/config/json-file.js @@ -22,6 +22,8 @@ class JsonFileConfig extends IConfig { this.path = this.basePath +'/config.json' this.defaults = defaults || {} this.content = Object.assign({}, this.defaults) + this.writing = false + this.started = false } async load(){ @@ -47,9 +49,16 @@ class JsonFileConfig extends IConfig { } async start () { + + if(this.started){return} + await this.touchDir('') await this.load() + + fs.watchFile(this.path, this.handleFileChange.bind(this)) logger('started') + + this.started = true } async clear () { @@ -79,7 +88,9 @@ class JsonFileConfig extends IConfig { } async save(){ + this.writing = true fs.writeFileSync(this.path, JSON.stringify(this.content, null, 2)) + this.writing = false } async touchDir (path) { @@ -98,6 +109,24 @@ class JsonFileConfig extends IConfig { }) }) } + + fileExists(path){ + var realPath = Path.join(this.basePath, Path.dirname(path), sanitize(Path.basename(path))) + + return fs.existsSync(realPath) + } + + filePath(path){ + return Path.join(this.basePath, Path.dirname(path), sanitize(Path.basename(path))) + } + + async handleFileChange(current, previous){ + if(this.writing){ return } + + logger('config changed, reloading') + + await this.load() + } } module.exports = JsonFileConfig \ No newline at end of file diff --git a/src/party/document-factory.js b/src/party/document-factory.js index ba5c85f..6f7adf8 100644 --- a/src/party/document-factory.js +++ b/src/party/document-factory.js @@ -19,7 +19,7 @@ class DocumentFactory { this.factories = factories || {} this.party = party || null this.ajv = new Ajv() - //this.model = model + this.model = model this.documentClass = documentClass || IDocument this.validators = {} diff --git a/src/party/index-browser.js b/src/party/index-browser.js index cf40f10..b084914 100644 --- a/src/party/index-browser.js +++ b/src/party/index-browser.js @@ -7,6 +7,7 @@ const ZangoParty = require('./local/zango-party') const IDocument = require('./idocument') const DocumentFactory = require('./document-factory') const CloudDocument = require('./cloud/cloud-document') +const EphemeralClient = require('./peer/ephemeral-client') const MatchMakerClient = require('./peer/match-maker-client') const LokiDb = require('../bouncer/db/loki-db') @@ -15,5 +16,5 @@ module.exports = { IDocument, IParty, DocumentFactory, CloudDocument, CloudParty, LokiParty, ZangoParty, PeerParty, - LokiDb, MatchMakerClient + LokiDb, EphemeralClient, MatchMakerClient } diff --git a/src/party/index-embedded.js b/src/party/index-embedded.js index 422f737..ba36e58 100644 --- a/src/party/index-embedded.js +++ b/src/party/index-embedded.js @@ -7,11 +7,12 @@ const TingoParty = require('./local/tingo-party') const IDocument = require('./idocument') const DocumentFactory = require('./document-factory') const CloudDocument = require('./cloud/cloud-document') +const EphemeralClient = require('./peer/ephemeral-client') const MatchMakerClient = require('./peer/match-maker-client') module.exports = { IDocument, IParty, DocumentFactory, CloudDocument, CloudParty, LokiParty, PeerParty, - TingoParty, MatchMakerClient + TingoParty, EphemeralClient, MatchMakerClient } \ No newline at end of file diff --git a/src/party/index.js b/src/party/index.js index e362eff..af6f1cc 100644 --- a/src/party/index.js +++ b/src/party/index.js @@ -9,4 +9,5 @@ exports.MongoParty = require('./mongo/mongo-party') exports.IDocument = require('./idocument') exports.DocumentFactory = require('./document-factory') exports.CloudDocument = require('./cloud/cloud-document') -exports.MatchMakerClient = require('./peer/match-maker-client') \ No newline at end of file +exports.EphemeralClient = require('./peer/ephemeral-client') +exports.MatchMakerClient = require('./peer/match-maker-client') diff --git a/src/party/peer/ephemeral-client.js b/src/party/peer/ephemeral-client.js new file mode 100644 index 0000000..8babd30 --- /dev/null +++ b/src/party/peer/ephemeral-client.js @@ -0,0 +1,276 @@ +const EventEmitter = require('eventemitter3') + +const debug = require('debug')('dataparty.ephemeral-client') + +const dataparty_crypto = require('@dataparty/crypto') +const LokiParty = require('../local/loki-party') +const PeerParty = require('./peer-party') +const MemoryConfig = require('../../config/memory') +const RestComms = require('../../comms/rest-comms') +const WebsocketComms = require('../../comms/websocket-comms') + +class EphemeralClient extends EventEmitter { + constructor({identity, role='guest', contacts, urlOrParty = 'https://api.dataparty.xyz/api', wsUrlOrParty = 'wss://api.dataparty.xyz/ws'}){ + + super() + + this.contacts = contacts + this.sessionKey = null + this.identity = identity + this.role = role || 'guest' //! todo/note - these are different from invite roles + this.wsParty = null + this.restParty = null + + if(typeof urlOrParty == 'string'){ + this.restUrl = urlOrParty + this.restParty = null + } else { + this.restParty = urlOrParty + } + + if(typeof wsUrlOrParty == 'string'){ + this.wsUrl = wsUrlOrParty + this.wsParty = null + } else { + this.wsParty = wsUrlOrParty + } + } + + + /*get restParty(){ + return this.restParty + }*/ + + get socketParty(){ + return this.wsParty + } + + + async start(){ + this.sessionKey = await dataparty_crypto.Identity.fromRandomSeed({id:'ephemeral-session-key'}) + + if(!this.restParty){ + let config = new MemoryConfig({ + basePath:'ephemeral-client', + cloud: { + uri: this.restUrl + } + }) + + this.restParty = new LokiParty({ + path: 'ephemeral-client', + dbAdapter: new LokiParty.Loki.LokiMemoryAdapter(), + config + }) + + await this.restParty.setIdentity(this.sessionKey) + + debug('starting restParty') + await this.restParty.start() + + if(!this.restParty.comms){ + this.restParty.comms = new RestComms({ + party:this.restParty, + config: this.restParty.config + }) + + this.restParty.comms.sessionId = this.sessionKey.key.hash + } + + await this.announcePublicKeys() + } + + if(!this.wsParty && this.wsUrl){ + + this.wsParty = new PeerParty({ + comms: new WebsocketComms({ + uri: this.wsUrl, + discoverRemoteIdentity: false, + remoteIdentity: await this.restParty.comms.getServiceIdentity(), + session: this.sessionKey.key.hash + }), + config: this.restParty.config + }) + + this.wsParty.comms.on('close', ()=>{ + + let stopped = this.wsParty.comms.stopped + console.log('hey the ws closed - stopped=', stopped) + + }) + + await this.wsParty.start() + + debug('starting wsParty') + await this.wsParty.start() + debug('waiting for websocket authorization') + await this.wsParty.comms.authorized() + } + + } + + async createSessionAnnoucement(){ + let currentActor = this.identity + + const announceData = { + annoucement: { + role: this.role, + created: Date.now(), + expiry: Date.now() + 24*60*60*1000, //! Set session expiry to 24hr from now + sessionKey: { + type: this.sessionKey.key.type, + hash: this.sessionKey.key.hash, + public: this.sessionKey.key.public + }, + actorKey: { + type: currentActor.key.type, + hash: currentActor.key.hash, + public: currentActor.key.public + } + }, + trust: { + actorSig: null, + sessionSig: null + } + } + + + const actorSigMsg = await currentActor.sign(announceData.annoucement, true) + const sessionSigMsg = await this.sessionKey.sign(announceData.annoucement, true) + + debug('actorSigMsg', actorSigMsg) + debug('sessionSigMsg', sessionSigMsg) + + announceData.trust.actorSig = dataparty_crypto.Routines.Utils.base64.encode( actorSigMsg.sig ) + announceData.trust.sessionSig = dataparty_crypto.Routines.Utils.base64.encode( sessionSigMsg.sig ) + + return announceData + } + + async announcePublicKeys(callPath='key/announce'){ + + let currentActor = this.identity + + const announceData = { + annoucement: { + role: this.role, + created: Date.now(), + expiry: Date.now() + 24*60*60*1000, //! Set session expiry to 24hr from now + sessionKey: { + type: this.sessionKey.key.type, + hash: this.sessionKey.key.hash, + public: this.sessionKey.key.public + }, + actorKey: { + type: currentActor.key.type, + hash: currentActor.key.hash, + public: currentActor.key.public + } + }, + trust: { + actorSig: null, + sessionSig: null + } + } + + + const actorSigMsg = await currentActor.sign(announceData.annoucement, true) + const sessionSigMsg = await this.sessionKey.sign(announceData.annoucement, true) + + debug('actorSigMsg', actorSigMsg) + debug('sessionSigMsg', sessionSigMsg) + + announceData.trust.actorSig = dataparty_crypto.Routines.Utils.base64.encode( actorSigMsg.sig ) + announceData.trust.sessionSig = dataparty_crypto.Routines.Utils.base64.encode( sessionSigMsg.sig ) + + debug('announcePublicKeys', announceData) + + const announceResult = await this.restParty.comms.call(callPath, announceData, { + expectClearTextReply: false, + sendClearTextRequest: false, + useSessions: false + }) + + if(announceResult.done != true){ + throw new Error('annoucement request failed - '+callPath) + } + } + + + async lookupPublicKey(hash){ + debug('lookupPublicKey - hash:', hash) + + if(hash == this.identity.key.hash){ + return this.identity + } + + if(this.contacts){ + return await this.contacts.lookupPublicKey(hash) + } + + const lookupData = { hash } + + const lookupResult = await this.socketParty.comms.call('key/lookup', lookupData, { + expectClearTextReply: false, + sendClearTextRequest: false, + useSessions: true + }) + + if(!lookupResult.done){ + return null + } + + debug('lookup result -', lookupResult) + + const identity = new dataparty_crypto.Identity({ + key: lookupResult.public_key + }) + + return identity + } + + async createShortCode(use_limit=3, expiry){ + debug('createShortCode') + + const request = { + use_limit, + expiry: !expiry ? Date.now()+24*60*60*3 : expiry + } + + const result = await this.socketParty.comms.call('short-code/create', request, { + expectClearTextReply: false, + sendClearTextRequest: false, + useSessions: true + }) + + console.log('createShortCode result', result) + + if(!result.done){ + return null + } + + return result.short_code + } + + async lookupPublicKeyByShortCode( code ){ + debug('lookupPublicKeyByShortCode') + + const request = { code } + + const result = await this.socketParty.comms.call('short-code/lookup', request, { + expectClearTextReply: false, + sendClearTextRequest: false, + useSessions: true + }) + + console.log('lookupPublicKeyByShortCode result', result) + + if(!result.done){ + return null + } + + return result.short_code + } +} + +module.exports = EphemeralClient diff --git a/src/party/peer/match-maker-client.js b/src/party/peer/match-maker-client.js index 3ebd7fa..7847d4e 100644 --- a/src/party/peer/match-maker-client.js +++ b/src/party/peer/match-maker-client.js @@ -2,40 +2,21 @@ const EventEmitter = require('eventemitter3') const debug = require('debug')('dataparty.match-maker-client') - const dataparty_crypto = require('@dataparty/crypto') const LokiParty = require('../local/loki-party') const PeerParty = require('./peer-party') const MemoryConfig = require('../../config/memory') +const RestComms = require('../../comms/rest-comms') const WebsocketComms = require('../../comms/websocket-comms') const PeerInvite = require('./peer-invite') class MatchMakerClient extends EventEmitter { - constructor(identity, contacts, urlOrParty = 'https://postquantum.one/api/', wsUrlOrParty = 'wss://postquantum.one/ws'){ + constructor(client){ super() - - this.contacts = contacts - this.sessionKey = null - this.identity = identity - this.wsParty = null - this.restParty = null - - if(typeof urlOrParty == 'string'){ - this.restUrl = urlOrParty - this.restParty = null - } else { - this.restParty = urlOrParty - } - - if(typeof wsUrlOrParty == 'string'){ - this.wsUrl = wsUrlOrParty - this.wsParty = null - } else { - this.wsParty = wsUrlOrParty - } + this.client = client this.invitesTx = null this.invitesRx = null @@ -44,80 +25,62 @@ class MatchMakerClient extends EventEmitter { tx: {}, rx: {} } + + this.started = false + + this.client.on('reconnected', this.handleReconnect.bind(this)) + this.client.on('disconnected', this.handleDisconnect.bind(this)) } + async handleReconnect(){ + //if(!this.started){ return } + + debug('handleReconnect') + await this.start() + } + async handleDisconnect(){ + if(!this.started){ return } + + thi.started = false + + debug('handleDisconnect') + this.invitesRx.unsubscribe( this.handleInviteRxMsg.bind(this) ) + this.invitesTx.unsubscribe( this.handleInviteTxMsg.bind(this) ) + + this.invitesRx = null + this.invitesTx = null + + this.emit('disconnected') + } async start(){ - this.sessionKey = await dataparty_crypto.Identity.fromRandomSeed({id:'ephemeral-session-key'}) - - if(!this.restParty){ - let config = new MemoryConfig({ - basePath:'match-maker-client', - cloud: { - uri: this.restUrl - } - }) - - this.restParty = new LokiParty({ - path: 'match-maker-client', - dbAdapter: new LokiParty.Loki.LokiMemoryAdapter(), - config - }) - - await this.restParty.setIdentity(this.sessionKey) - - debug('starting restParty') - await this.restParty.start() - - if(!this.restParty.comms){ - this.restParty.comms = new Dataparty.Comms.RestComms({ - party:this.restParty, - config: this.restParty.config - }) - - this.restParty.comms.sessionId = this.sessionKey.key.hash - } - await this.announcePublicKeys() - } + if(this.started){ return } - if(!this.wsParty){ - this.wsParty = new PeerParty({ - comms: new WebsocketComms({ - uri: this.wsUrl, - discoverRemoteIdentity: false, - remoteIdentity: await this.restParty.comms.getServiceIdentity(), - session: this.sessionKey.key.hash - }), - config: this.restParty.config - }) + this.started = true + await this.client.start() - await this.wsParty.start() + const party = this.client.socketPeerParty - debug('starting wsParty') - await this.wsParty.start() - debug('waiting for websocket authorization') - await this.wsParty.comms.authorized() + this.invitesRx = new party.ROSLIB.Topic({ + ros : party.comms.ros, + name : '/invites/' + encodeURIComponent(this.client.identity.key.hash) + '/rx', + messageType: 'Object' + }) - this.invitesRx = new this.wsParty.ROSLIB.Topic({ - ros : this.wsParty.comms.ros, - name : '/invites/' + encodeURIComponent(this.identity.key.hash) + '/rx', - messageType: 'Object' - }) + this.invitesRx.subscribe( this.handleInviteRxMsg.bind(this) ) - this.invitesRx.subscribe( this.handleInviteRxMsg.bind(this) ) + this.invitesTx = new party.ROSLIB.Topic({ + ros : party.comms.ros, + name : '/invites/' + encodeURIComponent(this.client.identity.key.hash) + '/tx', + messageType: 'Object' + }) - this.invitesTx = new this.wsParty.ROSLIB.Topic({ - ros : this.wsParty.comms.ros, - name : '/invites/' + encodeURIComponent(this.identity.key.hash) + '/tx', - messageType: 'Object' - }) + this.invitesTx.subscribe( this.handleInviteTxMsg.bind(this) ) - this.invitesTx.subscribe( this.handleInviteTxMsg.bind(this) ) - } - + this.emit('connected') } async handleInviteRxMsg( msg ){ @@ -127,8 +90,8 @@ class MatchMakerClient extends EventEmitter { if(!this.pendingInvites.rx[inviteId] && msg.invite.state == 'invited'){ - const from = await this.lookupPublicKey(msg.invite.fromHash) - const to = await this.lookupPublicKey(msg.invite.toHash) + const from = await this.client.lookupPublicKey(msg.invite.fromHash) + const to = await this.client.lookupPublicKey(msg.invite.toHash) let invite = new PeerInvite(msg.invite, to, this, from) @@ -157,93 +120,24 @@ class MatchMakerClient extends EventEmitter { debug('calling onInviteMsg') await pending.onInviteMsg(msg.invite) - + } } + async createInvite(toHashOrIdentity, {type, service, role, session}, info){ - async announcePublicKeys(){ - const announceData = { - annoucement: { - created: Date.now(), - expiry: Date.now() + 24*60*60*1000, //! Set session expiry to 24hr from now - sessionKey: { - type: this.sessionKey.key.type, - hash: this.sessionKey.key.hash, - public: this.sessionKey.key.public - }, - actorKey: { - type: this.identity.key.type, - hash: this.identity.key.hash, - public: this.identity.key.public - } - }, - trust: { - actorSig: null, - sessionSig: null - } - } - - - const actorSigMsg = await this.identity.sign(announceData.annoucement, true) - const sessionSigMsg = await this.sessionKey.sign(announceData.annoucement, true) - - debug('actorSigMsg', actorSigMsg) - debug('sessionSigMsg', sessionSigMsg) - - announceData.trust.actorSig = dataparty_crypto.Routines.Utils.base64.encode( actorSigMsg.sig ) - announceData.trust.sessionSig = dataparty_crypto.Routines.Utils.base64.encode( sessionSigMsg.sig ) - - debug('announcePublicKeys', announceData) - - const announceResult = await this.restParty.comms.call('key/announce', announceData, { - expectClearTextReply: false, - sendClearTextRequest: false, - useSessions: false - }) - } - - - async lookupPublicKey(hash){ - debug('lookupPublicKey - hash:', hash) - - if(hash == this.identity.key.hash){ - return this.identity - } - - if(this.contacts){ - return await this.contacts.lookupPublicKey(hash) - } - - const lookupData = { hash } - - const lookupResult = await this.restParty.comms.call('key/lookup', lookupData, { - expectClearTextReply: false, - sendClearTextRequest: false, - useSessions: true - }) + const roles = ['client', 'host'] - if(!lookupResult.done){ - return null + if(roles.indexOf(role) == -1){ + throw new Error("Invalid requested role [" + role + "]") } - debug('lookup result -', lookupResult) - - const identity = new dataparty_crypto.Identity({ - key: lookupResult.public_key - }) - - return identity - } - - async createInvite(toHashOrIdentity, {type, service, role, session}, info){ - debug('createInvite') let toIdentity = null if(typeof toHashOrIdentity == 'string'){ - toIdentity = await this.lookupPublicKey(toHashOrIdentity) + toIdentity = await this.client.lookupPublicKey(toHashOrIdentity) } else { toIdentity = toHashOrIdentity } @@ -255,7 +149,7 @@ class MatchMakerClient extends EventEmitter { service: service ? service : '@dataparty/video-chat', role: role ? role : 'client', timestamp: (new Date()).getTime(), - from: this.identity.key.hash, + from: this.client.identity.key.hash, to: toIdentity.key.hash, session: session ? session : Math.random().toString(36).slice(2), info: info ? info : { @@ -264,17 +158,17 @@ class MatchMakerClient extends EventEmitter { } } - const secureInvite = await this.identity.encrypt(invitePayload, toIdentity) + const secureInvite = await this.client.identity.encrypt(invitePayload, toIdentity) debug('secure-invite', secureInvite) const invitePostData = { to: toIdentity.key.hash, - from: this.identity.key.hash, + from: this.client.identity.key.hash, payload: JSON.stringify(secureInvite.toJSON()) } - const inviteResult = await this.restParty.comms.call('invite/create', invitePostData, { + const inviteResult = await this.client.socketPeerParty.comms.call('invite/create', invitePostData, { expectClearTextReply: false, sendClearTextRequest: false, useSessions: true @@ -284,9 +178,9 @@ class MatchMakerClient extends EventEmitter { if(!inviteDoc){ return } - let invite = new PeerInvite(inviteResult.invite, toIdentity, this, this.identity) + let invite = new PeerInvite(inviteResult.invite, toIdentity, this, this.client.identity, invitePayload) - invite.payload = invitePayload + //invite.payload = invitePayload this.pendingInvites.tx[inviteDoc.$meta.id] = invite @@ -296,16 +190,16 @@ class MatchMakerClient extends EventEmitter { } async lookupInvites({createdAfter, type='to', id, actorHash }){ - let actor = this.identity.key.hash + let actor = this.client.identity.key.hash const lookup = { invite: id, - actor: actorHash ? actorHash : this.identity.key.hash, + actor: actorHash ? actorHash : this.client.identity.key.hash, createdAfter, type: !type ? 'to' : type } - const lookupResult = await this.restParty.comms.call('invite/lookup', lookup, { + const lookupResult = await this.client.socketPeerParty.comms.call('invite/lookup', lookup, { expectClearTextReply: false, sendClearTextRequest: false, useSessions: true @@ -341,8 +235,8 @@ class MatchMakerClient extends EventEmitter { for(let i=0; i < invites.length; i++){ const invite = invites[i] - let to = await this.lookupPublicKey( invite.toHash ) - let from = await this.lookupPublicKey( invite.fromHash ) + let to = await this.client.lookupPublicKey( invite.toHash ) + let from = await this.client.lookupPublicKey( invite.fromHash ) let peerInvite = new PeerInvite( invites[i], to, this, from) @@ -363,14 +257,14 @@ class MatchMakerClient extends EventEmitter { async setInviteState(invite, newState){ debug('setInviteState') - let actor = this.identity.key.hash + let actor = this.client.identity.key.hash const inviteState = { invite: invite.inviteDoc.$meta.id, state: newState } - const inviteStateResult = await this.restParty.comms.call('invite/set-state', inviteState, { + const inviteStateResult = await this.client.socketPeerParty.comms.call('invite/set-state', inviteState, { expectClearTextReply: false, sendClearTextRequest: false, useSessions: true @@ -385,48 +279,6 @@ class MatchMakerClient extends EventEmitter { return inviteStateResult.invite } - async createShortCode(use_limit=3, expiry){ - debug('createShortCode') - - const request = { - use_limit, - expiry: !expiry ? Date.now()+24*60*60*3 : expiry - } - - const result = await this.restParty.comms.call('short-code/create', request, { - expectClearTextReply: false, - sendClearTextRequest: false, - useSessions: true - }) - - console.log('createShortCode result', result) - - if(!result.done){ - return null - } - - return result.short_code - } - - async lookupPublicKeyByShortCode( code ){ - debug('lookupPublicKeyByShortCode') - - const request = { code } - - const result = await this.restParty.comms.call('short-code/lookup', request, { - expectClearTextReply: false, - sendClearTextRequest: false, - useSessions: true - }) - - console.log('lookupPublicKeyByShortCode result', result) - - if(!result.done){ - return null - } - - return result.short_code - } } module.exports = MatchMakerClient diff --git a/src/party/peer/peer-client.js b/src/party/peer/peer-client.js new file mode 100644 index 0000000..704f6ab --- /dev/null +++ b/src/party/peer/peer-client.js @@ -0,0 +1,102 @@ +const EphemeralClient = require("./ephemeral-client") + + +class PeerClient extends EphemeralClient { + constructor({model=null, /*hostParty=null, */ contacts, identity, remoteIdentityHash, matchMaker, service, role='client', rtcSettings}){ + + super({identity, contacts, role}) + + this.model = model + this.hostParty = hostParty + this.matchMaker = matchMaker + + this.remoteIdentityHash = remoteIdentityHash + + + this.inviteSettings = { + type: 'webrtc', + service: model ? model.package.name : service, + role: role ? role : 'client', + session: null + } + + this.rtcSettings = this.rtcSettings + + this.peerParty = null + + /*if(role == 'host' && hostParty==null){ + throw + }*/ + } + + get restParty(){ + return this.peerParty + } + + get socketParty(){ + return this.peerParty + } + + async start(mediaSrc){ + // + + if(this.sessionKey){ return } + this.sessionKey = await dataparty_crypto.Identity.fromRandomSeed({id:'ephemeral-session-key'}) + + this.inviteSettings.session = this.sessionKey.key.hash + + const role = this.inviteSettings.role + + this.emit('connecting', {time: Date.now()}) + const invite = await this.announcePublicKeys() + + await invite.waitForAccepted() + + this.peerParty = await invite.establish({ + mediaSrc, + role, + hostParty: role == 'host' ? this. this.hostParty : undefined, + model: role == 'host' ? this.hostParty.factory.model : this.model, + rtcSettings: this.rtcSettings + }) + + this.emit('connected', {time: Date.now()}) + + return this.peerParty + } + + async rollSessionKey(){ + debug('rollSessionKey') + this.emit('session-end', {time: Date.now(), session: this.sessionKey.key.hash}) + + if(this.peerParty){ + await this.peerParty.stop() + } + + this.sessionKey = null + this.peerParty = null + + await this.start() + } + + async handleClose(){ + // + } + + async doReconnect(){ + // + } + + async announcePublicKeys(){ + const announceData = await this.createSessionAnnoucement() + + let invite = await this.matchMaker.createInvite(this.remoteIdentityHash, this.inviteSettings, announceData) + + this.emit('session', {time: Date.now(), session: this.sessionKey.key.hash}) + + return invite + } + +} + +module.exports = PeerClient diff --git a/src/party/peer/peer-invite.js b/src/party/peer/peer-invite.js index dbd3f8b..59bfa9f 100644 --- a/src/party/peer/peer-invite.js +++ b/src/party/peer/peer-invite.js @@ -8,6 +8,8 @@ const dataparty_crypto = require('@dataparty/crypto') const PeerParty = require('./peer-party') const RTCSocketComms = require('../../comms/rtc-socket-comms') +const DEFAULT_EXPIRY = 5*60*1000 + const END_STATES = [ 'cancelled', 'rejected', 'expired', 'completed' ] @@ -33,7 +35,7 @@ async function delay(ms){ } class PeerInvite extends EventEmitter { - constructor(inviteDoc, toIdentity, matchMakerClient, fromIdentity){ + constructor(inviteDoc, toIdentity, matchMakerClient, fromIdentity, payload=null){ super() this.peerParty = null @@ -42,7 +44,7 @@ class PeerInvite extends EventEmitter { this.matchMaker = matchMakerClient this.inviteDoc = inviteDoc this.inviteMsg = null //this.latestDoc = null - this.payload = null + this.payload = payload this.topicSub = null this.topicPub = null @@ -54,6 +56,21 @@ class PeerInvite extends EventEmitter { this.incomingStream = null + this.timeoutTimer = null + + this.role = null + + if(this.payload){ + this._updateRole() + const expiry = this.payload.timestamp + DEFAULT_EXPIRY + const now = Date.now() + + const delta = expiry - now + if(delta > 0){ + this.timeoutTimer = setTimeout(this.handleTimeout.bind(this)) + } + } + /*if(!this.isSender()){ this.inviteDoc. }*/ @@ -66,14 +83,26 @@ class PeerInvite extends EventEmitter { get to(){ return this.toIdentity } get from(){ return this.fromIdentity } + _updateRole(){ + + if(this.isSender()){ + + this.role = this.payload.role + return + + } + + this.role = this.payload.role == 'client' ? 'host' : 'client' + } + isSender(doc){ if(doc){ - if(doc.toHash == matchMaker.identity.key.hash){return false } + if(doc.toHash == matchMaker.client.identity.key.hash){return false } else { return true } } - if(this.inviteDoc.toHash == matchMaker.identity.key.hash){return false } + if(this.inviteDoc.toHash == matchMaker.client.identity.key.hash){return false } else { return true } } @@ -86,10 +115,10 @@ class PeerInvite extends EventEmitter { this.emit('done', this) } - async accept(mediaSrc, config){ + async accept({mediaSrc, model, hostParty, hostRunner, discoverRemoteIdentity=false}){ debug('accepting invite') - /*if(this.inviteDoc.toHash == matchMaker.wsParty.identity.key.hash){ + /*if(this.inviteDoc.toHash == this.matchMaker.client.socketPeerParty.identity.key.hash){ otherIdentity = await this.matchMaker.lookupPublicKey(this.inviteDoc.fromHash) } else { otherIdentity = await this.matchMaker.lookupPublicKey(this.inviteDoc.toHash) @@ -102,13 +131,22 @@ class PeerInvite extends EventEmitter { let msgWorkAround = new dataparty_crypto.Message({}) msgWorkAround.fromJSON(JSON.parse(changedInvite.payload)) - let payload = await this.matchMaker.identity.decrypt( + let payload = await this.matchMaker.client.identity.decrypt( msgWorkAround ) this.payload = payload.msg + this._updateRole() + + /*const expiry = this.payload.timestamp + DEFAULT_EXPIRY + const now = Date.now() - return await this.establish({mediaSrc, config}) + const delta = expiry - now + if(delta > 0 && this.timeoutTimer != null){ + this.timeoutTimer = setTimeout(this.handleTimeout.bind(this)) + }*/ + + return await this.establish({mediaSrc, model, hostParty, hostRunner, discoverRemoteIdentity}) } async reject(){ @@ -120,6 +158,10 @@ class PeerInvite extends EventEmitter { return (this.inviteMsg || this.inviteDoc).state } + async handleTimeout(){ + await this.matchMaker.setInviteState(this, 'expired') + } + async onInviteMsg(inviteMsg){ debug('onInviteMsg', inviteMsg) @@ -158,17 +200,17 @@ class PeerInvite extends EventEmitter { }) } - async establish({mediaSrc, hostParty, config, rtcSettings}){ + async establish({mediaSrc, model, hostParty, hostRunner, rtcSettings, discoverRemoteIdentity=false}){ if(!rtcSettings){ rtcSettings = {} } - let host = this.isSender() + let host = (this.role == 'host') let actorField = this.isSender() ? 'from' : 'to' let otherIdentity = this.isSender() ? this.to : this.from - let party = this.matchMaker.wsParty + let party = this.this.matchMaker.client.socketParty this.topicSub = new party.ROSLIB.Topic({ ros : party.comms.ros, @@ -190,7 +232,7 @@ class PeerInvite extends EventEmitter { let msgWorkAround = new dataparty_crypto.Message({}) msgWorkAround.fromJSON(msg.offers[i]) - let offer = await this.matchMaker.identity.decrypt(msgWorkAround) + let offer = await this.matchMaker.client.identity.decrypt(msgWorkAround) if(offer.from.hash != otherIdentity.key.hash){ debug('BAD IDENTITY') @@ -216,6 +258,10 @@ class PeerInvite extends EventEmitter { }*/ this.peerParty = new PeerParty({ + hostParty, + hostRunner, + model: hostParty.factory.model, + config: hostParty.config, comms: new RTCSocketComms({ host: this.isSender(), session: this.payload.session, @@ -230,11 +276,9 @@ class PeerInvite extends EventEmitter { config: DEFAULT_ICE_SERVERS }, trickle: rtcSettings.trickle? rtcSettings.trickle : true, - discoverRemoteIdentity: false, - remoteIdentity: otherIdentity - }), - hostParty: this.isSender() ? hostParty : undefined, - config: config ? config : hostParty.config + discoverRemoteIdentity: discoverRemoteIdentity ? discoverRemoteIdentity : false, + remoteIdentity: discoverRemoteIdentity ? undefined: otherIdentity + }) }) @@ -268,7 +312,7 @@ class PeerInvite extends EventEmitter { debug(' >> offer signal trickle', data) - const secureOffer = await this.matchMaker.identity.encrypt(data, otherIdentity) + const secureOffer = await this.matchMaker.client.identity.encrypt(data, otherIdentity) if(host && !sendFreely){ //console.log('am host') @@ -312,6 +356,7 @@ class PeerInvite extends EventEmitter { } catch (err){ console.log(err) + throw err } } diff --git a/src/party/peer/peer-party.js b/src/party/peer/peer-party.js index 889286d..0502d61 100644 --- a/src/party/peer/peer-party.js +++ b/src/party/peer/peer-party.js @@ -62,6 +62,12 @@ class PeerParty extends IParty { await this.comms.start() } + async stop(){ + if(this.comms){ + await this.comms.stop() + } + } + async handleCall(ask){ debug('handleCall') diff --git a/src/service/endpoint-context.js b/src/service/endpoint-context.js index 80c725a..272a550 100644 --- a/src/service/endpoint-context.js +++ b/src/service/endpoint-context.js @@ -15,7 +15,7 @@ class EndpointContext { * @param {Debug} options.debug Debug constructor (defaults to npm:Debug) * @param {boolean} options.sendFullErrors Enables sending full stack traces to client (defaults to false) */ - constructor({party, endpoint, req, res, input, debug=Debug, sendFullErrors=false}){ + constructor({party, endpoint, runner, req, res, input, debug=Debug, sendFullErrors=false}){ /** * @member module:Service.EndpointContext.debug @@ -27,6 +27,11 @@ class EndpointContext { */ this.endpoint = endpoint + /** + * @member module:Service.EndpointContext.runner + */ + this.runner = runner + /** * @member module:Service.EndpointContext.MiddlewareConfig */ diff --git a/src/service/endpoints/key-announce.js b/src/service/endpoints/key-announce.js new file mode 100644 index 0000000..26afb71 --- /dev/null +++ b/src/service/endpoints/key-announce.js @@ -0,0 +1,221 @@ +const Joi = require('@hapi/joi') +const Hoek = require('@hapi/hoek') +const debug = require('debug')('dataparty.endpoint.key-announce') + +const {Identity, Message, Routines} = require('@dataparty/crypto') + +//const IEndpoint = require('@dataparty/api/src/service/iendpoint') +const IEndpoint = require('../iendpoint') + +const KeyVerifier = Joi.object().keys({ + id: Joi.string().max(100), + type: Joi.string().max(300).required(), + hash: Joi.string().max(200).required(), + public: { + box: Joi.string().max(60).required(), + sign: Joi.string().max(60).required(), + pqkem: Joi.string().max(8000).required(), + pqsign_ml: Joi.string().max(8000).required(), + pqsign_slh: Joi.string().max(800).required() + } +}) + +module.exports = class KeyAnnounceEndpoint extends IEndpoint { + + static get Name(){ + return 'key/announce' + } + + static get Description(){ + return 'announce a public key' + } + + static get MiddlewareConfig(){ + return { + pre: { + decrypt: true, + validate: Joi.object().keys({ + + annoucement: { + role: Joi.string().valid('guest', 'billing').required(), + created: Joi.number().required(), + expiry: Joi.number().required(), + actorKey: KeyVerifier.required(), + sessionKey: KeyVerifier.required(), + }, + trust:{ + actorSig: Joi.string().required().description('actor signature of the annoucement in base64'), + sessionSig: Joi.string().required().description('session signature of the annoucement in base64') + } + + }).description('key to announce') + }, + post: { + encrypt: true, + validate: Joi.object().keys({ + done: Joi.boolean(), + }).description('public key') + } + } + } + + static async run(ctx, {Package}){ + + ctx.debug('hello key/announce') + + ctx.debug('ip', ctx.req.ip) + ctx.debug('input', ctx.input) + ctx.debug('sender', ctx.senderKey) + + const inputActorKey = { + type: ctx.input.annoucement.actorKey.type, + hash: ctx.input.annoucement.actorKey.hash, + public: ctx.input.annoucement.actorKey.public + } + + const inputSessionKey = { + type: ctx.input.annoucement.sessionKey.type, + hash: ctx.input.annoucement.sessionKey.hash, + public: ctx.input.annoucement.sessionKey.public + } + + + const computedActorHash = await Routines.hashKey( inputActorKey ) + const computedSessionHash = await Routines.hashKey( inputSessionKey ) + + ctx.debug('computed hash -', computedSessionHash) + + // verify keys are self consistent + if( + computedActorHash != inputActorKey.hash || + computedSessionHash != inputSessionKey.hash + ) { + ctx.debug('invalid actor or session key hash') + return {done: false} + } + + + // ensure sender is connected using session key mentioned in annoucement OR this is an internal call + if(computedSessionHash == inputSessionKey.hash && + ( + ctx.req.source == 'INTERNAL' || + //ctx.req.source == 'PeerComms' || + ( + inputSessionKey.public.sign == ctx.senderKey.public.sign && + inputSessionKey.public.box == ctx.senderKey.public.box + ) + ) + ){ + + const actorSigBson = Routines.Utils.base64.decode( ctx.input.trust.actorSig ) + const sessionSigBson = Routines.Utils.base64.decode( ctx.input.trust.sessionSig ) + + const actorSigMsg = new Message({ msg: ctx.input.annoucement }) + const sessionSigMsg = new Message({ msg: ctx.input.annoucement }) + + actorSigMsg.sig = actorSigBson + sessionSigMsg.sig = sessionSigBson + + const actorIdentity = Identity.fromJSON({ + id: '', + key: inputActorKey + }) + + const sessionIdentity = Identity.fromJSON({ + id: '', + key: inputSessionKey + }) + + //verify actor & session signature. Require postquantum signing + await actorSigMsg.assertVerified( actorIdentity, true ) + await sessionSigMsg.assertVerified( sessionIdentity, true ) + + // verify actor is allowed + const isAllowed = (await ctx.runner.auth.isAdmin(actorIdentity)) || + (await ctx.runner.auth.isSocketConnectionAllowed(actorIdentity)) + if(!isAllowed){ + ctx.debug('non-allowed user') + return {done: false} + } + + + let sessionKeyDoc = (await ctx.party.find() + .type('session_key') + .where('annoucement.sessionKey.hash').equals(computedSessionHash) + .exec())[0] + + if(!sessionKeyDoc){ + // create session document + + const now = Date.now() + const tomorrow = now + 24*60*60*1000 + + const fiveMinAgo = now - (5*1000*60) + const fiveMinFromNow = now + (5*1000*60) + + // verify start time is valid + if( + ctx.input.annoucement.created < fiveMinAgo || + ctx.input.annoucement.created > fiveMinFromNow + ) { + ctx.debug('invalid start time') + return {done: false} + } + + // verify expiry time is valid + if( + ctx.input.annoucement.expiry < now || + ctx.input.annoucement.expiry > tomorrow + ) { + ctx.debug('invalid expiry time') + return {done: false} + } + + ctx.debug('opening session -', computedSessionHash) + + let sessionDoc = await ctx.party.createDocument('session_key', { + created: now, + expiry: ctx.input.annoucement.expiry, + annoucement: ctx.input.annoucement, + trust: ctx.input.trust + }) + + ctx.debug('session created - ', computedSessionHash) + + } else { + ctx.debug('session key already known - ', computedSessionHash) + return {done: false} + } + + let publicKey = (await ctx.party.find() + .type('public_key') + .where('hash').equals(computedActorHash) + .exec())[0] + + if(!publicKey){ + + ctx.debug('annoucing key -', computedActorHash) + + let keyDoc = await ctx.party.createDocument('public_key', { + created: Date.now(), + role: ctx.input.annoucement.role || 'guest', + owner: computedActorHash, + ...inputActorKey + }) + + ctx.debug('actor annouced key -', computedActorHash) + } else { + ctx.debug('key already known -', computedActorHash) + } + + return {done: true} + + } else { + + ctx.debug('announce ERROR - BAD KEY HASH') + + return {done: false} + } + + } +} diff --git a/src/service/endpoints/service-version.js b/src/service/endpoints/service-version.js index b84c91e..ab5ed11 100644 --- a/src/service/endpoints/service-version.js +++ b/src/service/endpoints/service-version.js @@ -27,7 +27,8 @@ module.exports = class ServiceVersion extends IEndpoint { name: Joi.string(), branch: Joi.string(), version: Joi.string(), - githash: Joi.string() + githash: Joi.string(), + owner: Joi.string(), }) } } diff --git a/src/service/iauth.js b/src/service/iauth.js index c83ef0e..5db6e17 100644 --- a/src/service/iauth.js +++ b/src/service/iauth.js @@ -37,11 +37,16 @@ module.exports = class IAuth { return identity } + async isPeerConnectionAllowed(identity){ + return true + } + async isSocketConnectionAllowed(identity){ //throw new Error('not implemented') return true } + async isInternal(identity){ return false } diff --git a/src/service/index-browser.js b/src/service/index-browser.js index e026bf2..d33a83a 100644 --- a/src/service/index-browser.js +++ b/src/service/index-browser.js @@ -50,4 +50,12 @@ exports.endpoint_paths = { secureecho: Path.join(__dirname, './endpoints/secure-echo.js'), identity: Path.join(__dirname, './endpoints/service-identity.js'), version: Path.join(__dirname, './endpoints/service-version.js') +} + +exports.task = { + cleanup_ephemeral_sessions: require('./tasks/cleanup-ephemeral-sessions.js') +} + +exports.task_paths = { + cleanup_ephemeral_sessions: Path.join(__dirname, './tasks/cleanup-ephemeral-sessions.js') } \ No newline at end of file diff --git a/src/service/index.js b/src/service/index.js index bf2ed65..0472ce0 100644 --- a/src/service/index.js +++ b/src/service/index.js @@ -24,7 +24,8 @@ exports.MiddlewareRunner= require('./middleware-runner') exports.middleware = { pre: { decrypt: require('./middleware/pre/decrypt'), - validate: require('./middleware/pre/validate') + validate: require('./middleware/pre/validate'), + ephemeral_session: require('./middleware/pre/ephemeral-session.js') }, post: { validate: require('./middleware/post/validate.js'), @@ -35,7 +36,8 @@ exports.middleware = { exports.middleware_paths = { pre: { decrypt: Path.join(__dirname, './middleware/pre/decrypt.js'), - validate: Path.join(__dirname, './middleware/pre/validate.js') + validate: Path.join(__dirname, './middleware/pre/validate.js'), + ephemeral_session: Path.join(__dirname, './middleware/pre/ephemeral-session.js') }, post: { validate: Path.join(__dirname, './middleware/post/validate.js'), @@ -47,12 +49,33 @@ exports.endpoint = { echo: require('./endpoints/echo'), secureecho: require('./endpoints/secure-echo'), identity: require('./endpoints/service-identity'), - version: require('./endpoints/service-version') + version: require('./endpoints/service-version'), + key_announce: require('./endpoints/key-announce') } exports.endpoint_paths = { echo: Path.join(__dirname, './endpoints/echo.js'), secureecho: Path.join(__dirname, './endpoints/secure-echo.js'), identity: Path.join(__dirname, './endpoints/service-identity.js'), - version: Path.join(__dirname, './endpoints/service-version.js') + version: Path.join(__dirname, './endpoints/service-version.js'), + key_announce: Path.join(__dirname, './endpoints/key-announce.js') +} + +exports.schema = { + public_key: require('./schema/public-key'), + session_key: require('./schema/session-key') +} + +exports.schema_paths = { + public_key: Path.join(__dirname, './schema/public-key.js'), + session_key: Path.join(__dirname, './schema/session-key.js') +} + + +exports.task = { + cleanup_ephemeral_sessions: require('./tasks/cleanup-ephemeral-sessions.js') +} + +exports.task_paths = { + cleanup_ephemeral_sessions: Path.join(__dirname, './tasks/cleanup-ephemeral-sessions.js') } \ No newline at end of file diff --git a/src/service/iservice.js b/src/service/iservice.js index b9f6c01..2d46caf 100644 --- a/src/service/iservice.js +++ b/src/service/iservice.js @@ -17,7 +17,7 @@ module.exports = class IService { * @param {*} build */ constructor({ - name, version, githash='', branch='' + name, version, githash='', branch='', owner=null }, build){ this.constructors = { @@ -48,11 +48,13 @@ module.exports = class IService { }, tasks: {}, topics: {}, - auth: null + auth: null, + files: [], + files_root: null } this.compiled = { - package: { name, version, githash, branch }, + package: {owner, name, version, githash, branch }, schemas: { IndexSettings: {}, JSONSchema: [], @@ -70,7 +72,9 @@ module.exports = class IService { }, tasks: {}, topics: {}, - auth: {} + auth: {}, + files: {}, + //signatures: {} } this.compileSettings = { diff --git a/src/service/middleware/post/encrypt.js b/src/service/middleware/post/encrypt.js index a0912d8..09dbe3d 100644 --- a/src/service/middleware/post/encrypt.js +++ b/src/service/middleware/post/encrypt.js @@ -30,7 +30,14 @@ module.exports = class Encrypt extends IMiddleware { static async run(ctx, {Config}){ if (!Config){ return } - + + if(!ctx.req.source && + ( ctx.req.source == 'PeerComms' || + ctx.req.source == 'INTERNAL' ) + ){ + ctx.setOutput(ctx.output) + return + } const senderStr = JSON.stringify({key: ctx.senderKey}) diff --git a/src/service/middleware/pre/decrypt.js b/src/service/middleware/pre/decrypt.js index 20adf0c..195b521 100644 --- a/src/service/middleware/pre/decrypt.js +++ b/src/service/middleware/pre/decrypt.js @@ -31,8 +31,19 @@ module.exports = class Decrypt extends IMiddleware { if (!Config){ return } + + if(!context.input || !context.input.enc){ - throw new Error('insecure message') + + if(!context.req.source || + ( context.req.source != 'PeerComms' && + context.req.source != 'INTERNAL' ) + ){ + throw new Error('insecure message -' + context.req.source) + } + + context.setInput( context.input ) + return } context.debug('input', context.input, typeof context.input) diff --git a/src/service/middleware/pre/ephemeral-session.js b/src/service/middleware/pre/ephemeral-session.js new file mode 100644 index 0000000..fd099cf --- /dev/null +++ b/src/service/middleware/pre/ephemeral-session.js @@ -0,0 +1,101 @@ +const Joi = require('joi') +const Hoek = require('@hapi/hoek') +const {Identity} = require('@dataparty/crypto') +const debug = require('debug')('dataparty.middleware.pre.ephemeral-session') + +const IMiddleware = require('../../imiddleware') + +module.exports = class Decrypt extends IMiddleware { + + static get Name(){ + return 'ephemeral_session' + } + + static get Type(){ + return 'pre' + } + + static get Description(){ + return 'Decrypt inbound data' + } + + static get ConfigSchema(){ + return Joi.boolean() + } + + static async start(party){ + + } + + static async run(ctx, {Config}){ + + if (!Config){ return } + + if(!ctx.input_session_id){ + throw new Error('no session id') + } + + ctx.debug('looking up session -', ctx.input_session_id) + + let sessionKeyDoc = (await ctx.party.find() + .type('session_key') + .where('annoucement.sessionKey.hash') + .equals(ctx.input_session_id) + .exec() + )[0] + + if(!sessionKeyDoc){ + throw new Error('invalid session') + } + + ctx.debug('located session - ', ctx.input_session_id) + + const sessionKey = sessionKeyDoc.data.annoucement.sessionKey + const actorKey = sessionKeyDoc.data.annoucement.actorKey + + // ensure sender is connected using session key mentioned in db + if(sessionKey.hash == ctx.input_session_id && + sessionKey.public.sign == ctx.senderKey.public.sign && + sessionKey.public.box == ctx.senderKey.public.box + ){ + + const now = Date.now() + + if( sessionKeyDoc.data.expiry < now ){ + throw new Error('session expired!') + } + + const actorIdentity = Identity.fromJSON({ + id: 'actor', + key: actorKey + }) + + const sessionIdentity = Identity.fromJSON({ + id: 'ephemeral-session', + key: sessionKey + }) + + ctx.debug('looking up actor - ', actorKey.hash) + + let actorDoc = (await ctx.party.find() + .type('public_key') + .where('hash') + .equals(actorKey.hash) + .exec() + )[0] + + if(!actorDoc){ + throw new Error('failed to find actor') + } + + ctx.setSession( sessionKeyDoc ) + ctx.setIdentity( actorIdentity ) + ctx.setActor( actorDoc ) + + ctx.debug('session ready') + } else { + throw new Error('invalid sender key for this session') + } + + } +} \ No newline at end of file diff --git a/src/service/runner-router.js b/src/service/runner-router.js index c6897b8..24ae4dc 100644 --- a/src/service/runner-router.js +++ b/src/service/runner-router.js @@ -69,7 +69,7 @@ class RunnerRouter { * @returns {module:Service.ServiceRunner} */ getRunnerByHostIdentity(identity){ - const partyId = identity.toString() + const partyId = typeof identity !== 'string' ? identity.key.hash : identity debug('getRunnerByHostIdentity -', partyId) const runner = this.runnersByHost.get(partyId) @@ -83,7 +83,7 @@ class RunnerRouter { */ addRunner({domain, runner}){ - const partyId = runner.party.identity.toString() + const partyId = runner.party.identity.key.hash debug('addRunner - ', partyId, domain) if(!this.runnersByHost.has(partyId)){ diff --git a/src/service/schema/public-key.js b/src/service/schema/public-key.js new file mode 100644 index 0000000..597601b --- /dev/null +++ b/src/service/schema/public-key.js @@ -0,0 +1,47 @@ +'use strict' + +const ISchema = require('../../bouncer/ischema') + +class PublicKey extends ISchema { + + static get Type () { return 'public_key' } + + static get Schema(){ + return { + created: { + type: Number, + required: true + }, + + role: String, // [ guest, billing, service, wallet ] + + owner: {required: true, index: true, type: String}, // public key hash + + type: String, + hash: {required: true, index: true, type: String, unique: true}, + public: { + box: String, + sign: String, + pqkem: String, + pqsign_ml: String, + pqsign_slh: String + } + } + } + + static setupSchema(schema){ + //schema.index({ 'hash': 1 }, {unique: true}) + return schema + } + + static permissions (context) { + return { + read: false, + new: false, + change: false + } + } +} + + +module.exports = PublicKey diff --git a/src/service/schema/session-key.js b/src/service/schema/session-key.js new file mode 100644 index 0000000..a0da93c --- /dev/null +++ b/src/service/schema/session-key.js @@ -0,0 +1,69 @@ +'use strict' + +const ISchema = require('../../bouncer/ischema') + + +function PublicKeySchema(unique=true){ + return { + type: {required: true, type: String}, + hash: {required: true, index: true, type: String, unique}, + public: { + box: String, + sign: String, + pqkem: String, + pqsign_ml: String, + pqsign_slh: String + } + } +} + + +class SessionKey extends ISchema { + + static get Type () { return 'session_key' } + + static get Schema(){ + return { + created: { + type: Number, + required: true + }, + expiry: { + type: Number, + index: true, + required: true + }, + annoucement: { + created: { + type: Number, + required: true + }, + expiry: { + type: Number, + required: true + }, + sessionKey: PublicKeySchema(true), + actorKey: PublicKeySchema(false) + }, + trust: { + actorSig: {required: true, type: String}, //! base64 of BSON signature + sessionSig: {required: true, type: String} //! base64 of BSON signature + } + } + } + + static setupSchema(schema){ + return schema + } + + static permissions (context) { + return { + read: false, + new: false, + change: false + } + } +} + + +module.exports = SessionKey \ No newline at end of file diff --git a/src/service/service-builder.js b/src/service/service-builder.js index 614afd9..0598e4c 100644 --- a/src/service/service-builder.js +++ b/src/service/service-builder.js @@ -7,6 +7,18 @@ const gitRepoInfo = require('git-repo-info') const BouncerDb = require('@dataparty/bouncer-db') const mongoose = BouncerDb.mongoose() const debug = require('debug')('dataparty.service.ServiceBuilder') +const zlib = require('zlib') + +const safeStringify = require('fast-safe-stringify') + +const dataparty_crypto = require('@dataparty/crypto') + +const { + globSync +} = require('glob') +const { isArray } = require('lodash') + +const tar = require('tar') //const IService = require('../iservice') @@ -27,7 +39,7 @@ module.exports = class ServiceBuilder { * @param {boolean} writeFile When true, files will be written. Defaults to `true` * @returns */ - async compile(outputPath, writeFile=true){ + async compile(outputPath, writeFile=true, owner){ if(!outputPath){ throw new Error('no output path') @@ -40,7 +52,7 @@ module.exports = class ServiceBuilder { debug('compiling sources',this.service.sources) - await Promise.all([ + let results = await Promise.all([ this.compileMiddleware('pre'), this.compileMiddleware('post'), this.compileList('documents'), @@ -48,25 +60,57 @@ module.exports = class ServiceBuilder { this.compileList('tasks'), this.compileList('topics'), this.compileFile('auth'), - this.compileSchemas() + this.compileSchemas(), + this.compressFiles(outputPath, writeFile) ]) + + this.service.compiled.middleware_order = this.service.middleware_order this.service.compiled.compileSettings = this.service.compileSettings + if(owner){ + this.service.compiled.package.owner = owner.key.hash + const ownerSig = await owner.sign(this.service.compiled, true) + + console.log('sign keys', Object.keys(this.service.compiled)) + console.log(this.service.compiled.signature) + + this.service.compiled.signatures = { + [owner.key.hash]: dataparty_crypto.Routines.Utils.base64.encode(ownerSig.sig) + } + } + + let files = [] + if(writeFile){ - const buildOutput = outputPath+'/'+ this.service.compiled.package.name.replace('/', '-') +'.dataparty-service.json' + const buildOutput = outputPath+'/'+ this.service.compiled.package.name.replace('/', '-') +'.service.venue.json' fs.writeFileSync(buildOutput, JSON.stringify(this.service.compiled, null,2)) - const schemaOutput = outputPath+'/'+ this.service.compiled.package.name.replace('/', '-') +'.dataparty-schema.json' + const schemaOutput = outputPath+'/'+ this.service.compiled.package.name.replace('/', '-') +'.schema.venue.json' fs.writeFileSync(schemaOutput, JSON.stringify({ package: this.service.compiled.package, ...this.service.compiled.schemas }, null, 2)) + + // Gzip compression (most common for HTTP) + const compressed = zlib.gzipSync(JSON.stringify(this.service.compiled, null,2)); + + // Brotli compression (better ratio, Node.js 10.5.0+) + const compressedBrotli = zlib.brotliCompressSync(JSON.stringify(this.service.compiled, null,2)); + + console.log('Original:', JSON.stringify(this.service.compiled, null,2).length, 'bytes'); + console.log('Gzip:', compressed.length, 'bytes') + console.log('Brotli:', compressedBrotli.length, 'bytes') + + let tarFile = results[ results.length - 1 ] + files.push(buildOutput) + files.push(schemaOutput) + if(tarFile){files.push(tarFile)} } - return this.service.compiled + return {build: this.service.compiled, files} } @@ -142,24 +186,33 @@ module.exports = class ServiceBuilder { this.service.compiled.schemas.Permissions[model.Type] = await model.permissions() this.service.compiled.schemas.JSONSchema.push(jsonSchema) + const safePaths = JSON.parse(safeStringify(schema.paths)) + + //debug(schema.paths) debug('\t','type',model.Type) let indexed = JSONPath({ path: '$..options.index', - json: schema.paths, + json: safePaths, resultType: 'pointer' - }).map(p=>{return p.split('/')[1]}) + }).map(p=>{ + + debug('\t\t','indexed p',p) + return p.replace('/options/index', '').replace('/','') + }) + //return p.split('.')[1]}) debug('\t\tindexed', indexed) let unique = JSONPath({ path: '$..options.unique', - json: schema.paths, + json: safePaths, resultType: 'pointer' }).map(p=>{ - debug(typeof p) + debug(typeof p, 'unique', p) if(typeof p == 'string'){ - return p.split('/')[1] + let filteredP = p.replace('/options/unique', '').replace('/','') + return filteredP } return p @@ -315,4 +368,70 @@ module.exports = class ServiceBuilder { this.service.sources.auth = auth_path this.service.constructors.auth = TopicClass } + + addFiles(root, pattern, options){ + + let result = globSync(pattern, { + dotRelative: true, + cwd:root, + ...options + }) + + if(!this.service.files){ + this.service.sources.files = result + } else { + this.service.sources.files = this.service.sources.files.concat(result) + } + + this.service.sources.files_root = root + + debug('addFiles',result) + + } + + async compressFiles(outputPath, writeFile){ + + if(!this.service.sources.files){ return } + + let fileMap={} + + let files = this.service.sources.files.map(file=>{ + // + const content = fs.readFileSync(file) + const hash = dataparty_crypto.Routines.Utils.base64.encode( + dataparty_crypto.Routines.Utils.hash(content) + ) + + fileMap[file] = { hash, size: content.length } + + return hash + }) + + if(!files || files.length < 1){ return } + + const tarFileName = this.service.compiled.package.name.replace('/', '-')+'.files.venue.tgz' + const tarPath = Path.join(outputPath, tarFileName) + + await tar.create({ + cwd: this.service.sources.files_root, + gzip: true, + file: tarPath + }, this.service.sources.files) + + const staticTar = fs.readFileSync(tarPath) + + let tarHash = dataparty_crypto.Routines.Utils.hash( staticTar ) + let tarHash64 = dataparty_crypto.Routines.Utils.base64.encode(tarHash) + + this.service.compiled.files = { + [tarFileName]: { + tar: tarFileName, + hash:tarHash64, + size: staticTar.length, + files: fileMap + } + } + + return tarPath + } } \ No newline at end of file diff --git a/src/service/service-host-peer.js b/src/service/service-host-peer.js new file mode 100644 index 0000000..03c37a1 --- /dev/null +++ b/src/service/service-host-peer.js @@ -0,0 +1,77 @@ +const debug = require('debug')('dataparty.service.host-peer') + +class ServiceHostPeer { + + constructor({ + runner, + matchMaker, + mediaSrc, + discoverRemoteIdentity = false + }){ + this.runner = runner + this.matchMaker = matchMaker + this.mediaSrc = mediaSrc + this.discoverRemoteIdentity = discoverRemoteIdentity + } + + async start(){ + // + + this.matchMaker.on('invited', this.onInvite.bind(this)) + } + + async onInvite(invite){ + + + // Filter out none host mode requests + if(invite.role !== 'host'){ + debug('FAIL - unexpected role[', invite.role, '] we expecte to be host') + await invite.reject() + return + } + + let hostRunner = this.runner.party ? this.runner : this.runner.getRunnerByHostIdentity(invite.to) + + // Make sure we know the requested party & runner + if(!hostRunner){ + debug('FAIL - requested party not available', invite.to) + await invite.reject() + return + } + + //! Check if party wants to allow/deny invite sender + const isAllowed = (await hostRunner.auth.isAdmin(invite.from)) || + (await hostRunner.auth.isPeerConnectionAllowed(invite.from)) || + (await hostRunner.auth.isSocketConnectionAllowed(invite.from)) + if(!isAllowed){ + debug('NOT ALLOWED - user is not allowed', invite.from) + await invite.reject() + return + } + + + //! Announce the session key + const annoucement = invite.payload.info + const result = await hostRunner.internalRequest('key/announce', annoucement) + + debug('annoucement result', result) + + if(!result || !result.done){ + + debug('user session not allowed') + + return + } + + let hostParty = hostRunner.party + + const peerParty = await invite.accept({ + mediaSrc: this.mediaSrc, + hostParty, + hostRunner, + discoverRemoteIdentity: this.discoverRemoteIdentity //! todo - this is probably something a party config could reasonably over ride + }) + } +} + +module.exports = ServiceHostPeer \ No newline at end of file diff --git a/src/service/service-host.js b/src/service/service-host.js index 6fd1c68..85bb43e 100644 --- a/src/service/service-host.js +++ b/src/service/service-host.js @@ -58,8 +58,9 @@ class ServiceHost { i2pSamHost = '127.0.0.1', i2pSamPort = 7656, i2pKey = null, - i2pForwardHost = 'localhost', + i2pForwardHost = '127.0.0.1', i2pForwardPort = null, + i2pOptions = null, wsEnabled = true, wsPort = null, wsUpgradePath = '/ws', @@ -113,8 +114,8 @@ class ServiceHost { if(debug.enabled){ this.apiApp.use(morgan('combined')) } this.apiApp.use(bodyParser.urlencoded({ extended: true })) - this.apiApp.use(bodyParser.json()) - this.apiApp.use(bodyParser.raw()) + this.apiApp.use(bodyParser.json({limit:'10MB'})) + this.apiApp.use(bodyParser.raw({limit:'10MB'})) this.apiApp.set('trust proxy', trust_proxy) @@ -141,11 +142,17 @@ class ServiceHost { host: i2pSamHost, portTCP: i2pSamPort, publicKey: reach(i2pKey, 'publicKey'), - privateKey: reach(i2pKey, 'privateKey') + privateKey: reach(i2pKey, 'privateKey'), + versionMin: '3.1', + versionMax: '3.3' }, forward: { + silent: true, host: i2pForwardHost ? i2pForwardHost : this.apiServerUri.hostname, port: i2pForwardPort ? i2pForwardPort : parseInt( this.apiServerUri.port ) + }, + session: { + options: i2pOptions } } } @@ -246,29 +253,32 @@ class ServiceHost { if(this.i2pEnabled && this.i2p == null){ debug('starting i2p forward', this.i2pSettings) - const SAM = require('@diva.exchange/i2p-sam') - this.i2p = await SAM.createForward(this.i2pSettings) - this.i2pUri = this.i2p.getB32Address() - this.i2pSettings.privateKey = null // clear no longer needed + async function setup_i2p(){ + const SAM = require('@diva.exchange/i2p-sam') + this.i2p = await SAM.createForward(this.i2pSettings) + this.i2pUri = this.i2p.getB32Address() + this.i2pSettings.sam.privateKey = null // clear no longer needed + this.i2p.on('error', this.reportI2pError.bind(this)) - this.i2p.on('error', this.reportI2pError.bind(this)) + this.i2p.on('close', ()=>{ + debug('i2p closed') + }) - this.i2p.on('close', ()=>{ - debug('i2p closed') - }) + this.i2p.on('data', (data)=>{ + debug('i2p data') + debug(data.toString()) + }) - this.i2p.on('data', (data)=>{ - debug('i2p data') - debug(data.toString()) - }) + debug('i2p started') + debug('\t', 'address', this.i2pUri) + debug('\t', 'key', this.i2p.getPublicKey()) + } - debug('i2p started') - debug('\t', 'address', this.i2pUri) - debug('\t', 'key', this.i2p.getPublicKey()) + setup_i2p.bind(this)() } if(this.mdnsEnabled && this.apiServer && this.apiServerUri.protocol != 'file:'){ @@ -307,7 +317,7 @@ class ServiceHost { this.apiApp.use((err, req, res, _next) => { console.log('Error handler', err) if (err instanceof IpDeniedError) { - //res.status(401) + res.status(401) } else { res.status(err.status || 500) } diff --git a/src/service/service-runner-node.js b/src/service/service-runner-node.js index 5df899c..7d93b19 100644 --- a/src/service/service-runner-node.js +++ b/src/service/service-runner-node.js @@ -5,6 +5,7 @@ const Debug = require('debug') const debug = Debug('dataparty.service.runner-node') const EndpointContext = require('./endpoint-context') const DeltaTime = require('../utils/delta-time') +const HttpMocks = require('node-mocks-http') const Router = require('origin-router').Router const Runner = require('@dataparty/tasker').Runner @@ -110,13 +111,13 @@ class ServiceRunnerNode { let AuthClass = null - if(!this.useNative){ + if(!this.useNative && Hoek.reach(this.service, `compiled.auth`)){ var self={} const build = Hoek.reach(this.service, `compiled.auth`) eval(build.code/*, build.map*/) AuthClass = self.Lib } - else{ + else if(this.service.constructors.auth){ AuthClass = this.service.constructors.auth } @@ -446,6 +447,38 @@ class ServiceRunnerNode { } } + async internalRequest(endpoint, data){ + let bodyValue = data + + const req = HttpMocks.createRequest({ + method: 'GET', + url: '/'+endpoint, + body: bodyValue + }) + + const res = HttpMocks.createResponse() + + debug('\tthe request', req) + + debug('req ip type', typeof req.ip) + + const route = this.router.get(endpoint) + + debug('route',route) + + req.runner = this + req.source = 'INTERNAL' + + debug('call route', await route._events.route({ + method: req.method, + pathname: req.url, + request: req, + response: res + })) + + return {result: res._getData() } + } + async getTopic(path){ debug('looking up topic', path) @@ -516,6 +549,7 @@ class ServiceRunnerNode { req: event.request, res: event.response, endpoint, party: this.party, + runner: this, input: event.request.body, debug: Debug, sendFullErrors: this.sendFullErrors diff --git a/src/service/tasks/cleanup-ephemeral-sessions.js b/src/service/tasks/cleanup-ephemeral-sessions.js new file mode 100644 index 0000000..18cb7d0 --- /dev/null +++ b/src/service/tasks/cleanup-ephemeral-sessions.js @@ -0,0 +1,86 @@ +const debug = require('debug')('dataparty.task.cleanup-ephemeral-sessions') + +//const ITask = require('@dataparty/api/src/service/itask') +const ITask = require('../../service/itask') + +class CleanupEphemeralSessionsTask extends ITask { + + constructor(options){ + super({ + name: CleanupEphemeralSessionsTask.name, + background: CleanupEphemeralSessionsTask.Config.background, + ...options + }) + + debug('new') + + this.duration = Math.round(1000*60*15) + this.timeout = null + } + + static get Config(){ + return { + background: true, + autostart: true + } + } + + async exec(){ + + this.setTimer() + + return this.detach() + } + + + async lookupSessions(){ + + let now = Date.now() + + return (await this.context.party.find() + .type('session_key') + .where('expiry').lt(now) + .exec()) + } + + setTimer(){ + this.timeout = setTimeout(this.onTimeout.bind(this), this.duration) + } + + async onTimeout(){ + this.timeout = null + + debug('cleanup ephemeral sessions task') + + try{ + let sessions = await this.lookupSessions() + + if(sessions && sessions.length > 0){ + debug('expired sessions ', sessions.length) + let list = sessions.map(i=>{return i.data}) + await this.context.party.remove(...list) + } + } catch (err){ + debug(err) + } + + this.setTimer() + } + + stop(){ + if(this.timeout !== null){ + clearTimeout(this.timeout) + this.timeout = null + } + } + + static get Name(){ + return 'cleanup-ephemeral-sessions' + } + + static get Description(){ + return 'Cleanup Ephemeral sessions' + } +} + +module.exports = CleanupEphemeralSessionsTask diff --git a/src/venue/auth.js b/src/venue/auth.js index c2ff01c..7b8074c 100644 --- a/src/venue/auth.js +++ b/src/venue/auth.js @@ -1,6 +1,7 @@ const debug = require('debug')('dataparty.auth.venue-auth') const IAuth = require('../service/iauth') +const {Identity} = require('@dataparty/crypto') module.exports = class IAuth { @@ -36,12 +37,28 @@ module.exports = class IAuth { } async lookupIdentity(identity){ + + let sessionKeyDoc = (await this.context.party.find() + .type('session_key') + .where('annoucement.sessionKey.hash') + .equals(identity.key.hash) + .exec() + )[0] + + if(sessionKeyDoc){ + const actorIdentity = Identity.fromJSON({ + id: 'actor', + key: sessionKeyDoc.data.annoucement.actorKey + }) + + return actorIdentity + } + return identity } async isSocketConnectionAllowed(identity){ - //throw new Error('not implemented') - return true + return await this.isAdmin(identity) } async isInternal(identity){ @@ -49,7 +66,16 @@ module.exports = class IAuth { } async isAdmin(identity){ - return false + + // verify key-hash is an admin + const admins = (await this.context.party.config.read('admins')) || [] + + if(admins.indexOf(identity.key.hash) == -1){ + debug('non-admin user', identity.key.hash) + return false + } + + return true } async canReadDb(identity){ diff --git a/src/venue/bin/add-admin.js b/src/venue/bin/add-admin.js new file mode 100755 index 0000000..f09375e --- /dev/null +++ b/src/venue/bin/add-admin.js @@ -0,0 +1,40 @@ +#!/usr/bin/env node + +const Dataparty = require('../../index') + + +async function main(){ + + const path = '/data/dataparty/venue-service' + + let config = new Dataparty.Config.JsonFileConfig({ + basePath: path+'/config' + }) + + await config.start() + + console.log(process.argv) + + let admins = (await config.read('admins')) || [] + + const newAdmin = process.argv[2] + + console.log(await config.readAll()) + console.log(admins) + + if(admins.indexOf(newAdmin) != -1){ return } + + admins.push(newAdmin) + + await config.write('admins', admins) + + console.log('admin added -', newAdmin) + +} + + +main().catch(err=>{ + console.error(err) +}).finally(()=>{ + process.exit() +}) diff --git a/src/venue/bin/chill.js b/src/venue/bin/chill.js new file mode 100644 index 0000000..e69de29 diff --git a/src/venue/bin/commands/pkg-build.js b/src/venue/bin/commands/pkg-build.js new file mode 100644 index 0000000..ee297ba --- /dev/null +++ b/src/venue/bin/commands/pkg-build.js @@ -0,0 +1,170 @@ +const CmdTree = require('command-tree') +const Hoek = require('@hapi/hoek') +const debug = require('debug')('venue.package-build') +const Path = require('path') +const OS = require('os') +const fs = require('fs') +const mkdirp = require('mkdirp') + +const prompt = require('prompt') +const argon2 = require('argon2') + +const { execSync } = require('child_process') +const findUp = require('find-up-json').default + + +const Dataparty = require('../../../../') +const dataparty_crypto = require('@dataparty/crypto') + +const DEFINITION = { + h: { + description: 'Show help', + alias: 'help', + type: 'help' + }, + o: { + alias: 'output', + type: 'string', + default: Path.join(process.cwd(), 'dist/') + }, + nopassword: { + type: 'boolean', + default: false + }, + identity: { + type: 'string', + description: 'developer release identity', + require: true + }, + name: { + description: 'package name' + }, + version: { + description: 'package version' + }, + remote: { + type: 'string', + description: 'name of remote to build project for' + }, + deploy: { + type: 'boolean', + default: false + } +} + + +class VenuePackageBuild extends CmdTree.Command { + constructor(context){ + super({...VenuePackageBuild.Definition, context}) + debug('constructor') + } + + static get Command(){ + return 'package build' + } + + static get Definition(){ + return { + usage: `venue package build [service-code.js]`, + description: 'Build a package', + definition: DEFINITION + } + } + + async run({parsed}){ + //debug('context -', this.context) + //console.log('parsed -', parsed) + + if (parsed.h) { + throw new CmdTree.Error.HelpRequest('help request') + } + + if (parsed._.length != 3){ + throw new CmdTree.Error.UsageError('You must supply service code') + } + + const keyName = parsed.identity + + const phrase = await this.context.secureConfig.read('identity.'+keyName+'.phrase') + + if(!phrase){ + throw new CmdTree.Error.UsageError("Key doesn't exist!") + } + + const {password} = parsed.nopassword ? {password:null} : await prompt.get({ + properties: { + password: { + message: 'Enter password for identity['+keyName+']', + hidden: true + } + }}) + + let key = await dataparty_crypto.Identity.fromMnemonic(phrase, password, argon2) + + key.id = keyName + + const serviceClassPath = Path.resolve(parsed._[2]) + + let foundUp = findUp('package.json', Path.dirname(serviceClassPath)) + + let pkgJson = foundUp.content + + const ServiceClass = require( serviceClassPath ) + + const service = new ServiceClass({ + name: parsed.name ? parsed.name : pkgJson.name, + version: parsed.version ? parsed.version : pkgJson.version, + }) + + await mkdirp(parsed.output) + + const builder = new Dataparty.ServiceBuilder(service) + const build = await builder.compile(parsed.output, true, key) + + + if(parsed.deploy && parsed.remote){ + console.log('uploading...') + + const remote = await this.context.secureConfig.read('remote.'+parsed.remote) + + if(!remote){ + throw 'invalid remote ['+parsed.remote+']' + } + + let staticTar = undefined + + if(build.files.length == 3){ + staticTar = fs.readFileSync(build.files[ build.files.length - 1 ]) + } + + await this.pushPackage(key, remote, build.build, staticTar) + } + + return {files: build.files} + } + + + async pushPackage(devId, remote, build, staticTar=null){ + + let client = new Dataparty.EphemeralClient({ + identity: devId, + urlOrParty: remote.url, + wsUrlOrParty: remote.ws + }) + + await client.start() + + + let uploadResult = await client.restParty.comms.call('create-package', {build, staticTar}, { + expectClearTextReply: false, + sendClearTextRequest: false, + useSessions: true + }) + + console.log('result', uploadResult) + } +} + +module.exports = VenuePackageBuild + + diff --git a/src/venue/bin/commands/project-build.js b/src/venue/bin/commands/project-build.js new file mode 100644 index 0000000..0a9d8eb --- /dev/null +++ b/src/venue/bin/commands/project-build.js @@ -0,0 +1,283 @@ +const CmdTree = require('command-tree') +const Hoek = require('@hapi/hoek') +const debug = require('debug')('venue.project-build') +const Path = require('path') +const OS = require('os') +const fs = require('fs') +const mkdirp = require('mkdirp') + +const prompt = require('prompt') +const argon2 = require('argon2') + +const { execSync } = require('child_process') + +const { + globSync +} = require('glob') +const tar = require('tar') + + +const Dataparty = require('../../../../') +const dataparty_crypto = require('@dataparty/crypto') +const Joi = require('joi') + +const {Routines} = dataparty_crypto + +const DEFINITION = { + h: { + description: 'Show help', + alias: 'help', + type: 'help' + }, + o: { + alias: 'output', + type: 'string', + default: Path.join(process.cwd(), 'dist/') + }, + nopassword: { + type: 'boolean', + default: false + }, + identity: { + type: 'string', + description: 'developer release identity', + require: true + }, + name: { + description: 'project name' + }, + version: { + description: 'project version' + }, + remote: { + type: 'string', + description: 'name of remote to build project for' + }, + deploy: { + type: 'boolean', + default: false + } +} + + +async function compressFiles(projectName, root, fileList, outputPath, writeFile){ + + if(!fileList){ return } + + let fileMap={} + + let files = fileList.map(file=>{ + // + const content = fs.readFileSync(file) + const hash = dataparty_crypto.Routines.Utils.base64.encode( + dataparty_crypto.Routines.Utils.hash(content) + ) + + fileMap[file] = { hash, size: content.length } + + return hash + }) + + if(!files || files.length < 1){ return } + + const tarFileName = projectName.replace('/', '-')+'.project.files.venue.tgz' + const tarPath = Path.join(outputPath, tarFileName) + + await tar.create({ + cwd: root, + gzip: true, + file: tarPath + }, fileList) + + const staticTar = fs.readFileSync(tarPath) + + let tarHash = dataparty_crypto.Routines.Utils.hash( staticTar ) + let tarHash64 = dataparty_crypto.Routines.Utils.base64.encode(tarHash) + + const fileInfo = { + [tarFileName]: { + tar: tarFileName, + hash:tarHash64, + size: staticTar.length, + files: fileMap + } + } + + return {tarPath, files: fileInfo} +} + + +class VenueProjectBuild extends CmdTree.Command { + constructor(context){ + super({...VenueProjectBuild.Definition, context}) + debug('constructor') + + this.project = {} + this.project_sources = [] + } + + static get Command(){ + return 'project build' + } + + static get Definition(){ + return { + usage: `venue project build [project.json]`, + description: 'Build a project', + definition: DEFINITION + } + } + + async run({parsed}){ + //debug('context -', this.context) + //console.log('parsed -', parsed) + + if (parsed.h) { + throw new CmdTree.Error.HelpRequest('help request') + } + + if (parsed._.length != 3){ + throw new CmdTree.Error.UsageError('You must supply project json/js') + } + + const keyName = parsed.identity + + const phrase = await this.context.secureConfig.read('identity.'+keyName+'.phrase') + + if(!phrase){ + throw new CmdTree.Error.UsageError("Key doesn't exist!") + } + + const {password} = parsed.nopassword ? {password:null} : await prompt.get({ + properties: { + password: { + message: 'Enter password for identity['+keyName+']', + hidden: true + } + }}) + + let key = await dataparty_crypto.Identity.fromMnemonic(phrase, password, argon2) + + key.id = keyName + + const projectJsonPath = Path.resolve(parsed._[2]) + + /*let foundUp = findUp('package.json', Path.dirname(serviceClassPath)) + + let pkgJson = foundUp.content*/ + + let projectJson = require( projectJsonPath ) + + + const remoteName = parsed.remote || projectJson.venue + const remote = await this.context.secureConfig.read('remote.'+remoteName) + + if(!remote){ + throw new CmdTree.Error.UsageError('Invalid remote ['+remoteName+']') + } + + const project = { + owner: key.key.hash, + //created: Date.now(), + + name: parsed.name ? parsed.name : projectJson.name, + version: parsed.version ? parsed.version : projectJson.version, + + venue: remote.identity.key.hash, + domain: projectJson.domain, + + i2p: projectJson.i2p, + party: projectJson.party, + routes: projectJson.routes, + files: projectJson.files + + } + + await mkdirp(parsed.output) + + const buildOutput = parsed.output+'/'+ project.name.replace('/', '-') +'.project.venue.json' + + + let prjFiles = [] + prjFiles.push(buildOutput) + + if(project.files){ + this.addProjectFiles( + Path.dirname(projectJsonPath), + projectJson.files, + { nodir: true, follow: true } + ) + + const {tarPath, files} = await compressFiles(project.name,Path.dirname(projectJsonPath), this.project_sources.files, parsed.output, true) + + project.files = files + + if(tarPath){prjFiles.push(tarPath)} + + + } + + const ownerSig = await key.sign( project, true ) + + project.signatures = { + [key.key.hash]: dataparty_crypto.Routines.Utils.base64.encode(ownerSig.sig) + } + + fs.writeFileSync(buildOutput, JSON.stringify(project, null,2)) + + let staticTar = undefined + + if(prjFiles.length == 2){ + staticTar = fs.readFileSync(prjFiles[ prjFiles.length - 1 ]) + } + + await this.pushProject(key, remote, project, staticTar) + + return {files: prjFiles, project} + } + + + addProjectFiles(root, pattern, options){ + + let result = globSync(pattern, { + dotRelative: true, + cwd:root, + ...options + }) + + if(!this.project_sources.files){ + this.project_sources.files = result + } else { + this.project_sources.files = this.project_sources.files.concat(result) + } + + this.project_sources.files_root = root + + debug('addFiles',result) + + } + + async pushProject(devId, remote, build, staticTar=null){ + + let client = new Dataparty.EphemeralClient({ + identity: devId, + urlOrParty: remote.url, + wsUrlOrParty: remote.ws + }) + + await client.start() + + + let uploadResult = await client.restParty.comms.call('create-project', {project:build, staticTar}, { + expectClearTextReply: false, + sendClearTextRequest: false, + useSessions: true + }) + + console.log('result', uploadResult) + } +} + +module.exports = VenueProjectBuild + + diff --git a/src/venue/bin/commands/venue-identity-gen.js b/src/venue/bin/commands/venue-identity-gen.js new file mode 100644 index 0000000..6cb6d2b --- /dev/null +++ b/src/venue/bin/commands/venue-identity-gen.js @@ -0,0 +1,85 @@ +const CmdTree = require('command-tree') +const Hoek = require('@hapi/hoek') +const debug = require('debug')('venue.identity-gen') +const Path = require('path') +const OS = require('os') +const fs = require('fs') + +const prompt = require('prompt') +const argon2 = require('argon2') + +const { execSync } = require('child_process') + +const Dataparty = require('../../../../') +const dataparty_crypto = require('@dataparty/crypto') + +const DEFINITION = { + h: { + description: 'Show help', + alias: 'help', + type: 'help' + }, + nopassword: { + type: 'boolean', + default: false + }, + force: { + type: 'boolean', + default: false + } +} + + +class VenueIdentityGen extends CmdTree.Command { + constructor(context){ + super({...VenueIdentityGen.Definition, context}) + debug('constructor') + } + + static get Command(){ + return 'identity gen' + } + + static get Definition(){ + return { + usage: `venue identity gen [name]`, + description: 'Create an identity', + definition: DEFINITION + } + } + + async run({parsed}){ + //debug('context -', this.context) + //console.log('parsed -', parsed) + + if (parsed.h) { + throw new CmdTree.Error.HelpRequest('help request') + } + + if (parsed._.length != 3){ + throw new CmdTree.Error.UsageError('You must supply a name for the identity') + } + + const keyName = parsed._[2] + + if(await this.context.secureConfig.read('identity.'+keyName+'.phrase') && !parsed.force){ + throw new CmdTree.Error.UsageError('Key already exists!') + } + + const phrase = await dataparty_crypto.Routines.generateMnemonic() + + const password = parsed.nopassword ? null : await this.context.collectPassword() + + let key = await dataparty_crypto.Identity.fromMnemonic(phrase, password, argon2) + + key.id = keyName + + await this.context.secureConfig.write('identity.'+keyName+'.phrase', phrase) + + return {...key.toJSON()} + } +} + +module.exports = VenueIdentityGen + + diff --git a/src/venue/bin/commands/venue-identity-list.js b/src/venue/bin/commands/venue-identity-list.js new file mode 100644 index 0000000..171fdf3 --- /dev/null +++ b/src/venue/bin/commands/venue-identity-list.js @@ -0,0 +1,56 @@ +const CmdTree = require('command-tree') +const Hoek = require('@hapi/hoek') +const debug = require('debug')('venue.identity-list') +const Path = require('path') +const OS = require('os') +const fs = require('fs') + +const prompt = require('prompt') +const argon2 = require('argon2') + +const { execSync } = require('child_process') + +const Dataparty = require('../../../../') +const dataparty_crypto = require('@dataparty/crypto') + +const DEFINITION = { + h: { + description: 'Show help', + alias: 'help', + type: 'help' + } +} + + +class VenueIdentityList extends CmdTree.Command { + constructor(context){ + super({...VenueIdentityList.Definition, context}) + debug('constructor') + } + + static get Command(){ + return 'identity list' + } + + static get Definition(){ + return { + usage: `venue identity list [name]`, + description: 'List identity nicknames', + definition: DEFINITION + } + } + + async run({parsed}){ + if (parsed.h) { + throw new CmdTree.Error.HelpRequest('help request') + } + + const names = Object.keys(await this.context.secureConfig.read('identity')) + + return {names} + } +} + +module.exports = VenueIdentityList + + diff --git a/src/venue/bin/commands/venue-identity-show.js b/src/venue/bin/commands/venue-identity-show.js new file mode 100644 index 0000000..e289c74 --- /dev/null +++ b/src/venue/bin/commands/venue-identity-show.js @@ -0,0 +1,85 @@ +const CmdTree = require('command-tree') +const Hoek = require('@hapi/hoek') +const debug = require('debug')('venue.identity-show') +const Path = require('path') +const OS = require('os') +const fs = require('fs') + +const prompt = require('prompt') +const argon2 = require('argon2') + +const { execSync } = require('child_process') + +const Dataparty = require('../../../../') +const dataparty_crypto = require('@dataparty/crypto') + +const DEFINITION = { + h: { + description: 'Show help', + alias: 'help', + type: 'help' + }, + nopassword: { + type: 'boolean', + default: false + } +} + + +class VenueIdentityShow extends CmdTree.Command { + constructor(context){ + super({...VenueIdentityShow.Definition, context}) + debug('constructor') + } + + static get Command(){ + return 'identity show' + } + + static get Definition(){ + return { + usage: `venue identity show [name]`, + description: 'Show an identity', + definition: DEFINITION + } + } + + async run({parsed}){ + //debug('context -', this.context) + //console.log('parsed -', parsed) + + if (parsed.h) { + throw new CmdTree.Error.HelpRequest('help request') + } + + if (parsed._.length != 3){ + throw new CmdTree.Error.UsageError('You must supply a name for the identity') + } + + const keyName = parsed._[2] + + const phrase = await this.context.secureConfig.read('identity.'+keyName+'.phrase') + + if(!phrase){ + throw new CmdTree.Error.UsageError("Key doesn't exists!") + } + + const {password} = parsed.nopassword ? {password:null} : await prompt.get({ + properties: { + password: { + message: 'Enter password for identity['+keyName+']', + hidden: true + } + }}) + + let key = await dataparty_crypto.Identity.fromMnemonic(phrase, password, argon2) + + key.id = keyName + + return {...key.toJSON()} + } +} + +module.exports = VenueIdentityShow + + diff --git a/src/venue/bin/commands/venue-remote-add.js b/src/venue/bin/commands/venue-remote-add.js new file mode 100644 index 0000000..96a8093 --- /dev/null +++ b/src/venue/bin/commands/venue-remote-add.js @@ -0,0 +1,94 @@ +const CmdTree = require('command-tree') +const Hoek = require('@hapi/hoek') +const debug = require('debug')('venue.remote-add') +const Path = require('path') +const OS = require('os') +const fs = require('fs') + +const prompt = require('prompt') +const argon2 = require('argon2') + +const Dataparty = require('../../../../') +const dataparty_crypto = require('@dataparty/crypto') + +const DEFINITION = { + h: { + description: 'Show help', + alias: 'help', + type: 'help' + }, + i2p: { + type: 'string', + description: 'i2p address' + }, + ws: { + type: 'string', + description: 'websocket url' + }, + url: { + type: 'string', + description: 'api base url' + }, + hash: { + type: 'string', + description: 'key hash of remote' + }, + force: { + type: 'boolean', + default: false + } +} + + +class VenueRemoteAdd extends CmdTree.Command { + constructor(context){ + super({...VenueRemoteAdd.Definition, context}) + } + + static get Command(){ + return 'remote add' + } + + static get Definition(){ + return { + usage: `venue remote add [name]`, + description: 'Add remote party', + definition: DEFINITION + } + } + + async run({parsed}){ + if (parsed.h) { + throw new CmdTree.Error.HelpRequest('help request') + } + + if (parsed._.length != 3){ + throw new CmdTree.Error.UsageError('You must supply a name for the remote') + } + + const remoteName = parsed._[2] + + if(await this.context.secureConfig.read('remote.'+remoteName) && !parsed.force){ + throw new CmdTree.Error.UsageError('Remote already exists!') + } + + const identityUrl = parsed.url+'/identity' + const versionUrl = parsed.url+'/version' + + const identity = await Dataparty.Comms.RestComms.HttpGet(identityUrl) + const version = await Dataparty.Comms.RestComms.HttpGet(versionUrl) + + const remote = { + url: parsed.url, + ws: parsed.ws, + i2p: parsed.i2p, + identity, version + } + + await this.context.secureConfig.write('remote.'+remoteName, remote) + + return { remote } + } +} + +module.exports = VenueRemoteAdd diff --git a/src/venue/bin/commands/venue-remote-check.js b/src/venue/bin/commands/venue-remote-check.js new file mode 100644 index 0000000..bae2595 --- /dev/null +++ b/src/venue/bin/commands/venue-remote-check.js @@ -0,0 +1,127 @@ +const CmdTree = require('command-tree') +const Hoek = require('@hapi/hoek') +const debug = require('debug')('venue.remote-check') +const Path = require('path') +const OS = require('os') +const fs = require('fs') + +const prompt = require('prompt') +const argon2 = require('argon2') + +const { execSync } = require('child_process') + +const Dataparty = require('../../../../') +const dataparty_crypto = require('@dataparty/crypto') + +const DEFINITION = { + h: { + description: 'Show help', + alias: 'help', + type: 'help' + }, + remote: { + type: 'string', + require: true + }, + identity: { + type: 'string', + description: 'developer release identity', + require: true + } +} + + +class VenueRemoteCheck extends CmdTree.Command { + constructor(context){ + super({...VenueRemoteCheck.Definition, context}) + debug('constructor') + } + + static get Command(){ + return 'remote check' + } + + static get Definition(){ + return { + usage: `venue remote check`, + description: 'Check a remote party', + definition: DEFINITION + } + } + + async run({parsed}){ + //debug('context -', this.context) + //console.log('parsed -', parsed) + + if (parsed.h) { + throw new CmdTree.Error.HelpRequest('help request') + } + + const keyName = parsed.identity + + const phrase = await this.context.secureConfig.read('identity.'+keyName+'.phrase') + + if(!phrase){ + throw new CmdTree.Error.UsageError("Key doesn't exist!") + } + + const {password} = parsed.nopassword ? {password:null} : await prompt.get({ + properties: { + password: { + message: 'Enter password for identity['+keyName+']', + hidden: true + } + }}) + + let key = await dataparty_crypto.Identity.fromMnemonic(phrase, password, argon2) + + key.id = keyName + + const remote = await this.context.secureConfig.read('remote.'+parsed.remote) + + const client = new Dataparty.EphemeralClient({ + identity: key, + urlOrParty: remote.url, + wsUrlOrParty: remote.ws + }) + + client.on('session',(id)=>{ + console.log('session', id) + }) + + client.on('session-end',(id)=>{ + console.log('session-end', id) + }) + + client.on('connecting',(info)=>{ + console.log('connecting', info) + }) + + client.on('connected',(info)=>{ + console.log('connected', info) + }) + + client.on('disconnected',(info)=>{ + console.log('disconnected', info) + }) + + client.on('reconnected',(info)=>{ + console.log('reconnected', info) + }) + + client.on('reconnecting',(info)=>{ + console.log('reconnecting',info) + }) + + await client.start() + console.log('client started') + + this.context.exiting = false + + return {remote, client} + } +} + +module.exports = VenueRemoteCheck + + diff --git a/src/venue/bin/commands/venue-remote-list.js b/src/venue/bin/commands/venue-remote-list.js new file mode 100644 index 0000000..ed9b1fe --- /dev/null +++ b/src/venue/bin/commands/venue-remote-list.js @@ -0,0 +1,71 @@ +const CmdTree = require('command-tree') +const Hoek = require('@hapi/hoek') +const debug = require('debug')('venue.remote-list') +const Path = require('path') +const OS = require('os') +const fs = require('fs') + +const prompt = require('prompt') +const argon2 = require('argon2') + +const Dataparty = require('../../../../') +const dataparty_crypto = require('@dataparty/crypto') + +const DEFINITION = { + h: { + description: 'Show help', + alias: 'help', + type: 'help' + }, + i2p: { + type: 'string', + description: 'i2p address' + }, + ws: { + type: 'string', + description: 'websocket url' + }, + url: { + type: 'string', + description: 'api base url' + }, + hash: { + type: 'string', + description: 'key hash of remote' + }, + force: { + type: 'boolean', + default: false + } +} + + +class VenueRemoteList extends CmdTree.Command { + constructor(context){ + super({...VenueRemoteList.Definition, context}) + } + + static get Command(){ + return 'remote list' + } + + static get Definition(){ + return { + usage: `venue remote list [name]`, + description: 'List remote parties', + definition: DEFINITION + } + } + + async run({parsed}){ + if (parsed.h) { + throw new CmdTree.Error.HelpRequest('help request') + } + + const names = Object.keys(await this.context.secureConfig.read('remote')) + + return { names } + } +} + +module.exports = VenueRemoteList diff --git a/src/venue/bin/commands/venue-remote-repl.js b/src/venue/bin/commands/venue-remote-repl.js new file mode 100644 index 0000000..118fd2d --- /dev/null +++ b/src/venue/bin/commands/venue-remote-repl.js @@ -0,0 +1,138 @@ +const CmdTree = require('command-tree') +const Hoek = require('@hapi/hoek') +const debug = require('debug')('venue.remote-repl') +const Path = require('path') +const OS = require('os') +const fs = require('fs') +const repl = require("repl"); + +const prompt = require('prompt') +const argon2 = require('argon2') + +const { execSync } = require('child_process') + +const Dataparty = require('../../../../') +const dataparty_crypto = require('@dataparty/crypto') + +const DEFINITION = { + h: { + description: 'Show help', + alias: 'help', + type: 'help' + }, + remote: { + type: 'string', + require: true + }, + identity: { + type: 'string', + description: 'admin identity', + require: true + } +} + + +class VenueRemoteRepl extends CmdTree.Command { + constructor(context){ + super({...VenueRemoteRepl.Definition, context}) + debug('constructor') + } + + static get Command(){ + return 'remote repl' + } + + static get Definition(){ + return { + usage: `venue remote repl`, + description: 'Run a repl with access to a remote party', + definition: DEFINITION + } + } + + async run({parsed}){ + //debug('context -', this.context) + //console.log('parsed -', parsed) + + if (parsed.h) { + throw new CmdTree.Error.HelpRequest('help request') + } + + const keyName = parsed.identity + + const phrase = await this.context.secureConfig.read('identity.'+keyName+'.phrase') + + if(!phrase){ + throw new CmdTree.Error.UsageError("Key doesn't exist!") + } + + const {password} = parsed.nopassword ? {password:null} : await prompt.get({ + properties: { + password: { + message: 'Enter password for identity['+keyName+']', + hidden: true + } + }}) + + let key = await dataparty_crypto.Identity.fromMnemonic(phrase, password, argon2) + + key.id = keyName + + const remote = await this.context.secureConfig.read('remote.'+parsed.remote) + + const client = new Dataparty.EphemeralClient({ + identity: key, + urlOrParty: remote.url, + wsUrlOrParty: remote.ws + }) + + client.on('session',(id)=>{ + console.log('session', id) + }) + + client.on('session-end',(id)=>{ + console.log('session-end', id) + }) + + client.on('connecting',(info)=>{ + console.log('connecting', info) + }) + + client.on('connected',(info)=>{ + console.log('connected', info) + }) + + client.on('disconnected',(info)=>{ + console.log('disconnected', info) + }) + + client.on('reconnected',(info)=>{ + console.log('reconnected', info) + }) + + client.on('reconnecting',(info)=>{ + console.log('reconnecting',info) + }) + + await client.start() + console.log('client started') + + + + const replSrv = repl.start({ + prompt: parsed.identity +'@'+ parsed.remote+'> ' + }) + + replSrv.context.remote = remote + replSrv.context.client = client + + + this.context.exiting = false + + return // {remote, client, replSrv} + } +} + +module.exports = VenueRemoteRepl + + diff --git a/src/venue/bin/commands/venue-remote-show.js b/src/venue/bin/commands/venue-remote-show.js new file mode 100644 index 0000000..c8f1e63 --- /dev/null +++ b/src/venue/bin/commands/venue-remote-show.js @@ -0,0 +1,66 @@ +const CmdTree = require('command-tree') +const Hoek = require('@hapi/hoek') +const debug = require('debug')('venue.remote-show') +const Path = require('path') +const OS = require('os') +const fs = require('fs') + +const prompt = require('prompt') +const argon2 = require('argon2') + +const { execSync } = require('child_process') + +const Dataparty = require('../../../../') +const dataparty_crypto = require('@dataparty/crypto') + +const DEFINITION = { + h: { + description: 'Show help', + alias: 'help', + type: 'help' + } +} + + +class VenueRemoteShow extends CmdTree.Command { + constructor(context){ + super({...VenueRemoteShow.Definition, context}) + debug('constructor') + } + + static get Command(){ + return 'remote show' + } + + static get Definition(){ + return { + usage: `venue remote show [name]`, + description: 'Show a remote party', + definition: DEFINITION + } + } + + async run({parsed}){ + //debug('context -', this.context) + //console.log('parsed -', parsed) + + if (parsed.h) { + throw new CmdTree.Error.HelpRequest('help request') + } + + if (parsed._.length != 3){ + throw new CmdTree.Error.UsageError('You must supply a name for the remote') + } + + const remoteName = parsed._[2] + + const remote = await this.context.secureConfig.read('remote.'+remoteName) + + + return {remote} + } +} + +module.exports = VenueRemoteShow + + diff --git a/src/venue/bin/commands/venued-admin-add.js b/src/venue/bin/commands/venued-admin-add.js new file mode 100644 index 0000000..209e6fe --- /dev/null +++ b/src/venue/bin/commands/venued-admin-add.js @@ -0,0 +1,85 @@ +const CmdTree = require('command-tree') +const Hoek = require('@hapi/hoek') +const debug = require('debug')('venued.admin-add') +const Path = require('path') +const OS = require('os') +const fs = require('fs') + +const { execSync } = require('child_process') + +const Dataparty = require('../../../../') + +const HOMEDIR = OS.homedir() +const DEFAULT_FOLDER = (HOMEDIR.indexOf('opt')==-1) ? '.venued' : '' +const DEFAULT_PATH = Path.join( HOMEDIR, DEFAULT_FOLDER ) + + +const DEFINITION = { + h: { + description: 'Show help', + alias: 'help', + type: 'help' + }, + p: { + alias: 'path', + description: 'venued path', + default: DEFAULT_PATH + } +} + + +class VenuedAdminAdd extends CmdTree.Command { + constructor(context){ + super({...VenuedAdminAdd.Definition, context}) + debug('constructor') + } + + static get Command(){ + return 'admin add' + } + + static get Definition(){ + return { + usage: `venued admin add [key-hash]`, + description: 'Add admin key', + definition: DEFINITION + } + } + + async run({parsed}){ + //debug('context -', this.context) + //console.log('parsed -', parsed) + + if (parsed.h) { + throw new CmdTree.Error.HelpRequest('help request') + } + + const config = new Dataparty.Config.JsonFileConfig({basePath: parsed.path}) + + await config.start() + + + let admins = (await config.read('admins')) || [] + + + let newAdmins = [] + + + for(let admin of parsed._.slice(2)){ + if(admins.indexOf(admin) != -1){ continue } + + newAdmins.push(admin) + } + + admins = admins.concat(newAdmins) + + await config.write('admins', admins) + await config.save() + + //console.log('admin added -', newAdmins) + + return {newAdmins} + } +} + +module.exports = VenuedAdminAdd \ No newline at end of file diff --git a/src/venue/bin/commands/venued-admin-list.js b/src/venue/bin/commands/venued-admin-list.js new file mode 100644 index 0000000..da0555e --- /dev/null +++ b/src/venue/bin/commands/venued-admin-list.js @@ -0,0 +1,69 @@ +const CmdTree = require('command-tree') +const Hoek = require('@hapi/hoek') +const debug = require('debug')('venued.admin-list') +const Path = require('path') +const OS = require('os') +const fs = require('fs') + +const { execSync } = require('child_process') + +const Dataparty = require('../../../../') + +const HOMEDIR = OS.homedir() +const DEFAULT_FOLDER = (HOMEDIR.indexOf('opt')==-1) ? '.venued' : '' +const DEFAULT_PATH = Path.join( HOMEDIR, DEFAULT_FOLDER ) + + +const DEFINITION = { + h: { + description: 'Show help', + alias: 'help', + type: 'help' + }, + p: { + alias: 'path', + description: 'venued path', + default: DEFAULT_PATH + } +} + + +class VenuedAdminList extends CmdTree.Command { + constructor(context){ + super({...VenuedAdminList.Definition, context}) + debug('constructor') + } + + static get Command(){ + return 'admin list' + } + + static get Definition(){ + return { + usage: `venued admin list [key-hash]`, + description: 'List admin keys', + definition: DEFINITION + } + } + + async run({parsed}){ + //debug('context -', this.context) + //debug('parsed -', parsed) + + if (parsed.h) { + throw new CmdTree.Error.HelpRequest('help request') + } + + let config = new Dataparty.Config.JsonFileConfig({basePath: parsed.path}) + + await config.start() + + let admins = (await config.read('admins')) || [] + + config=null + + return {admins} + } +} + +module.exports = VenuedAdminList \ No newline at end of file diff --git a/src/venue/bin/commands/venued-host.js b/src/venue/bin/commands/venued-host.js new file mode 100644 index 0000000..81200cb --- /dev/null +++ b/src/venue/bin/commands/venued-host.js @@ -0,0 +1,406 @@ +const CmdTree = require('command-tree') +const Hoek = require('@hapi/hoek') +const debug = require('debug')('venued.host') +const Path = require('path') +const OS = require('os') +const fs = require('fs') +const zlib = require('zlib') +const { execSync } = require('child_process') + +const Dataparty = require('../../../../') +const {Routines} = require('@dataparty/crypto') +const express = require('express') + +const HOMEDIR = OS.homedir() +const DEFAULT_FOLDER = (HOMEDIR.indexOf('opt')==-1) ? '.venued' : '' +const DEFAULT_PATH = Path.join( HOMEDIR, DEFAULT_FOLDER ) + + +const DEFINITION = { + h: { + description: 'Show help', + alias: 'help', + type: 'help' + }, + p: { + alias: 'path', + description: 'venued path', + default: DEFAULT_PATH + }, + l: { + alias: 'listen', + description: 'listen uri this can be an https:// with any local IP or file:/// to use a Unix socket', + default: 'https://0.0.0.0:3000' + }, + k: { + alias: 'ssl-key', + description: 'SSL key in pem format', + default: Path.join(DEFAULT_PATH, 'key.pem') + }, + c: { + alias: 'ssl-cert', + description: 'SSL Certificate in pem format', + default: Path.join(DEFAULT_PATH, 'cert.pem') + }, + A: { + alias: 'allow-ip', + description: 'IP to add to allow list', + multiple: true + }, + 'db-type': { + description: 'Type of database', + valid: ['tingo', 'loki', 'mongo', 'peer'], + default: 'tingo' + }, + 'db-uri': { + description: 'url to connect to database', + default: Path.join(DEFAULT_PATH, 'db/') + }, + 'db-peer': { + description: 'Peer database identity hash' + }, + 'service-code': { + description: 'path to service implementation', + default: Path.join(__dirname, '../../venue-service.js') + }, + 'service-build': { + description: 'path to service build', + default: Path.join(__dirname, '../../dataparty/@dataparty-venue.service.venue.json') + }, + 'full-errors': { + description: 'full server side error messages sent to client', + type:'boolean', + default: false + }, + i2p: { + description: 'Enable i2p hosting', + type: 'boolean', + default: false + }, + 'i2p-host':{ + default: '127.0.0.1' + }, + 'i2p-port': { + default: 7656 + }, + 'trust-proxy': { + type: 'boolean', + default: false + } +} + +function getPartyByType(type){ + if(type == 'tingo'){ + return Dataparty.TingoParty + } +} + +class VenuedHost extends CmdTree.Command { + constructor(context){ + super({...VenuedHost.Definition, context}) + debug('constructor') + } + + static get Command(){ + return 'host' + } + + static get Definition(){ + return { + usage: `venued host [options]`, + description: 'Start the hosting venued', + definition: DEFINITION + } + } + + async run({parsed}){ + //debug('context -', this.context) + //debug('parsed -', parsed) + + if (parsed.h) { + throw new CmdTree.Error.HelpRequest('help request') + } + + /*if (!parsed.name){ + throw new CmdTree.Error.UsageError('no name provided') + }*/ + + + const ServiceCode = require(parsed['service-code']) + const ServiceBuild = require(parsed['service-build']) + const ServiceSchema = { + package: ServiceBuild.package, + ...ServiceBuild.schemas + } + + const PARTY = getPartyByType(parsed['db-type']) + + const config = new Dataparty.Config.JsonFileConfig({basePath: parsed.path}) + + await config.start() + + //if(parsed['db-uri'][0] == '/'){ + config.touchDir( 'db' ) + //} + + const party = new PARTY({ + path: parsed['db-uri'], + model: ServiceSchema, + config: config, + noCache: false + }) + + const service = new ServiceCode( ServiceSchema.package, ServiceBuild ) + + debug('loaded service') + + debug('party db location', parsed['db-uri']) + + const CustomIpFilter = { + options: { + mode: 'allow', + //trustProxy: true + }, + ips: [ + '173.245.48.0/20', + '103.21.244.0/22', + '103.22.200.0/22', + '103.31.4.0/22', + '141.101.64.0/18', + '108.162.192.0/18', + '190.93.240.0/20', + '188.114.96.0/20', + '197.234.240.0/22', + '198.41.128.0/17', + '162.158.0.0/15', + '104.16.0.0/13', + '104.24.0.0/14', + '172.64.0.0/13', + '131.0.72.0/22', + '2400:cb00::/32', + '2606:4700::/32', + '2803:f800::/32', + '2405:b500::/32', + '2405:8100::/32', + '2a06:98c0::/29', + '2c0f:f248::/32', + '10.115.68.55/32' //! + ] + } + + if(parsed['allow-ip']){ + for(let ip of parsed['allow-ip']){ + CustomIpFilter.ips.push(ip) + } + } + + + if(!fs.existsSync(parsed['ssl-key'])){ + execSync('openssl req -newkey rsa:2048 -new -nodes -x509 -days 3650 -keyout key.pem -out cert.pem -subj "/C=US/ST=State/L=City/O=Organization/OU=Unit/CN=example.com"', + {cwd: parsed.path} + ) + } + + const ssl_key = fs.readFileSync( parsed['ssl-key'], 'utf8') + const ssl_cert = fs.readFileSync( parsed['ssl-cert'], 'utf8') + + + const runner = new Dataparty.ServiceRunnerNode({ + party, service, + sendFullErrors: parsed['full-errors'], + useNative: false, + prefix: 'venue/' + }) + + + let runnerRouter = new Dataparty.RunnerRouter(runner) + + + + + if(parsed.i2p && !await config.read('i2p.sam')){ + debug('i2p - creating key') + + const SAM = require('@diva.exchange/i2p-sam') + + const i2pSettings = { + sam: { + host: parsed['i2p-host'], + portTCP: parsed['i2p-port'], + }, + session: { + options: 'i2cp.leaseSetEncType=6,4' + //options: 'i2cp.leaseSetEncType=4' + } + } + + let i2p = await SAM.createLocalDestination(i2pSettings) + + + await Promise.all([ + config.write('i2p.address', i2p.address), + config.write('i2p.sam.publicKey', i2p.public), + config.write('i2p.sam.privateKey', i2p.private), + ]) + + await config.save() + } + + const host = new Dataparty.ServiceHost({ + runner: runnerRouter, + trust_proxy: parsed['trust-proxy/'], + wsEnabled: true, + ssl_key, ssl_cert, + listenUri: parsed.listen, + staticPath: Path.join(__dirname,'../../public'), + staticPrefix: '/venue/', + ipFilter: CustomIpFilter, + i2pEnabled: parsed.i2p, + i2pSamHost: parsed['i2p-host'], + i2pSamPort: parsed['i2p-port'], + i2pForwardHost: '127.0.0.1', + i2pForwardPort: '3000', + i2pOptions: 'i2cp.leaseSetEncType=6,4', + //i2pOptions: 'i2cp.leaseSetEncType=4', + i2pKey: await config.read('i2p.sam') + }) + + await party.start() + await runner.start() + await host.start() + + debug('started') + console.log('partying') + console.log('\t', parsed.listen) + + const i2pAddress = await config.read('i2p.address') + if(i2pAddress){ + console.log('\t', i2pAddress) + } + + /** + * config section + * + * projects: { + * [name]: latest-hash + * } + */ + + const projects = await config.read('projects') + + if(projects){ + for(let name in projects){ + + let projectParties = {} + + //! todo - load parties and put them in projectParties map + + const hash = projects[name] + console.log('\tloading project', name, hash) + + const project = (await party.find() + .type('venue_project') + .where('hash').equals(hash).exec())[0] + + const workspace = project.data.workspace + + for(let route of project.data.project.routes){ + console.log('route', route.package.name) + let pkgDoc = (await party.find() + .type('venue_pkg') + .or() + .where('package.name').equals(route.package.name) + .where('package.githash').equals(route.package.githash) + .sort('-created') + .limit(1) + .exec())[0] + + if(!pkgDoc){ + throw new Error(`package ${JSON.stringify(route.package)} not found. required by ${route.prefix}`) + } + + const {compressedBuild, ...printablePkg} = pkgDoc.data + + console.log('found package', printablePkg) + + const serviceFile = JSON.parse( + zlib.brotliDecompressSync( + Routines.Utils.base64.decode( compressedBuild ) + ) + ) + + console.log('decompressed', serviceFile.package) + + let serviceParty = null; + + + if(route.party == 'SYSTEM'){ + serviceParty = party + } else if( projectParties[route.party] ){ + serviceParty = projectParties[route.party] + } + + serviceParty.topics = new Dataparty.LocalTopicHost() + + debug('loading service') + const service = new Dataparty.IService(serviceFile.package, serviceFile) + debug('loaded service') + + let projectRunner = new Dataparty.ServiceRunnerNode({ + party: serviceParty, service, + sendFullErrors: route.settings.sendFullErrors, + useNative: route.settings.useNative, + prefix: route.prefix + }) + + if(route.party == 'SYSTEM'){ + //projectRunner.router = runner.router + } + + //await serviceParty.start() + + await projectRunner.start() + + console.log('workspace', workspace) + + const projectStaticPath = Path.join(workspace, 'public') + + let handler = (req,res)=>{ + + let staticHandler = express.static(projectStaticPath, { index: ['index.html']}) + + let results = staticHandler(req.request,req.response, req.request.next) + + console.log('returning results', Object.keys(req.request)) + + + + console.log('sent already - headers?', req.response.headersSent) + console.log('sent already - date?', req.response.sendDate) + console.log('sent already - outputSize?', req.response.outputSize) + + } + + projectRunner.router.add('static-files1', '/:path*', handler) + projectRunner.router.add('static-files2', '/', handler) + + + + + await runnerRouter.addRunner({ + domain: project.data.project.domain, + runner: projectRunner + }) + } + } + } + + + + this.context.exiting = false + + return + } +} + +module.exports = VenuedHost diff --git a/src/venue/bin/venue.js b/src/venue/bin/venue.js new file mode 100755 index 0000000..7ce2db9 --- /dev/null +++ b/src/venue/bin/venue.js @@ -0,0 +1,158 @@ +#!/usr/bin/env node + +const Pkg = require('../../../package.json') +const debug = require('debug')('venue') +const CommandTree = require('command-tree').CommandTree +const prompt = require('prompt') +const argon2 = require('argon2') +const OS = require('os') +const Path = require('path') + + +const Dataparty = require('../../../') + + +const commandTree = new CommandTree({ usage: 'venue [command] \nVersion: ' + Pkg.version }) + +commandTree.addCommand(require('./commands/venue-identity-gen')) +commandTree.addCommand(require('./commands/venue-identity-list')) +commandTree.addCommand(require('./commands/venue-identity-show')) + +commandTree.addCommand(require('./commands/venue-remote-add')) +commandTree.addCommand(require('./commands/venue-remote-list')) +commandTree.addCommand(require('./commands/venue-remote-show')) +commandTree.addCommand(require('./commands/venue-remote-check')) +commandTree.addCommand(require('./commands/venue-remote-repl')) + +commandTree.addCommand(require('./commands/pkg-build')) +commandTree.addCommand(require('./commands/project-build')) + +const HOMEDIR = OS.homedir() +const DEFAULT_FOLDER = '.venue' +const DEFAULT_PATH = Path.join( HOMEDIR, DEFAULT_FOLDER ) + +let secureConfig = null + +async function collectPassword(info=''){ + let password = '' + + while(1){ + let passes = await prompt.get({ + properties: { + password1: { + message: 'Set'+info+' password', + hidden: true + }, + password2: { + message: 'Confim'+info+' password', + hidden: true + } + } + }) + + if(passes.password1 == passes.password2){ + + password = passes.password1 + break + } + + console.log("passwords don't match") + } + + return password +} + +async function onSetupRequired(){ + + console.log('setup-required') + + const password = await collectPassword(' keychain') + + await secureConfig.setPassword(password, { + created: Date.now() + }) + + console.log('password set') + + + await secureConfig.unlock(password) +} + +let context = { + exiting: true +} + +async function main(){ + + if(process.argv.length < 3 || process.argv[2] == 'help' || process.argv[2] == '--help'){ + console.log(commandTree.getHelp()) + if(process.send){ process.send(commandTree.getHelp()) } + return + } + + + let config = new Dataparty.Config.JsonFileConfig({basePath: DEFAULT_PATH}) + secureConfig = new Dataparty.Config.SecureConfig({ + config, + timeoutMs: 60*1000*5, + argon: argon2 + }) + + secureConfig.on('setup-required', onSetupRequired) + + console.log('starting') + + await config.start() + await secureConfig.start() + + + if(await secureConfig.isInitialized() && secureConfig.isLocked()){ + + const {password} = await prompt.get({ + properties: { + password: { + message: 'Enter password', + hidden: true + } + }}) + + await secureConfig.unlock(password) + } + + + await secureConfig.waitForUnlocked('startup') + + context = { + secureConfig, collectPassword, + exiting: true + } + + const output = await commandTree.run({context}) + + if(output){ + console.log(output) + + if(process.send){ process.send({output}) } + } + +} + +// Run main +main().catch((error) => { + console.log(error) + console.error(error.message) + debug(error) + console.log(commandTree.getHelp()) + if(process.send){ + process.send({ + error: error, + output: commandTree.getHelp() + }) + } + //process.exit() +}).finally(()=>{ + + if(context.exiting){ + process.exit() + } +}) \ No newline at end of file diff --git a/src/venue/bin/venued.js b/src/venue/bin/venued.js new file mode 100755 index 0000000..fbe37cd --- /dev/null +++ b/src/venue/bin/venued.js @@ -0,0 +1,82 @@ +#!/usr/bin/env node + +const Pkg = require('../../../package.json') +const debug = require('debug')('venue') +const CommandTree = require('command-tree').CommandTree + + +const commandTree = new CommandTree({ usage: 'venued [command] \nVersion: ' + Pkg.version }) + +commandTree.addCommand(require('./commands/venued-host')) +commandTree.addCommand(require('./commands/venued-admin-add')) +commandTree.addCommand(require('./commands/venued-admin-list')) + +/*commandTree.addCommand(require('./project/project-init')) +commandTree.addCommand(require('./project/project-show')) +commandTree.addCommand(require('./project/project-mount')) +commandTree.addCommand(require('./developer/developer-add')) +commandTree.addCommand(require('./team/team-add')) +commandTree.addCommand(require('./cloud/cloud-add')) +commandTree.addCommand(require('./cloud/cloud-list')) +commandTree.addCommand(require('./package/package-add')) +commandTree.addCommand(require('./service/service-add'))*/ + + +/* +venued host --path /opt/venue +venued get version/identity/config +venued admin add/rm/list + +venue remote add --url [] --ws [] --identity <> +venue remote rm +venue remote switch + +venue package build --output [] +venue package deploy + +venue project build +venue project deploy +*/ + +let context = { + exiting: true +} + +async function main(){ + + if(process.argv.length < 3 || process.argv[2] == 'help' || process.argv[2] == '--help'){ + console.log(commandTree.getHelp()) + if(process.send){ process.send(commandTree.getHelp()) } + return + } + + + const output = await commandTree.run({context}) + + if(output){ + console.log(output) + + if(process.send){ process.send({output}) } + } + +} + +// Run main +main().catch((error) => { + console.log(error) + console.error(error.message) + debug(error) + console.log(commandTree.getHelp()) + if(process.send){ + process.send({ + error: error, + output: commandTree.getHelp() + }) + } + //process.exit() +}).finally(()=>{ + + if(context.exiting){ + process.exit() + } +}) diff --git a/src/venue/build.js b/src/venue/build.js index 756f098..04e0416 100644 --- a/src/venue/build.js +++ b/src/venue/build.js @@ -1,12 +1,145 @@ +const fs = require('fs') const Path = require('path') const debug = require('debug')('build') const Dataparty = require('../index.js') +const dataparty_crypto = require('@dataparty/crypto') + const Pkg = require('../../package.json') const VenueService = require('./venue-service') +/** + * build/ + * - service.venue.json + * - schema.venue.json + * - client.venue.json + * - staticFiles.venue.tgz + * - package.venue.json + * { author, venue, files[], signatures{} } + */ + + +/* + +venue: { + url, + publicKey, + +} + +.venue-admin/ + - config/config.json + - db/ + - packages/ + - AUTHOR/PACKAGE-NAME + - package.venue.json + - service.venue.json + - static-files.venue.tgz + - parties/ + - PARTY_PUBLIC/ + - config/config.json (optional) + - db/ (optional) + - static-files.venue.tgz + - projects/ + - AUTHOR/PROJECT_PUBLIC + - project.venue.json + - static-files.venue.tgz + - party/ + - PARTY_NAME/ + - static-files.venue.tgz + + +~/code/my_venue_project/ + - package.json + - build.js + - public/ + - party/NAME + - default-config.json + - public/ + - package/NAME + - project/NAME + - venue.json + { + projects: { + NAME: { + name: string + domain, (optional) + venue, (optional) + parties: [NAME], + routes: { + PREFIX: [ { PACKAGE(owner,name,version), party } ] + } + } + }, + packages: { + NAME: { + name: String + version: String, (optional) + service: localPathToService.js OR built service.json + } + } + } + +*/ + +async function buildVenuePackage({authorIdentity, venueIdentity, outputPath, existingBuildPath, staticFilePaths}){ + + /* + + 0. create service.venue.json + 1. create schema.venue.json + 2. create client.venue.json + 3. create staticFiles.venue.tgz + 4. create package.venue.json + + 6. upload to venue + - create-package + * service.venue.json + * package.venue.json + + 7. upload files to venue + - create-files + * staticFiles.venue.tgz + * schema.venue.json + * client.venue.json + + 8. + + */ + +} + + +/** + * + * 1. generate admin/developer key + * 2. add venue identity (by url) + * 3. venue package build - build a package at some path + * 4. venue package push - c + */ + +async function pushService(devId, build, staticTar){ + + let client = new Dataparty.MatchMakerClient( + devId, + null, + 'https://api.dataparty.xyz/venue', + 'wss://api.dataparty.xyz/ws' + ) + + await client.start() + + + let uploadResult = await client.restParty.comms.call('create-package', {build, staticTar}, { + expectClearTextReply: false, + sendClearTextRequest: false, + useSessions: true + }) + + console.log('result', uploadResult) +} async function main(){ const service = new VenueService({ @@ -14,13 +147,36 @@ async function main(){ version: Pkg.version }) + const path = Path.join(process.env.HOME, '.venue-admin') + + let config = new Dataparty.Config.JsonFileConfig({ + basePath: path+'/config' + }) + + let party = new Dataparty.TingoParty({ + path: path+'/db', + config, + noCache: false + }) + + await party.start() + + console.log( 'identity - ', party.identity.key.hash ) + const builder = new Dataparty.ServiceBuilder(service) - const build = await builder.compile(Path.join(__dirname,'./dataparty'), true) + const build = await builder.compile(Path.join(__dirname,'./dataparty'), true, party.privateIdentity) debug('compiled') + + const staticTar = fs.readFileSync('./dataparty/@dataparty-venue.files.venue.tgz') + + debug('is staticTar a buffer? ', staticTar instanceof Buffer); // true + await pushService( party.privateIdentity, build, staticTar ) + + } main().catch(err=>{ console.error('CRASH') console.error(err) -}) \ No newline at end of file +}) diff --git a/src/venue/endpoints/create-package.js b/src/venue/endpoints/create-package.js new file mode 100644 index 0000000..18d05d9 --- /dev/null +++ b/src/venue/endpoints/create-package.js @@ -0,0 +1,353 @@ +const fs = require('fs') +const Joi = require('joi') +const Path = require('path') +const Hoek = require('@hapi/hoek') +const {Message, Routines, Identity} = require('@dataparty/crypto') +const debug = require('debug')('dataparty.endpoint.create-package') +const zlib = require('zlib') + +const IEndpoint = require('../../service/iendpoint') + +const typedArraySchema = (value, helpers) => { + // 1. Ensure the value is an instance of a TypedArray (e.g., Uint8Array) + if (!(value instanceof Uint8Array)) { + return helpers.message({ custom: '"value" must be a Uint8Array' }); + } + + return value +} + +module.exports = class CreatePkgEndpoint extends IEndpoint { + + static get Name(){ + return 'create-package' + } + + + static get Description(){ + return 'Create venue package' + } + + /* + { + venue_package: { + package:{ + owner: String, + info: { + name, version, githash, branch + }, + files: [ //package, service, static.tgz, + {hash: String, name: String, size: Number, signature} + ], + statics: { + PREFIX: [localPathGlob] + } + }, + trust: { + owner: signature + } + + } + } + + { + venue_project: { + + project:{ + + owner: String, + domain: String, + venue: String, + + parties: { + NAME: { + keys: {public, private: Secret(private)}, + defaultConfig: Object, + files: [ //static.tgz, + {hash: String, name: String, signature} + ], + statics: {prefix, [localPath]} + } + }, + + + + + routes: { + PREFIX: [{ + party: String, + package: { + owner: String, + info: {name, version, branch, githash}, + settings: { + sendFullErrors, + useNative + } + } + }] + } + }, + trust: { + owner: signature + } + } + } + + */ + + static get MiddlewareConfig(){ + return { + pre: { + decrypt: true, + ephemeral_session: true, + validate: Joi.object().keys({ + settings: Joi.object().keys({ + //enabled: Joi.boolean().default(true).required(), + //domain: Joi.string().required(), + staticPrefix: Joi.string().default('/'), + sendFullErrors: Joi.boolean().default(false), + useNative: Joi.boolean().default(false), + defaultConfig: Joi.object().keys(null) + }), + build: Joi.object().keys({ + package: Joi.object().keys({ + owner: Joi.string(), + name: Joi.string().required(), + version: Joi.string().required(), + githash: Joi.string().required(), + branch: Joi.string().required() + }).required(), + schemas: Joi.object().keys(null), + documents: Joi.object().keys(null), + endpoints: Joi.object().keys(null), + middleware: Joi.object().keys(null), + middleware_order: Joi.object().keys(null), + tasks: Joi.object().keys(null), + topics: Joi.object().keys(null), + auth: Joi.object().keys(null), + files: Joi.object().keys(null), + signatures: Joi.object().keys(null).required(), + compileSettings: Joi.object().keys(null) + }).required(), + //staticTar: Joi.binary() + staticTar: Joi.any().custom(typedArraySchema) + }) + }, + post: { + encrypt: true, + validate: Joi.object().keys(null).description('any output allowed') + } + } + } + + static async run(ctx){ + + let {signatures, ...buildWithoutSig} = ctx.input.build + + const pkgOwnerIdentityDoc = (await ctx.party.find() + .type('public_key') + .where('hash').equals( ctx.input.build.package.owner ) + .exec() + )[0] + + if(!pkgOwnerIdentityDoc){ + throw new Error('package owner not authorized') + } + + debug('found pkg owner', pkgOwnerIdentityDoc.hash) + + const pkgOwnerIdentity = Identity.fromJSON({ + id: '', + key: { + type: pkgOwnerIdentityDoc.data.type, + hash: pkgOwnerIdentityDoc.data.hash, + public: pkgOwnerIdentityDoc.data.public + } + }) + + + debug('inflated identity') + + //debug('build', buildWithoutSig) + + const devSig = Routines.Utils.base64.decode(signatures[ctx.input.build.package.owner]) + + let signedBuildMsg = new Message({ + msg: buildWithoutSig, + sig: devSig + }) + + debug('sigs', signatures) + + debug('verifying package signature') + + await signedBuildMsg.assertVerified(pkgOwnerIdentity, true) + + debug('verified package signature') + + const safeFileName = ctx.input.build.package.name.replace('/', '-') + const tarFileName = safeFileName+'.files.venue.tgz' + + if(ctx.input.staticTar){ + + const tarHash = Routines.Utils.hash(ctx.input.staticTar) + const tarHash64 = Routines.Utils.base64.encode( tarHash ) + + const buildFiles = ctx.input.build.files[tarFileName] + + if(buildFiles && buildFiles.hash != tarHash64){ + throw new Error("staticTar hash doesn't match package definition") + } + + debug('verified staticTar') + } + + debug('verified package - '+ctx.input.build.package.name+'@'+ctx.input.build.package.version) + + const buildBSON = Routines.BSON.serializeBSONWithoutOptimiser(/*ctx.input.build*/buildWithoutSig) + + const buildHash = Routines.Utils.base64.encode( + Routines.Utils.hash( + buildBSON + ) + ) + + const safeBuildHash = buildHash.replace(/\//g, "-") + + debug('\t'+'hash', buildHash) + + const buildWorkspace = Path.join('packages', safeFileName, ctx.input.build.package.version, safeBuildHash) + + const config = ctx.party.config + + const workspacePath = await config.touchDir(buildWorkspace) + + + debug('\t'+'workspace - local', buildWorkspace) + debug('\t'+'workspace - global', workspacePath) + + + + const compressedBrotliBuild = zlib.brotliCompressSync(JSON.stringify(ctx.input.build)) + + const build = ctx.input.build + const serviceId = build.package.name + '@' + build.package.version + debug('addService', serviceId) + + let srvDoc = (await ctx.party.find() + .type('venue_pkg') + .where('package.name').equals(build.package.name) + .where('package.version').equals(build.package.version) + .where('hash').equals(buildHash) + .exec())[0] + + + if(!srvDoc){ + debug('creating service') + + const {owner, ...pkgWithoutOwner} = build.package + + srvDoc = await ctx.party.createDocument('venue_pkg', { + owner: build.package.owner, + 'created': Date.now(), + venue: ctx.party.identity.key.hash, + hash: buildHash, + workspace: workspacePath, + tarpath: Path.join(workspacePath, tarFileName), + settings: ctx.input.settings, + package: pkgWithoutOwner, + compressedBuild: Routines.Utils.base64.encode(compressedBrotliBuild) + }) + + debug('service created') + } else { + debug('need to update service?') + } + + if(ctx.input.staticTar){ + fs.writeFileSync( + Path.join(workspacePath, tarFileName), + ctx.input.staticTar + ) + } + + /*fs.writeFileSync( + Path.join(workspacePath, safeFileName+'.service.venue.bson'), + Routines.BSON.serializeBSONWithoutOptimiser(ctx.input.build) + )*/ + + fs.writeFileSync( + Path.join(workspacePath, safeFileName+'.service.venue.json'), + JSON.stringify(ctx.input.build, null, 2) + ) + + // verify build signature + + //ctx.input. + + // untar listed files + + // verify static file checksums match verified signatures + + // create db entry + + /* + + const compiledSrv = JSON.parse(ctx.input.service) + const serviceId = compiledSrv.package.name + '-' + compiledSrv.package.version + debug('addService', serviceId) + + let srvDoc = (await ctx.party.find() + .type('venue_srv') + .where('name').equals(compiledSrv.package.name) + .exec())[0] + + + + if(!srvDoc){ + debug('creating service') + srvDoc = await ctx.party.createDocument('venue_srv', { + name: compiledSrv.package.name, + 'created': (new Date()).toISOString(), + package: compiledSrv.package, + schemas: compiledSrv.schemas, + endpoints: compiledSrv.endpoints, + midddleware: compiledSrv.middleware, + middleware_order: compiledSrv.middleware_order + }) + + debug('service created') + } + else{ + + + try{ + + debug('updating service') + debug(srvDoc.data) + await srvDoc.mergeData({ + package: compiledSrv.package, + schemas: compiledSrv.schemas, + endpoints: compiledSrv.endpoints, + midddleware: compiledSrv.middleware, + middleware_order: compiledSrv.middleware_order + }) + + //debug(srvDoc.data) + + debug('saving doc') + + + await srvDoc.save() + } + catch(err){ + console.log(err) + } + debug('updated service') + }*/ + + return {done: true, package: srvDoc.data} + + //return {srv:srvDoc.data} + } +} \ No newline at end of file diff --git a/src/venue/endpoints/create-project.js b/src/venue/endpoints/create-project.js new file mode 100644 index 0000000..83c1f2a --- /dev/null +++ b/src/venue/endpoints/create-project.js @@ -0,0 +1,404 @@ +const fs = require('fs') +const Joi = require('joi') +const Path = require('path') +const Hoek = require('@hapi/hoek') +const {Message, Routines, Identity} = require('@dataparty/crypto') +const debug = require('debug')('dataparty.endpoint.create-project') + +const process = require('process') +const tar = require('tar') +const zlib = require('zlib') + +const IEndpoint = require('../../service/iendpoint') + +const typedArraySchema = (value, helpers) => { + // 1. Ensure the value is an instance of a TypedArray (e.g., Uint8Array) + if (!(value instanceof Uint8Array)) { + return helpers.message({ custom: '"value" must be a Uint8Array' }); + } + + return value +} + +module.exports = class CreateProjectEndpoint extends IEndpoint { + + static get Name(){ + return 'create-project' + } + + + static get Description(){ + return 'Create venue project' + } + + + + static get MiddlewareConfig(){ + return { + pre: { + decrypt: true, + ephemeral_session: true, + validate: Joi.object().keys({ + project: Joi.object().keys({ + owner: Joi.string().required(), + //created: Joi.number(), + + name: Joi.string().required(), + version: Joi.string().required(), + venue: Joi.string().required(), + domain: Joi.string(), + + i2p: Joi.object().keys({ + address: Joi.string(), + public: Joi.string(), + securePrivate: Joi.string() + }), + party: Joi.array().items(Joi.object().keys({ + name: Joi.string(), + type: Joi.string().required(), + tingo: { path: Joi.string() }, + loki: { path: Joi.string() }, + peer: { + venue: Joi.string(), + remoteIdentity: Joi.string() + }, + key: { + hash: Joi.string(), + securePrivate: Joi.string() + }, + settings: { noCache: Joi.string() }, + defaultConfig: Joi.string() + })), + routes: Joi.array().items(Joi.object().keys({ + prefix: Joi.string(), + party: Joi.string(), + package: Joi.object().keys({ + owner: Joi.string(), + name: Joi.string().required(), + version: Joi.string(), + branch: Joi.string(), + hash: Joi.string() + }), + settings: { + sendFullErrors: Joi.boolean().required(), + useNative: Joi.boolean().required() + } + })), + files: Joi.object().pattern(Joi.string(), Joi.object().keys({ + tar: Joi.string(), + hash: Joi.string().required(), + size: Joi.number().required(), + files: Joi.object().pattern(Joi.string(), Joi.object().keys({ + hash: Joi.string().required(), + size: Joi.number().required() + })) + })), + signatures: Joi.object().pattern(Joi.string(), Joi.string()).required() + }).required(), + staticTar: Joi.any().custom(typedArraySchema) + }) + }, + post: { + encrypt: true, + validate: Joi.object().keys(null).description('any output allowed') + } + } + } + + static async run(ctx){ + + if(ctx.party.identity.key.hash != ctx.input.project.venue){ + throw new Error('project venue does not match this host') + } + + let {signatures, ...projectWithoutSig} = ctx.input.project + + const projectOwnerIdentityDoc = (await ctx.party.find() + .type('public_key') + .where('hash').equals( ctx.input.project.owner ) + .exec() + )[0] + + if(!projectOwnerIdentityDoc){ + throw new Error('project owner not authorized') + } + + debug('found project owner', projectOwnerIdentityDoc.hash) + + const projectOwnerIdentity = Identity.fromJSON({ + id: '', + key: { + type: projectOwnerIdentityDoc.data.type, + hash: projectOwnerIdentityDoc.data.hash, + public: projectOwnerIdentityDoc.data.public + } + }) + + + debug('inflated identity') + + const devSig = Routines.Utils.base64.decode(signatures[ctx.input.project.owner]) + + let signedProjectMsg = new Message({ + msg: projectWithoutSig, + sig: devSig + }) + + debug('sigs', signatures) + + debug('verifying package signature') + + await signedProjectMsg.assertVerified(projectOwnerIdentity, true) + + debug('verified package signature') + + const safeFileName = ctx.input.project.name.replace('/', '-') + const tarFileName = safeFileName+'.project.files.venue.tgz' + + const projectFiles = ctx.input.project.files[tarFileName] + + if(projectFiles && !ctx.input.staticTar){ + throw new Error('project definition lists a static tar but none was uploaded') + } + + if(ctx.input.staticTar){ + const tarHash = Routines.Utils.hash(ctx.input.staticTar) + const tarHash64 = Routines.Utils.base64.encode( tarHash ) + + if(projectFiles && projectFiles.hash != tarHash64){ + throw new Error("staticTar hash doesn't match project definition") + } + + debug('verified staticTar') + } + + // check route packages are valid + let packages = {} + let tarList = [] + + for(let route of ctx.input.project.routes){ + console.log(route) + let pkgDoc = (await ctx.party.find() + .type('venue_pkg') + .or() + .where('package.name').equals(route.package.name) + .where('package.githash').equals(route.package.githash) + .sort('-created') + .limit(1) + .exec())[0] + + if(!pkgDoc){ + throw new Error(`package ${JSON.stringify(route.package)} not found. required by ${route.prefix}`) + } + + const {compressedBuild, ...printablePkg} = pkgDoc.data + + console.log('found package', printablePkg) + + let pkgTarPath = pkgDoc.data.tarpath + + if(pkgTarPath && pkgTarPath.length > 0){ + tarList.push(pkgTarPath) + } + + packages[route.prefix] = pkgDoc.data + } + + + debug('verified project - '+ctx.input.project.name+'@'+ctx.input.project.version) + + const projectBSON = Routines.BSON.serializeBSONWithoutOptimiser(projectWithoutSig) + + const projectHash = Routines.Utils.base64.encode( + Routines.Utils.hash( + projectBSON + ) + ) + + const safeProjectHash = projectHash.replace(/\//g, "-") + + debug('\t'+'hash', projectHash) + + const projectWorkspace = Path.join('projects/',safeFileName, ctx.input.project.version, safeProjectHash) + + const config = ctx.party.config + + const workspacePath = await config.touchDir(projectWorkspace) + + + debug('\t'+'workspace - local', projectWorkspace) + debug('\t'+'workspace - global', workspacePath) + + + + const compressedBrotliBuild = zlib.brotliCompressSync(JSON.stringify(ctx.input.project)) + + const project = ctx.input.project + const projectId = project.name + '@' + project.version + debug('addProject', projectId) + + let projectDoc = (await ctx.party.find() + .type('venue_project') + .where('project.name').equals(project.name) + .where('project.version').equals(project.version) + .where('hash').equals(projectHash) + .exec())[0] + + + if(!projectDoc){ + debug('creating project') + + const {owner, ...pkgWithoutOwner} = project + + projectDoc = await ctx.party.createDocument('venue_project', { + owner: project.owner, + created: Date.now(), + changed: Date.now(), + hash: projectHash, + workspace: workspacePath, + tarpath: Path.join(workspacePath, tarFileName), + project: ctx.input.project, + }) + + debug('project created') + } else { + debug('need to update project?') + } + + debug('extracting other tars', tarList) + + for(let tarPath of tarList){ + debug('extracting', tarPath) + + await tar.extract({ + cwd: workspacePath, + file: tarPath, + newer: true, + unlink: true, + uid: process.getuid(), + gid: process.getgid() + }, /*tarFileList*/ ) + } + + if(ctx.input.staticTar){ + debug('saving tar file', tarFileName) + fs.writeFileSync( + Path.join(workspacePath, tarFileName), + ctx.input.staticTar + ) + + debug('extracting contents') + + /*await tar.t({ + cwd: workspacePath, + file: Path.join(workspacePath, tarFileName), + onReadEntry: entry => { console.log('\t\t', entry) } + })*/ + + //const tarFileInfo = projectDoc.data.project.files[ tarFileName ] + + //if(projectFiles){ + const tarFileList = Object.keys(projectFiles.files) + + debug('file list - ', tarFileList) + + await tar.extract({ + cwd: workspacePath, + file: Path.join(workspacePath, tarFileName), + newer: true, + unlink: true, + uid: process.getuid(), + gid: process.getgid() + }, tarFileList ) + + //} + } + + /*fs.writeFileSync( + Path.join(workspacePath, safeFileName+'.service.venue.bson'), + Routines.BSON.serializeBSONWithoutOptimiser(ctx.input.build) + )*/ + + fs.writeFileSync( + Path.join(workspacePath, safeFileName+'.project.venue.json'), + JSON.stringify(ctx.input.project, null, 2) + ) + + await ctx.party.config.write('projects.'+ctx.input.project.name, projectHash) + + // verify build signature + + //ctx.input. + + // untar listed files + + // verify static file checksums match verified signatures + + // create db entry + + /* + + const compiledSrv = JSON.parse(ctx.input.service) + const projectId = compiledSrv.package.name + '-' + compiledSrv.package.version + debug('addService', projectId) + + let projectDoc = (await ctx.party.find() + .type('venue_srv') + .where('name').equals(compiledSrv.package.name) + .exec())[0] + + + + if(!projectDoc){ + debug('creating service') + projectDoc = await ctx.party.createDocument('venue_srv', { + name: compiledSrv.package.name, + 'created': (new Date()).toISOString(), + package: compiledSrv.package, + schemas: compiledSrv.schemas, + endpoints: compiledSrv.endpoints, + midddleware: compiledSrv.middleware, + middleware_order: compiledSrv.middleware_order + }) + + debug('service created') + } + else{ + + + try{ + + debug('updating service') + debug(projectDoc.data) + await projectDoc.mergeData({ + package: compiledSrv.package, + schemas: compiledSrv.schemas, + endpoints: compiledSrv.endpoints, + midddleware: compiledSrv.middleware, + middleware_order: compiledSrv.middleware_order + }) + + //debug(projectDoc.data) + + debug('saving doc') + + + await projectDoc.save() + } + catch(err){ + console.log(err) + } + debug('updated service') + }*/ + + console.log('restarting in 3 seconds...') + setTimeout(()=>{ + process.exit(), + 3000 + }) + + return {done: true, project: projectDoc.data} + + //return {srv:projectDoc.data} + } +} \ No newline at end of file diff --git a/src/venue/endpoints/create-service.js b/src/venue/endpoints/file-create.js similarity index 89% rename from src/venue/endpoints/create-service.js rename to src/venue/endpoints/file-create.js index ac3e324..d2e20d6 100644 --- a/src/venue/endpoints/create-service.js +++ b/src/venue/endpoints/file-create.js @@ -1,19 +1,19 @@ const Joi = require('joi') const Hoek = require('@hapi/hoek') const {Message, Routines} = require('@dataparty/crypto') -const debug = require('debug')('dataparty.endpoint.create-service') +const debug = require('debug')('dataparty.endpoint.file-create') const IEndpoint = require('../../service/iendpoint') -module.exports = class CreateSrvEndpoint extends IEndpoint { +module.exports = class FileCreateEndpoint extends IEndpoint { static get Name(){ - return 'create-service' + return 'file-create' } static get Description(){ - return 'Create venue service' + return 'Create venue package file' } static get MiddlewareConfig(){ @@ -21,13 +21,13 @@ module.exports = class CreateSrvEndpoint extends IEndpoint { pre: { decrypt: true, validate: Joi.object().keys({ - settings: Joi.object().keys({ + /*settings: Joi.object().keys({ enabled: Joi.boolean().default(true).required(), - domain: Joi.string().required(), - prefix: Joi.string().default('').required(), + //domain: Joi.string().required(), + staticPrefix: Joi.string().default('/'), sendFullErrors: Joi.boolean().default(false).required(), useNative: Joi.boolean().default(false).required() - }).required(), + }).required(),*/ service: Joi.object().keys({ package: Joi.object().keys({ name: Joi.string().required(), diff --git a/src/venue/endpoints/file-download.js b/src/venue/endpoints/file-download.js new file mode 100644 index 0000000..e69de29 diff --git a/src/venue/endpoints/file-write.js b/src/venue/endpoints/file-write.js new file mode 100644 index 0000000..e69de29 diff --git a/src/venue/example-project.js b/src/venue/example-project.js new file mode 100644 index 0000000..315dd1f --- /dev/null +++ b/src/venue/example-project.js @@ -0,0 +1,25 @@ +module.exports = { + owner: 'null', + name: 'example-project', + version: '1.0', + + venue: 'dataparty-venue', + domain: 'api.dataparty.xyz', + + routes: [{ + prefix: '/api', + party:'SYSTEM', + package: { + name: '@dataparty/api' + }, + settings: { + sendFullErrors: false, + useNative: false + } + }], + files: [ + 'public/*', + 'public/dist/dataparty-browser.*', + 'public/node_modules/argon2-browser/dist/*' + ] + } \ No newline at end of file diff --git a/src/venue/middleware/pre/decrypt-nacl.js b/src/venue/middleware/pre/decrypt-nacl.js index 069812a..857f2e9 100644 --- a/src/venue/middleware/pre/decrypt-nacl.js +++ b/src/venue/middleware/pre/decrypt-nacl.js @@ -32,7 +32,7 @@ module.exports = class DecryptNaCl extends IMiddleware { if (!Config){ return } if(!context.input || !context.input.enc){ - throw new Error('insecure message') + throw new Error('insecure message - here') } context.debug('input', context.input, typeof context.input) diff --git a/src/venue/public/p2p-test.html b/src/venue/public/p2p-test.html index 6c57aa8..a45f222 100644 --- a/src/venue/public/p2p-test.html +++ b/src/venue/public/p2p-test.html @@ -256,7 +256,12 @@

Offers recieved

await hostLocal.start() - matchMaker = new Dataparty.MatchMakerClient(hostLocal.privateIdentity, null) + matchMaker = new Dataparty.MatchMakerClient( + hostLocal.privateIdentity, + null, + 'https://api.dataparty.xyz/api', + 'wss://api.dataparty.xyz/ws' + ) await matchMaker.start() diff --git a/src/venue/schema/venue_package.js b/src/venue/schema/venue_package.js new file mode 100644 index 0000000..8ce3914 --- /dev/null +++ b/src/venue/schema/venue_package.js @@ -0,0 +1,55 @@ +'use strict' + +const debug = require('debug')('venue.venue_pkg') + +const ISchema = require('../../bouncer/ischema') + +//const Utils = ISchema.Utils + + +class VenuePkg extends ISchema { + + static get Type () { return 'venue_pkg' } + + static get Schema(){ + return { + owner: {type: String, required: true, index: true}, //public_key.key.hash + created: {type: Number, required: true}, + changed: {type: Number}, + venue: {type: String}, + workspace: {type: String, required: true}, + tarpath: {type: String}, + hash: {type: String, required: true, index: true}, + settings: { + //enabled: {type: Boolean, required: true}, + staticPrefix: String, + sendFullErrors: {type: Boolean, required: true}, + useNative: {type: Boolean, required: true}, + defaultConfig: {type: Object} + }, + package: { + name: {type: String, required: true, index: true}, + version: {type: String, required: true, index: true}, + githash: {type: String, required: true}, + branch: {type: String, required: true}, + }, + compressedBuild: {type: String, required: true} //! brotli compressed + } + } + + static setupSchema(schema){ + //schema.index({ 'package.name': 1 }, {unique: true}) + return schema + } + + static permissions (context) { + return { + read: false, + new: false, + change: false + } + } +} + + +module.exports = VenuePkg \ No newline at end of file diff --git a/src/venue/schema/venue_project.js b/src/venue/schema/venue_project.js new file mode 100644 index 0000000..2d65721 --- /dev/null +++ b/src/venue/schema/venue_project.js @@ -0,0 +1,100 @@ +'use strict' + +const debug = require('debug')('venue.venue_project') + +const ISchema = require('../../bouncer/ischema') + +const Utils = ISchema.Utils + + +class VenueProject extends ISchema { + + static get Type () { return 'venue_project' } + + static get Schema(){ + return { + owner: {type: String, required: true}, + created: {type: Number, required: true}, + changed: {type: Number, required: true}, + workspace: {type: String, required: true}, + tarpath: {type: String}, + hash: {type: String, required: true, index: true}, + + enabled: {type: Boolean}, + + + project: { + owner: {type: String, required: true, index: true}, //public_key.key.hash + //created: {type: Number, required: true}, + + name: {type: String, required: true, index: true}, + version: {type: String, required: true, index: true}, + venue: {type: String}, + domain: {type: String, index: true/*, unique: true*/}, + + i2p: { + address: String, + public: String, + securePrivate: String + }, + + party: [{ + name: String, + type: {type: String, enum: ['tingo', 'loki', 'peer', 'mongo']}, + tingo: { + path: String + }, + loki: { + path: String + }, + peer: { + venue: String, + remoteIdentity: String + }, + key: { + hash: String, + securePrivate: String + }, + settings: { + noCache: Boolean, + }, + defaultConfig: String + }], + routes: [{ + prefix: String, + party: String, + package: { + owner: String, + name: {type: String, required: true}, + version: String, + branch: String, + hash: String, + }, + settings: { + sendFullErrors: {type: Boolean, required: true}, + useNative: {type: Boolean, required: true}, + } + }], + files: Object, + signatures: {type: Object, required: true} + } + + } + } + + static setupSchema(schema){ + //schema.index({ 'package.name': 1 }, {unique: true}) + return schema + } + + static permissions (context) { + return { + read: false, + new: false, + change: false + } + } +} + + +module.exports = VenueProject \ No newline at end of file diff --git a/src/venue/schema/venue_service.js b/src/venue/schema/venue_service.js index f5f8d08..181a556 100644 --- a/src/venue/schema/venue_service.js +++ b/src/venue/schema/venue_service.js @@ -2,7 +2,6 @@ const debug = require('debug')('venue.venue_srv') - const ISchema = require('../../bouncer/ischema') const Utils = ISchema.Utils @@ -31,12 +30,6 @@ class VenueSrv extends ISchema { version: {type: String, required: true}, githash: {type: String, required: true}, branch: {type: String, required: true} - }, - compressedBuild: {type: String, required: true}, //! zlib compressed - signature: { - timestamp: {type: Number, required: true}, - type: {type: String, required: true, maxlength: 10}, - value: {type: String, required: true} } } } diff --git a/src/venue/venue-host.js b/src/venue/venue-host.js index fef4471..6e72731 100644 --- a/src/venue/venue-host.js +++ b/src/venue/venue-host.js @@ -5,8 +5,8 @@ const Dataparty = require('../index') const VenueService = require('./venue-service') -const VenueServiceSchema = require('./dataparty/@dataparty-venue.dataparty-schema.json') -const VenueSrv = require('./dataparty/@dataparty-venue.dataparty-service.json') +const VenueServiceSchema = require('./dataparty/@dataparty-venue.schema.venue.json') +const VenueSrv = require('./dataparty/@dataparty-venue.service.venue.json') async function loadService(runnerRouter, settings, serviceFilePath){ @@ -55,7 +55,8 @@ async function main(){ const CloudFlareIpFilter = { options: { - mode: 'allow' + mode: 'allow', + //trustProxy: true }, ips: [ '173.245.48.0/20', @@ -79,14 +80,15 @@ async function main(){ '2405:b500::/32', '2405:8100::/32', '2a06:98c0::/29', - '2c0f:f248::/32' + '2c0f:f248::/32', + '10.115.68.55/32' ] } let party = new Dataparty.TingoParty({ path: path+'/db', model: VenueServiceSchema, - config: new Dataparty.Config.JsonFileConfig({basePath: path+'/config'}), + config: new Dataparty.Config.JsonFileConfig({basePath: path}), noCache: false }) @@ -101,7 +103,7 @@ async function main(){ party, service, sendFullErrors: true, useNative: false, - prefix: 'api/' + prefix: 'venue/' }) let runnerRouter = new Dataparty.RunnerRouter(runner) @@ -114,7 +116,7 @@ async function main(){ trust_proxy: true, wsEnabled: true, ssl_key, ssl_cert, - listenUri: 'https://0.0.0.0:443', + listenUri: 'https://0.0.0.0:3000', staticPath: Path.join(__dirname,'public'), staticPrefix: '/venue/', ipFilter: CloudFlareIpFilter @@ -126,7 +128,7 @@ async function main(){ debug('started') console.log('partying') - +/* await loadService(runnerRouter, { enabled: true, domain: 'postquantum.one', @@ -135,7 +137,7 @@ async function main(){ sendFullErrors: true, useNative: false - }, '/home/ubuntu/match-maker/dataparty/@datapartyjs-match-maker.dataparty-service.json') + }, '/home/ubuntu/match-maker/dataparty/@datapartyjs-match-maker.dataparty-service.json')*/ } diff --git a/src/venue/venue-service.js b/src/venue/venue-service.js index f5c7ad3..e98511e 100644 --- a/src/venue/venue-service.js +++ b/src/venue/venue-service.js @@ -11,19 +11,38 @@ class VenueService extends DatapartySrv.IService { let builder = new DatapartySrv.ServiceBuilder(this) - builder.addSchema(Path.join(__dirname, './schema/venue_service.js')) + + builder.addSchema(Path.join(__dirname, './schema/venue_package.js')) + builder.addSchema(Path.join(__dirname, './schema/venue_project.js')) + + builder.addSchema(DatapartySrv.schema_paths.public_key) + builder.addSchema(DatapartySrv.schema_paths.session_key) builder.addMiddleware(DatapartySrv.middleware_paths.pre.decrypt) builder.addMiddleware(DatapartySrv.middleware_paths.pre.validate) + builder.addMiddleware(DatapartySrv.middleware_paths.pre.ephemeral_session) + builder.addMiddleware(DatapartySrv.middleware_paths.post.validate) builder.addMiddleware(DatapartySrv.middleware_paths.post.encrypt) builder.addEndpoint(DatapartySrv.endpoint_paths.identity) builder.addEndpoint(DatapartySrv.endpoint_paths.version) + builder.addEndpoint(DatapartySrv.endpoint_paths.key_announce) + + builder.addEndpoint(Path.join(__dirname, './endpoints/create-package.js')) + builder.addEndpoint(Path.join(__dirname, './endpoints/create-project.js')) + + + builder.addTask(DatapartySrv.task_paths.cleanup_ephemeral_sessions) - builder.addEndpoint(Path.join(__dirname, './endpoints/create-service.js')) builder.addAuth(Path.join(__dirname, './auth.js')) + + builder.addFiles(__dirname, [ + 'public/*', + 'public/dist/dataparty-browser.*', + 'public/node_modules/argon2-browser/dist/*' + ], { nodir: true, follow: true }) } }