12657192105

Committed 07 Jan 2025 06:32PM UTC coverage: 95.778% (+0.04%) from 95.741%

Build # 12657192105

Build Type

Pull #514

github

Committed by

macterra

Commit Message

Added DbMongo class

Pull Request Pull Request #514: refactor: Gatekeeper classes

Run Details

1036 of 1103 branches covered (93.93%)

Branch coverage included in aggregate %.

62 of 67 new or added lines in 1 file covered. (92.54%)

7 existing lines in 1 file now uncovered.

1482 of 1526 relevant lines covered (97.12%)

445.28 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

96.45

/packages/gatekeeper/src/gatekeeper-lib.js

import canonicalize from 'canonicalize';
import * as cipher from '@mdip/cipher/node';
import { copyJSON } from '@mdip/common/utils';
import {
    InvalidDIDError,
    InvalidParameterError,
    InvalidOperationError
} from '@mdip/common/errors';
import IPFS from '@mdip/ipfs';
import config from './config.js';

const validVersions = [1];
const validTypes = ['agent', 'asset'];
// We'll leave TESS here so existing TESS DIDs are not deleted
// Remove TESS when we switch to did:mdip
const validRegistries = ['local', 'hyperswarm', 'TESS', 'TBTC', 'TFTC'];
let supportedRegistries = null;

let db = null;
let eventsQueue = [];
let eventsSeen = {};
let verifiedDIDs = {};
let isProcessingEvents = false;
const ipfs = new IPFS({ minimal: true });

export async function start(options = {}) {
    if (options.db) {
        db = options.db;
    }
    else {
        throw new InvalidParameterError('missing options.db');
    }

    // Only used for unit testing
    if (options.console) {
        // eslint-disable-next-line
        console = options.console;
    }

    return ipfs.start();
}

export async function stop() {
    return ipfs.stop();
}

export async function verifyDb(options = {}) {
    const { chatty = true } = options;

    const dids = await getDIDs();
    const total = dids.length;
    let n = 0;
    let expired = 0;
    let invalid = 0;
    let verified = Object.keys(verifiedDIDs).length;

    if (chatty) {
        console.time('verifyDb');
    }

    for (const did of dids) {
        n += 1;

        if (verifiedDIDs[did]) {
            continue;
        }

        let validUntil = null;

        try {
            const doc = await resolveDID(did, { verify: true });
            validUntil = doc.mdip.validUntil;
        }
        catch (error) {
            if (chatty) {
                console.log(`removing ${n}/${total} ${did} invalid`);
            }
            invalid += 1;
            await db.deleteEvents(did);
            continue;
        }

        if (validUntil) {
            const expires = new Date(validUntil);
            const now = new Date();

            if (expires < now) {
                if (chatty) {
                    console.log(`removing ${n}/${total} ${did} expired`);
                }
                await db.deleteEvents(did);
                expired += 1;
            }
            else {
                const minutesLeft = Math.round((expires.getTime() - now.getTime()) / 60 / 1000);

                if (chatty) {
                    console.log(`expiring ${n}/${total} ${did} in ${minutesLeft} minutes`);
                }
                verified += 1;
            }
        }
        else {
            if (chatty) {
                console.log(`verifying ${n}/${total} ${did} OK`);
            }
            verifiedDIDs[did] = true;
            verified += 1;
        }
    }

    if (chatty) {
        console.timeEnd('verifyDb');
    }

    return { total, verified, expired, invalid };
}

export async function checkDIDs(options = {}) {
    let { chatty = false, dids } = options;

    if (!dids) {
        dids = await getDIDs();
    }

    const total = dids.length;
    let n = 0;
    let agents = 0;
    let assets = 0;
    let confirmed = 0;
    let unconfirmed = 0;
    let ephemeral = 0;
    let invalid = 0;
    const byRegistry = {};
    const byVersion = {};

    for (const did of dids) {
        n += 1;
        try {
            const doc = await resolveDID(did);
            if (chatty) {
                console.log(`resolved ${n}/${total} ${did} OK`);
            }

            if (doc.mdip.type === 'agent') {
                agents += 1;
            }

            if (doc.mdip.type === 'asset') {
                assets += 1;
            }

            if (doc.didDocumentMetadata.confirmed) {
                confirmed += 1;
            }
            else {
                unconfirmed += 1;
            }

            if (doc.mdip.validUntil) {
                ephemeral += 1;
            }

            const registry = doc.mdip.registry;
            byRegistry[registry] = (byRegistry[registry] || 0) + 1;

            const version = doc.didDocumentMetadata.version;
            byVersion[version] = (byVersion[version] || 0) + 1;
        }
        catch (error) {
            invalid += 1;
            if (chatty) {
                console.log(`can't resolve ${n}/${total} ${did} ${error}`);
            }
        }
    }

    const byType = { agents, assets, confirmed, unconfirmed, ephemeral, invalid };
    return { total, byType, byRegistry, byVersion };
}

export async function initRegistries(csvRegistries) {
    if (!csvRegistries) {
        supportedRegistries = validRegistries;
    }
    else {
        const registries = csvRegistries.split(',').map(registry => registry.trim());
        supportedRegistries = [];

        for (const registry of registries) {
            if (validRegistries.includes(registry)) {
                supportedRegistries.push(registry);
            }
            else {
                throw new InvalidParameterError(`registry=${registry}`);
            }
        }
    }

    return supportedRegistries;
}

export async function listRegistries() {
    return supportedRegistries || validRegistries;
}

// For testing purposes
export async function resetDb() {
    await db.resetDb();
    verifiedDIDs = {};
}

export async function generateCID(operation) {
    return ipfs.add(JSON.parse(canonicalize(operation)));
}

export async function generateDID(operation) {
    const cid = await generateCID(operation);
    return `${config.didPrefix}:${cid}`;
}

export async function verifyOperation(operation) {
    try {
        if (operation.type === 'create') {
            return verifyCreateOperation(operation);
        }

        if (operation.type === 'update' || operation.type === 'delete') {
            const doc = await resolveDID(operation.did);
            return verifyUpdateOperation(operation, doc);
        }
    }
    catch (error) {
        return false;
    }
}

function verifyDIDFormat(did) {
    return did && typeof did === 'string' && did.startsWith('did:');
}

function verifyDateFormat(time) {
    const date = new Date(time);
    return !isNaN(date.getTime());
}

function verifyHashFormat(hash) {
    // Check if hash is a hexadecimal string of length 64
    const hex64Regex = /^[a-f0-9]{64}$/i;
    return hex64Regex.test(hash);
}

function verifySignatureFormat(signature) {
    if (!signature) {
        return false;
    }

    if (!verifyDateFormat(signature.signed)) {
        return false;
    }

    if (!verifyHashFormat(signature.hash)) {
        return false;
    }

    // eslint-disable-next-line
    if (signature.signer && !verifyDIDFormat(signature.signer)) {
        return false;
    }

    return true;
}

async function verifyCreateOperation(operation) {
    if (!operation) {
        throw new InvalidOperationError('missing');
    }

    if (operation.type !== "create") {
        throw new InvalidOperationError(`type=${operation.type}`);
    }

    if (!verifyDateFormat(operation.created)) {
        // TBD ensure valid timestamp format
        throw new InvalidOperationError(`created=${operation.created}`);
    }

    if (!operation.mdip) {
        throw new InvalidOperationError('mdip');
    }

    if (!validVersions.includes(operation.mdip.version)) {
        throw new InvalidOperationError(`mdip.version=${operation.mdip.version}`);
    }

    if (!validTypes.includes(operation.mdip.type)) {
        throw new InvalidOperationError(`mdip.type=${operation.mdip.type}`);
    }

    if (!validRegistries.includes(operation.mdip.registry)) {
        throw new InvalidOperationError(`mdip.registry=${operation.mdip.registry}`);
    }

    if (!verifySignatureFormat(operation.signature)) {
        throw new InvalidOperationError('signature');
    }

    if (operation.mdip.validUntil && !verifyDateFormat(operation.mdip.validUntil)) {
        throw new InvalidOperationError(`mdip.validUntil=${operation.mdip.validUntil}`);
    }

    if (operation.mdip.type === 'agent') {
        if (!operation.publicJwk) {
            throw new InvalidOperationError('publicJwk');
        }

        const operationCopy = copyJSON(operation);
        delete operationCopy.signature;

        const msgHash = cipher.hashJSON(operationCopy);
        return cipher.verifySig(msgHash, operation.signature.value, operation.publicJwk);
    }

    if (operation.mdip.type === 'asset') {
        if (operation.controller !== operation.signature?.signer) {
            throw new InvalidOperationError('signer is not controller');
        }

        const doc = await resolveDID(operation.signature.signer, { confirm: true, atTime: operation.signature.signed });

        if (doc.mdip.registry === 'local' && operation.mdip.registry !== 'local') {
            throw new InvalidOperationError(`non-local registry=${operation.mdip.registry}`);
        }

        const operationCopy = copyJSON(operation);
        delete operationCopy.signature;
        const msgHash = cipher.hashJSON(operationCopy);
        // TBD select the right key here, not just the first one
        const publicJwk = doc.didDocument.verificationMethod[0].publicKeyJwk;
        return cipher.verifySig(msgHash, operation.signature.value, publicJwk);
    }

    throw new InvalidOperationError(`mdip.type=${operation.mdip.type}`);
}

async function verifyUpdateOperation(operation, doc) {
    if (!verifySignatureFormat(operation.signature)) {
        throw new InvalidOperationError('signature');
    }

    if (!doc?.didDocument) {
        throw new InvalidOperationError('doc.didDocument');
    }

    if (doc.didDocumentMetadata?.deactivated) {
        throw new InvalidOperationError('DID deactivated');
    }

    if (doc.didDocument.controller) {
        // This DID is an asset, verify with controller's keys
        const controllerDoc = await resolveDID(doc.didDocument.controller, { confirm: true, atTime: operation.signature.signed });
        return verifyUpdateOperation(operation, controllerDoc);
    }

    if (!doc.didDocument.verificationMethod) {
        throw new InvalidOperationError('doc.didDocument.verificationMethod');
    }

    const signature = operation.signature;
    const jsonCopy = copyJSON(operation);
    delete jsonCopy.signature;
    const msgHash = cipher.hashJSON(jsonCopy);

    if (signature.hash && signature.hash !== msgHash) {
        return false;
    }

    // TBD get the right signature, not just the first one
    const publicJwk = doc.didDocument.verificationMethod[0].publicKeyJwk;
    return cipher.verifySig(msgHash, signature.value, publicJwk);
}

export async function createDID(operation) {
    const valid = await verifyCreateOperation(operation);

    if (valid) {
        const did = await generateDID(operation);
        const ops = await exportDID(did);

        // Check to see if we already have this DID in the db
        if (ops.length === 0) {
            await db.addEvent(did, {
                registry: 'local',
                time: operation.created,
                ordinal: 0,
                operation,
                did
            });

            // Create events are distributed only by hyperswarm
            // (because the DID's registry specifies where to look for *update* events)
            // Don't distribute local DIDs
            if (operation.mdip.registry !== 'local') {
                await db.queueOperation('hyperswarm', operation);
            }
        }

        return did;
    }
    else {
        throw new InvalidOperationError('signature');
    }
}

export async function generateDoc(anchor) {
    let doc = {};
    try {
        if (!anchor?.mdip) {
            return {};
        }

        if (!validVersions.includes(anchor.mdip.version)) {
            return {};
        }

        if (!validTypes.includes(anchor.mdip.type)) {
            return {};
        }

        if (!validRegistries.includes(anchor.mdip.registry)) {
            return {};
        }

        const did = await generateDID(anchor);

        if (anchor.mdip.type === 'agent') {
            // TBD support different key types?
            doc = {
                "@context": "https://w3id.org/did-resolution/v1",
                "didDocument": {
                    "@context": ["https://www.w3.org/ns/did/v1"],
                    "id": did,
                    "verificationMethod": [
                        {
                            "id": "#key-1",
                            "controller": did,
                            "type": "EcdsaSecp256k1VerificationKey2019",
                            "publicKeyJwk": anchor.publicJwk,
                        }
                    ],
                    "authentication": [
                        "#key-1"
                    ],
                },
                "didDocumentMetadata": {
                    "created": anchor.created,
                },
                "didDocumentData": {},
                "mdip": anchor.mdip,
            };
        }

        if (anchor.mdip.type === 'asset') {
            doc = {
                "@context": "https://w3id.org/did-resolution/v1",
                "didDocument": {
                    "@context": ["https://www.w3.org/ns/did/v1"],
                    "id": did,
                    "controller": anchor.controller,
                },
                "didDocumentMetadata": {
                    "created": anchor.created,
                },
                "didDocumentData": anchor.data,
                "mdip": anchor.mdip,
            };
        }
    }
    catch (error) {
        // console.error(error);
    }

    return doc;
}

export async function resolveDID(did, options = {}) {
    const { atTime, atVersion, confirm = false, verify = false } = options;

    const events = await db.getEvents(did);

    if (events.length === 0) {
        throw new InvalidDIDError();
    }

    const anchor = events[0];
    let doc = await generateDoc(anchor.operation);

    if (atTime && new Date(doc.mdip.created) > new Date(atTime)) {
        // TBD What to return if DID was created after specified time?
    }

    let version = 1; // initial version is version 1 by definition
    let confirmed = true; // create event is always confirmed by definition

    doc.didDocumentMetadata.version = version;
    doc.didDocumentMetadata.confirmed = confirmed;

    for (const { time, operation, registry, blockchain } of events) {
        const opid = await generateCID(operation);

        if (operation.type === 'create') {
            if (verify) {
                const valid = await verifyCreateOperation(operation);

                if (!valid) {
                    throw new InvalidOperationError('signature');
                }
            }
            doc.mdip.opid = opid;
            continue;
        }

        if (atTime && new Date(time) > new Date(atTime)) {
            break;
        }

        if (atVersion && version === atVersion) {
            break;
        }

        confirmed = confirmed && doc.mdip.registry === registry;

        if (confirm && !confirmed) {
            break;
        }

        if (verify) {
            const valid = await verifyUpdateOperation(operation, doc);

            if (!valid) {
                throw new InvalidOperationError('signature');
            }

            // TEMP during did:test, operation.previd is optional
            if (operation.previd && operation.previd !== doc.mdip.opid) {
                throw new InvalidOperationError('previd');
            }
        }

        if (operation.type === 'update') {
            // Increment version
            version += 1;

            // TBD if registry change in operation.doc.didDocumentMetadata.mdip,
            // fetch updates from new registry and search for same operation
            doc = operation.doc;
            doc.didDocumentMetadata.updated = time;
            doc.didDocumentMetadata.version = version;
            doc.didDocumentMetadata.confirmed = confirmed;
            doc.mdip.opid = opid;

            if (blockchain) {
                doc.mdip.registration = blockchain;
            }
            else {
                delete doc.mdip.registration;
            }
        }
        else if (operation.type === 'delete') {
            doc.didDocument = {};
            doc.didDocumentData = {};
            doc.didDocumentMetadata.deactivated = true;
            doc.didDocumentMetadata.deleted = time;
            doc.didDocumentMetadata.confirmed = confirmed;
            doc.mdip.opid = opid;

            if (blockchain) {
                doc.mdip.registration = blockchain;
            }
            else {
                delete doc.mdip.registration;
            }
        }
        else {
            if (verify) {
                throw new InvalidOperationError('signature');
            }

            // console.error(`unknown type ${operation.type}`);
        }
    }

    return copyJSON(doc);
}

export async function updateDID(operation) {
    const doc = await resolveDID(operation.did);
    const updateValid = await verifyUpdateOperation(operation, doc);

    if (!updateValid) {
        return false;
    }

    const registry = doc.mdip.registry;

    await db.addEvent(operation.did, {
        registry: 'local',
        time: operation.signature.signed,
        ordinal: 0,
        operation,
        did: operation.did
    });

    if (registry === 'local') {
        return true;
    }

    await db.queueOperation(registry, operation);

    if (registry !== 'hyperswarm') {
        await db.queueOperation('hyperswarm', operation);
    }

    return true;
}

export async function deleteDID(operation) {
    return updateDID(operation);
}

export async function getDIDs(options = {}) {
    let { dids, updatedAfter, updatedBefore, confirm, verify, resolve } = options;
    if (!dids) {
        const keys = await db.getAllKeys();
        dids = keys.map(key => `${config.didPrefix}:${key}`);
    }

    if (updatedAfter || updatedBefore || resolve) {
        const start = updatedAfter ? new Date(updatedAfter) : null;
        const end = updatedBefore ? new Date(updatedBefore) : null;
        const response = [];

        for (const did of dids) {
            const doc = await resolveDID(did, { confirm, verify });
            const updated = new Date(doc.didDocumentMetadata.updated || doc.didDocumentMetadata.created);

            if (start && updated <= start) {
                continue;
            }

            if (end && updated >= end) {
                continue;
            }

            response.push(resolve ? doc : did);
        }

        return response;
    }

    return dids;
}

export async function exportDID(did) {
    return db.getEvents(did);
}

export async function exportDIDs(dids) {
    if (!dids) {
        dids = await getDIDs();
    }

    const batch = [];

    for (const did of dids) {
        batch.push(await exportDID(did));
    }

    return batch;
}

export async function importDIDs(dids) {
    return importBatch(dids.flat());
}

export async function removeDIDs(dids) {
    if (!Array.isArray(dids)) {
        throw new InvalidParameterError('dids');
    }

    for (const did of dids) {
        await db.deleteEvents(did);
    }

    return true;
}

async function importEvent(event) {
    if (!event.did) {
        if (event.operation.did) {
            event.did = event.operation.did;
        }
        else {
            event.did = await generateDID(event.operation);
        }
    }

    const did = event.did;
    const currentEvents = await db.getEvents(did);

    const match = currentEvents.find(item => item.operation.signature.value === event.operation.signature.value);

    if (match) {
        const first = currentEvents[0];
        const nativeRegistry = first.operation.mdip.registry;

        if (match.registry === nativeRegistry) {
            // If this event is already confirmed on the native registry, no need to update
            return false;
        }

        if (event.registry === nativeRegistry) {
            // If this import is on the native registry, replace the current one
            const index = currentEvents.indexOf(match);
            currentEvents[index] = event;
            await db.setEvents(did, currentEvents);
            return true;
        }

        return false;
    }
    else {
        const ok = await verifyOperation(event.operation);

        if (ok) {
            // TEMP during did:test, operation.previd is optional
            if (currentEvents.length > 0 && event.operation.previd) {
                const lastEvent = currentEvents[currentEvents.length - 1];
                const opid = await generateCID(lastEvent.operation);

                if (opid !== event.operation.previd) {
                    throw new InvalidOperationError('previd');
                }
            }

            await db.addEvent(did, event);
            return true;
        }
        else {
            throw new InvalidOperationError('signature');
        }
    }
}

async function importEvents() {
    let tempQueue = eventsQueue;
    const total = tempQueue.length;
    let event = tempQueue.shift();
    let i = 0;
    let added = 0;
    let merged = 0;

    eventsQueue = [];

    while (event) {
        //console.time('importEvent');
        i += 1;
        try {
            const imported = await importEvent(event);

            if (imported) {
                added += 1;
                console.log(`import ${i}/${total}: added event for ${event.did}`);
            }
            else {
                merged += 1;
                console.log(`import ${i}/${total}: merged event for ${event.did}`);
            }
        }
        catch (error) {
            eventsQueue.push(event);
            console.log(`import ${i}/${total}: deferred event for ${event.did}`);
        }
        //console.timeEnd('importEvent');
        event = tempQueue.shift();
    }

    return { added, merged };
}

export async function processEvents() {
    if (isProcessingEvents) {
        return { busy: true };
    }

    let added = 0;
    let merged = 0;
    let done = false;

    try {
        isProcessingEvents = true;

        while (!done) {
            //console.time('importEvents');
            const response = await importEvents();
            //console.timeEnd('importEvents');

            added += response.added;
            merged += response.merged;

            done = (response.added === 0 && response.merged === 0);
        }
    }
    catch (error) {
        console.log(error);
    }
    finally {
        isProcessingEvents = false;
    }

    //console.log(JSON.stringify(eventsQueue, null, 4));
    const pending = eventsQueue.length;
    const response = { added, merged, pending };

    console.log(`processEvents: ${JSON.stringify(response)}`);

    return response;
}

export async function verifyEvent(event) {
    if (!event.registry || !event.time || !event.operation) {
        return false;
    }

    const eventTime = new Date(event.time).getTime();

    if (isNaN(eventTime)) {
        return false;
    }

    const operation = event.operation;

    if (!verifySignatureFormat(operation.signature)) {
        return false;
    }

    if (operation.type === 'create') {
        if (!operation.created) {
            return false;
        }

        if (!operation.mdip) {
            return false;
        }

        if (!validVersions.includes(operation.mdip.version)) {
            return false;
        }

        if (!validTypes.includes(operation.mdip.type)) {
            return false;
        }

        if (!validRegistries.includes(operation.mdip.registry)) {
            return false;
        }

        // eslint-disable-next-line
        if (operation.mdip.type === 'agent') {
            if (!operation.publicJwk) {
                return false;
            }
        }

        // eslint-disable-next-line
        if (operation.mdip.type === 'asset') {
            if (operation.controller !== operation.signature?.signer) {
                return false;
            }
        }
    }
    else if (operation.type === 'update') {
        const doc = operation.doc;

        if (!doc || !doc.didDocument || !doc.didDocumentMetadata || !doc.didDocumentData || !doc.mdip) {
            return false;
        }

        if (!operation.did) {
            return false;
        }
    }
    else if (operation.type === 'delete') {
        if (!operation.did) {
            return false;
        }
    }
    else {
        return false;
    }

    return true;
}

export async function importBatch(batch) {
    if (!batch || !Array.isArray(batch) || batch.length < 1) {
        throw new InvalidParameterError('batch');
    }

    let queued = 0;
    let rejected = 0;
    let processed = 0;

    for (let i = 0; i < batch.length; i++) {
        const event = batch[i];
        const ok = await verifyEvent(event);

        if (ok) {
            const eventKey = `${event.registry}/${event.operation.signature.hash}`;
            if (!eventsSeen[eventKey]) {
                eventsSeen[eventKey] = true;
                eventsQueue.push(event);
                queued += 1;
            }
            else {
                processed += 1;
            }
        }
        else {
            rejected += 1;
        }
    }

    return {
        queued,
        processed,
        rejected,
        total: eventsQueue.length
    };
}

export async function exportBatch(dids) {
    const allDIDs = await exportDIDs(dids);
    const nonlocalDIDs = allDIDs.filter(events => {
        if (events.length > 0) {
            const create = events[0];
            const registry = create.operation?.mdip?.registry;
            return registry && registry !== 'local'
        }
        return false;
    });

    const events = nonlocalDIDs.flat();
    return events.sort((a, b) => new Date(a.operation.signature.signed) - new Date(b.operation.signature.signed));
}

export async function getQueue(registry) {
    if (!validRegistries.includes(registry)) {
        throw new InvalidParameterError(`registry=${registry}`);
    }

    return db.getQueue(registry);
}

export async function clearQueue(registry, events) {
    if (!validRegistries.includes(registry)) {
        throw new InvalidParameterError(`registry=${registry}`);
    }

    return db.clearQueue(registry, events);
}

KeychainMDIP / kc / 12657192105

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous