From 9ba3bb7d17b6ea4a2ad0c976bdda5e12f4a85fa0 Mon Sep 17 00:00:00 2001 From: Sean Sica <23294618+seansica@users.noreply.github.com> Date: Thu, 30 Jul 2026 16:47:59 -0400 Subject: [PATCH] fix(release-tracks): scope graph migration to active relationships Ignore deprecated and revoked dangling relationship history during deterministic graph backfill while preserving strict validation for active relationships. Improve migration diagnostics and ensure manifest indexes are always established. --- .../deterministic-graph-migration.spec.js | 65 ++++++++++++++++++- docs/admin/release-track-graph-migration.md | 28 ++++---- docs/developer/TODO.md | 23 +++++++ ...-backfill-deterministic-snapshot-graphs.js | 40 ++++++++++-- ...viewDeterministicSnapshotGraphMigration.js | 9 +++ 5 files changed, 145 insertions(+), 20 deletions(-) diff --git a/app/tests/api/release-tracks/deterministic-graph-migration.spec.js b/app/tests/api/release-tracks/deterministic-graph-migration.spec.js index 7a149c2e..9e4689e2 100644 --- a/app/tests/api/release-tracks/deterministic-graph-migration.spec.js +++ b/app/tests/api/release-tracks/deterministic-graph-migration.spec.js @@ -25,6 +25,9 @@ describe('Deterministic snapshot graph migration', function () { let group; let relationship; let trackId; + const deprecatedDanglingRelationshipId = 'relationship--f7a41277-6599-49df-9567-82c9227fb8b5'; + const activeDanglingRelationshipId = 'relationship--932fabf0-2868-46ed-9453-41e33dab7f39'; + const missingEndpointId = 'campaign--5f4e747c-11d7-49ae-a947-a0f436879d62'; before(async function () { await database.initializeConnection(); @@ -110,9 +113,65 @@ describe('Deterministic snapshot graph migration', function () { ReleaseTrackGraphManifest.deleteMany({ track_id: trackId }), ReleaseTrackGraphManifestEntry.deleteMany({ track_id: trackId }), ]); + await mongoose.connection.db.collection('relationships').insertOne({ + workspace: {}, + stix: { + type: 'relationship', + spec_version: '2.1', + id: deprecatedDanglingRelationshipId, + created: new Date(timestamp), + modified: new Date(timestamp), + relationship_type: 'uses', + source_ref: missingEndpointId, + target_ref: technique.stix.id, + revoked: false, + x_mitre_deprecated: true, + object_marking_refs: [markingDefinitionId], + }, + }); + }); + + it('fails closed when an active latest relationship has a dangling endpoint', async function () { + const timestamp = new Date(); + await mongoose.connection.db.collection('relationships').insertOne({ + workspace: {}, + stix: { + type: 'relationship', + spec_version: '2.1', + id: activeDanglingRelationshipId, + created: timestamp, + modified: timestamp, + relationship_type: 'uses', + source_ref: group.stix.id, + target_ref: missingEndpointId, + revoked: false, + x_mitre_deprecated: false, + object_marking_refs: [markingDefinitionId], + }, + }); + + try { + await expect( + migration._private.run(mongoose.connection.db, { + dryRun: true, + }), + ).rejects.toMatchObject({ + message: expect.stringContaining(activeDanglingRelationshipId), + missing_relationship_endpoints: [ + expect.objectContaining({ + relationship_ref: activeDanglingRelationshipId, + missing_endpoints: [missingEndpointId], + }), + ], + }); + } finally { + await mongoose.connection.db + .collection('relationships') + .deleteOne({ 'stix.id': activeDanglingRelationshipId }); + } }); - it('supports a non-mutating dry run', async function () { + it('supports a non-mutating dry run with unrelated deprecated dangling data', async function () { const report = await migration._private.run(mongoose.connection.db, { dryRun: true, }); @@ -145,6 +204,10 @@ describe('Deterministic snapshot graph migration', function () { object_ref: technique.stix.id, object_modified: new Date(technique.stix.modified), }); + const deprecatedDanglingRelationship = await mongoose.connection.db + .collection('relationships') + .findOne({ 'stix.id': deprecatedDanglingRelationshipId }); + expect(deprecatedDanglingRelationship.workspace.relationship_endpoints).toBeUndefined(); const manifests = await ReleaseTrackGraphManifest.find({ track_id: trackId, diff --git a/docs/admin/release-track-graph-migration.md b/docs/admin/release-track-graph-migration.md index 227e562f..91855ca5 100644 --- a/docs/admin/release-track-graph-migration.md +++ b/docs/admin/release-track-graph-migration.md @@ -14,21 +14,25 @@ DATABASE_URL='mongodb://host/database' \ npm run preview:deterministic-snapshot-graphs ``` -The report includes the latest relationship revisions scanned, endpoint pins -that would be written, release-track snapshots found, and baseline manifests -that would be created. No database writes or indexes are created by this -command. - -The preview fails if a latest relationship references a source or target -object that no longer exists. Repair those dangling endpoints before -deployment. Snapshot graph capture fails closed rather than silently producing -an incomplete deterministic baseline. +The report includes the latest active relationship revisions scanned, endpoint +pins that would be written, release-track snapshots found, and baseline +manifests that would be created. No database writes or indexes are created by +this command. + +The preview fails if an active latest relationship references a source or +target object that no longer exists. The error identifies the affected +relationship and missing endpoint IDs; repair those dangling endpoints before +deployment. Deprecated and revoked relationships are not eligible for bundle +graphs, so the migration leaves that inactive legacy history untouched. +Snapshot graph capture fails closed rather than silently producing an +incomplete deterministic baseline. ## What the migration writes -- Exact source and target revision metadata is added only to the latest - revision of each relationship in the underlying `relationships` collection. - `view.relationships.latest` may be used for discovery but is never written. +- Exact source and target revision metadata is added only to active latest + relationship revisions in the underlying `relationships` collection. + `view.relationships.latest.active` may be used for discovery but is never + written. - Each existing release-track snapshot receives a graph manifest containing its exact primary, relationship, secondary, supporting, and LinkById dependencies. diff --git a/docs/developer/TODO.md b/docs/developer/TODO.md index 1ad59401..591677be 100644 --- a/docs/developer/TODO.md +++ b/docs/developer/TODO.md @@ -258,6 +258,29 @@ revision on which that bundle depends. Bruno request definitions require no transport change. - [x] Run focused specs while iterating, then lint, OpenAPI validation, and the complete `npm test` suite. + +### Production-shaped migration repair + +- [x] Scope legacy endpoint pinning to active latest relationships, matching + the relationship set eligible for deterministic snapshot graphs. +- [x] Preserve fail-closed behavior for active dangling relationships while + allowing deprecated or revoked dangling history to remain untouched. +- [x] Surface actionable missing-endpoint diagnostics in the migration preview + and startup failure. +- [x] Establish manifest indexes independently of whether relationship pin + updates happen to be required. +- [x] Add production-shaped migration regressions and update the operator + documentation. + +Verification result (2026-07-30): + +- The focused migration spec passes all 4 cases. +- A read-only preview against the restored production database scans 24,818 + active relationship revisions without encountering the 118 dangling + endpoints confined to deprecated relationship history. +- Lint, formatting, diff checks, and the complete `npm test` suite pass; the + API suite passes all 972 cases. + - [x] Propose conventional commits split by independently reviewable architectural slice; do not commit until requested. diff --git a/migrations/20260730180000-backfill-deterministic-snapshot-graphs.js b/migrations/20260730180000-backfill-deterministic-snapshot-graphs.js index 2643e96d..ff5ba792 100644 --- a/migrations/20260730180000-backfill-deterministic-snapshot-graphs.js +++ b/migrations/20260730180000-backfill-deterministic-snapshot-graphs.js @@ -14,6 +14,11 @@ const TRACK_COLLECTION_PATTERN = /^release-track--[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i; const CONCURRENCY = 4; +const ACTIVE_RELATIONSHIP_FILTER = { + 'stix.x_mitre_deprecated': { $in: [null, false] }, + 'stix.revoked': { $in: [null, false] }, +}; +const ERROR_SAMPLE_LIMIT = 10; async function mapWithConcurrency(items, mapper) { let nextIndex = 0; @@ -29,11 +34,18 @@ async function mapWithConcurrency(items, mapper) { } async function latestRelationships(db) { + const latestActiveViewExists = await db + .listCollections({ name: 'view.relationships.latest.active' }, { nameOnly: true }) + .hasNext(); + if (latestActiveViewExists) { + return db.collection('view.relationships.latest.active').find({}).toArray(); + } + const latestViewExists = await db .listCollections({ name: 'view.relationships.latest' }, { nameOnly: true }) .hasNext(); if (latestViewExists) { - return db.collection('view.relationships.latest').find({}).toArray(); + return db.collection('view.relationships.latest').find(ACTIVE_RELATIONSHIP_FILTER).toArray(); } return db @@ -42,6 +54,7 @@ async function latestRelationships(db) { { $sort: { 'stix.id': 1, 'stix.modified': -1 } }, { $group: { _id: '$stix.id', document: { $first: '$$ROOT' } } }, { $replaceRoot: { newRoot: '$document' } }, + { $match: ACTIVE_RELATIONSHIP_FILTER }, ]) .toArray(); } @@ -109,6 +122,21 @@ async function buildRelationshipPinOperations(db) { return { relationships, operations, missing }; } +function missingEndpointError(missing) { + const sample = missing + .slice(0, ERROR_SAMPLE_LIMIT) + .map((entry) => `${entry.relationship_ref} -> ${entry.missing_endpoints.join(', ')}`) + .join('; '); + const remaining = missing.length - ERROR_SAMPLE_LIMIT; + const suffix = remaining > 0 ? `; and ${remaining} more` : ''; + const error = new Error( + `Cannot pin ${missing.length} active latest relationship(s) because referenced objects are ` + + `missing: ${sample}${suffix}`, + ); + error.missing_relationship_endpoints = missing; + return error; +} + async function findTrackIds(db) { const [registeredTracks, collections] = await Promise.all([ db.collection('releaseTrackRegistry').find({}).project({ track_id: 1, _id: 0 }).toArray(), @@ -206,17 +234,15 @@ async function backfillSnapshotManifests(db, options) { async function run(db, options = {}) { const relationshipPins = await buildRelationshipPinOperations(db); if (relationshipPins.missing.length > 0) { - const error = new Error( - 'Cannot pin latest relationship endpoints because one or more referenced objects are missing', - ); - error.missing_relationship_endpoints = relationshipPins.missing; - throw error; + throw missingEndpointError(relationshipPins.missing); } if (!options.dryRun && relationshipPins.operations.length > 0) { await db.collection('relationships').bulkWrite(relationshipPins.operations, { ordered: false, }); + } + if (!options.dryRun) { await ensureManifestIndexes(db); } const manifests = await backfillSnapshotManifests(db, options); @@ -233,7 +259,7 @@ module.exports = { async up(db) { const report = await run(db); console.log( - `Pinned ${report.relationship_pins_written} latest relationship revision(s) and ` + + `Pinned ${report.relationship_pins_written} active latest relationship revision(s) and ` + `created ${report.manifests_created} baseline snapshot manifest(s)`, ); }, diff --git a/scripts/previewDeterministicSnapshotGraphMigration.js b/scripts/previewDeterministicSnapshotGraphMigration.js index 348298e5..60ffb74b 100644 --- a/scripts/previewDeterministicSnapshotGraphMigration.js +++ b/scripts/previewDeterministicSnapshotGraphMigration.js @@ -15,6 +15,15 @@ async function main() { main() .catch((err) => { process.stderr.write(`${err.stack || err.message}\n`); + if (err.missing_relationship_endpoints) { + process.stderr.write( + `${JSON.stringify( + { missing_relationship_endpoints: err.missing_relationship_endpoints }, + null, + 2, + )}\n`, + ); + } process.exitCode = 1; }) .finally(async () => {