From 5bb5748d2775817bbf0a13439be15419d4ee46a6 Mon Sep 17 00:00:00 2001 From: John Hopper Date: Fri, 15 Nov 2024 10:10:36 -0800 Subject: [PATCH] fix: BED-5044 - revert edge constraint map checking in API and use SQL upsert --- packages/go/analysis/azure/post.go | 121 +++++++++---------- packages/go/analysis/post.go | 54 --------- packages/go/dawgs/drivers/pg/query/format.go | 3 +- packages/go/dawgs/drivers/pg/statements.go | 9 +- 4 files changed, 66 insertions(+), 121 deletions(-) diff --git a/packages/go/analysis/azure/post.go b/packages/go/analysis/azure/post.go index 6239b60551..41b7dd5867 100644 --- a/packages/go/analysis/azure/post.go +++ b/packages/go/analysis/azure/post.go @@ -205,38 +205,35 @@ func AppRoleAssignments(ctx context.Context, db graph.Database) (*analysis.Atomi if tenants, err := FetchTenants(ctx, db); err != nil { return &analysis.AtomicPostProcessingStats{}, err } else { - var ( - operation = analysis.NewPostRelationshipOperation(ctx, db, "Azure App Role Assignments Post Processing") - edgeConstraintMap = analysis.NewEdgeConstraintMap() - ) + operation := analysis.NewPostRelationshipOperation(ctx, db, "Azure App Role Assignments Post Processing") for _, tenant := range tenants { if err := db.ReadTransaction(ctx, func(tx graph.Transaction) error { if tenantContainsServicePrincipalRelationships, err := fetchTenantContainsRelationships(tx, tenant, azure.ServicePrincipal); err != nil { return err - } else if err := createAZMGApplicationReadWriteAllEdges(ctx, db, edgeConstraintMap, operation, tenant, tenantContainsServicePrincipalRelationships); err != nil { + } else if err := createAZMGApplicationReadWriteAllEdges(ctx, db, operation, tenant, tenantContainsServicePrincipalRelationships); err != nil { return err - } else if err := createAZMGAppRoleAssignmentReadWriteAllEdges(ctx, db, edgeConstraintMap, operation, tenantContainsServicePrincipalRelationships); err != nil { + } else if err := createAZMGAppRoleAssignmentReadWriteAllEdges(ctx, db, operation, tenantContainsServicePrincipalRelationships); err != nil { return err - } else if err := createAZMGDirectoryReadWriteAllEdges(ctx, db, edgeConstraintMap, operation, tenant, tenantContainsServicePrincipalRelationships); err != nil { + } else if err := createAZMGDirectoryReadWriteAllEdges(ctx, db, operation, tenant, tenantContainsServicePrincipalRelationships); err != nil { return err - } else if err := createAZMGGroupReadWriteAllEdges(ctx, db, edgeConstraintMap, operation, tenant, tenantContainsServicePrincipalRelationships); err != nil { + } else if err := createAZMGGroupReadWriteAllEdges(ctx, db, operation, tenant, tenantContainsServicePrincipalRelationships); err != nil { return err - } else if err := createAZMGGroupMemberReadWriteAllEdges(ctx, db, edgeConstraintMap, operation, tenant, tenantContainsServicePrincipalRelationships); err != nil { + } else if err := createAZMGGroupMemberReadWriteAllEdges(ctx, db, operation, tenant, tenantContainsServicePrincipalRelationships); err != nil { return err - } else if err := createAZMGRoleManagementReadWriteDirectoryEdgesPart1(ctx, db, edgeConstraintMap, operation, tenant, tenantContainsServicePrincipalRelationships); err != nil { + } else if err := createAZMGRoleManagementReadWriteDirectoryEdgesPart1(ctx, db, operation, tenant, tenantContainsServicePrincipalRelationships); err != nil { return err - } else if err := createAZMGRoleManagementReadWriteDirectoryEdgesPart2(ctx, db, edgeConstraintMap, operation, tenant, tenantContainsServicePrincipalRelationships); err != nil { + } else if err := createAZMGRoleManagementReadWriteDirectoryEdgesPart2(ctx, db, operation, tenant, tenantContainsServicePrincipalRelationships); err != nil { return err - } else if err := createAZMGRoleManagementReadWriteDirectoryEdgesPart3(ctx, db, edgeConstraintMap, operation, tenantContainsServicePrincipalRelationships); err != nil { + } else if err := createAZMGRoleManagementReadWriteDirectoryEdgesPart3(ctx, db, operation, tenantContainsServicePrincipalRelationships); err != nil { return err - } else if err := createAZMGRoleManagementReadWriteDirectoryEdgesPart4(ctx, db, edgeConstraintMap, operation, tenant, tenantContainsServicePrincipalRelationships); err != nil { + } else if err := createAZMGRoleManagementReadWriteDirectoryEdgesPart4(ctx, db, operation, tenant, tenantContainsServicePrincipalRelationships); err != nil { return err - } else if err := createAZMGRoleManagementReadWriteDirectoryEdgesPart5(ctx, db, edgeConstraintMap, operation, tenant, tenantContainsServicePrincipalRelationships); err != nil { + } else if err := createAZMGRoleManagementReadWriteDirectoryEdgesPart5(ctx, db, operation, tenant, tenantContainsServicePrincipalRelationships); err != nil { return err - } else if err := createAZMGServicePrincipalEndpointReadWriteAllEdges(ctx, db, edgeConstraintMap, operation, tenantContainsServicePrincipalRelationships); err != nil { + } else if err := createAZMGServicePrincipalEndpointReadWriteAllEdges(ctx, db, operation, tenantContainsServicePrincipalRelationships); err != nil { return err - } else if err := addSecret(edgeConstraintMap, operation, tenant); err != nil { + } else if err := addSecret(operation, tenant); err != nil { return err } @@ -249,11 +246,12 @@ func AppRoleAssignments(ctx context.Context, db graph.Database) (*analysis.Atomi return &operation.Stats, err } } + return &operation.Stats, operation.Done() } } -func createAZMGApplicationReadWriteAllEdges(ctx context.Context, db graph.Database, edgeConstraintMap analysis.EdgeConstraintMap, operation analysis.StatTrackedOperation[analysis.CreatePostRelationshipJob], tenant *graph.Node, tenantContainsServicePrincipalRelationships []*graph.Relationship) error { +func createAZMGApplicationReadWriteAllEdges(ctx context.Context, db graph.Database, operation analysis.StatTrackedOperation[analysis.CreatePostRelationshipJob], tenant *graph.Node, tenantContainsServicePrincipalRelationships []*graph.Relationship) error { if err := db.ReadTransaction(ctx, func(tx graph.Transaction) error { if tenantContainsAppRelationships, err := fetchTenantContainsRelationships(tx, tenant, azure.App); err != nil { return err @@ -269,7 +267,7 @@ func createAZMGApplicationReadWriteAllEdges(ctx context.Context, db graph.Databa Kind: azure.AZMGAddSecret, } - if edgeConstraintMap.TrackCreatePostRelationshipJob(AZMGAddSecretRelationship) && !channels.Submit(ctx, outC, AZMGAddSecretRelationship) { + if !channels.Submit(ctx, outC, AZMGAddSecretRelationship) { return nil } @@ -279,7 +277,7 @@ func createAZMGApplicationReadWriteAllEdges(ctx context.Context, db graph.Databa Kind: azure.AZMGAddOwner, } - if edgeConstraintMap.TrackCreatePostRelationshipJob(AZMGAddOwnerRelationship) && !channels.Submit(ctx, outC, AZMGAddOwnerRelationship) { + if !channels.Submit(ctx, outC, AZMGAddOwnerRelationship) { return nil } } @@ -295,7 +293,7 @@ func createAZMGApplicationReadWriteAllEdges(ctx context.Context, db graph.Databa } } -func createAZMGAppRoleAssignmentReadWriteAllEdges(ctx context.Context, db graph.Database, edgeConstraintMap analysis.EdgeConstraintMap, operation analysis.StatTrackedOperation[analysis.CreatePostRelationshipJob], tenantContainsServicePrincipalRelationships []*graph.Relationship) error { +func createAZMGAppRoleAssignmentReadWriteAllEdges(ctx context.Context, db graph.Database, operation analysis.StatTrackedOperation[analysis.CreatePostRelationshipJob], tenantContainsServicePrincipalRelationships []*graph.Relationship) error { if err := db.ReadTransaction(ctx, func(tx graph.Transaction) error { if sourceNodes, err := aggregateSourceReadWriteServicePrincipals(tx, tenantContainsServicePrincipalRelationships, azure.AppRoleAssignmentReadWriteAll); err != nil { return err @@ -309,7 +307,7 @@ func createAZMGAppRoleAssignmentReadWriteAllEdges(ctx context.Context, db graph. Kind: azure.AZMGGrantAppRoles, } - if edgeConstraintMap.TrackCreatePostRelationshipJob(AZMGGrantAppRolesRelationship) && !channels.Submit(ctx, outC, AZMGGrantAppRolesRelationship) { + if !channels.Submit(ctx, outC, AZMGGrantAppRolesRelationship) { return nil } } @@ -325,7 +323,7 @@ func createAZMGAppRoleAssignmentReadWriteAllEdges(ctx context.Context, db graph. } } -func createAZMGDirectoryReadWriteAllEdges(ctx context.Context, db graph.Database, edgeConstraintMap analysis.EdgeConstraintMap, operation analysis.StatTrackedOperation[analysis.CreatePostRelationshipJob], tenant *graph.Node, tenantContainsServicePrincipalRelationships []*graph.Relationship) error { +func createAZMGDirectoryReadWriteAllEdges(ctx context.Context, db graph.Database, operation analysis.StatTrackedOperation[analysis.CreatePostRelationshipJob], tenant *graph.Node, tenantContainsServicePrincipalRelationships []*graph.Relationship) error { if err := db.ReadTransaction(ctx, func(tx graph.Transaction) error { if sourceNodes, err := aggregateSourceReadWriteServicePrincipals(tx, tenantContainsServicePrincipalRelationships, azure.DirectoryReadWriteAll); err != nil { return err @@ -341,7 +339,7 @@ func createAZMGDirectoryReadWriteAllEdges(ctx context.Context, db graph.Database Kind: azure.AZMGAddMember, } - if edgeConstraintMap.TrackCreatePostRelationshipJob(AZMGAddMemberRelationship) && !channels.Submit(ctx, outC, AZMGAddMemberRelationship) { + if !channels.Submit(ctx, outC, AZMGAddMemberRelationship) { return nil } @@ -351,7 +349,7 @@ func createAZMGDirectoryReadWriteAllEdges(ctx context.Context, db graph.Database Kind: azure.AZMGAddOwner, } - if edgeConstraintMap.TrackCreatePostRelationshipJob(AZMGAddOwnerRelationship) && !channels.Submit(ctx, outC, AZMGAddOwnerRelationship) { + if !channels.Submit(ctx, outC, AZMGAddOwnerRelationship) { return nil } } @@ -366,7 +364,7 @@ func createAZMGDirectoryReadWriteAllEdges(ctx context.Context, db graph.Database } } -func createAZMGGroupReadWriteAllEdges(ctx context.Context, db graph.Database, edgeConstraintMap analysis.EdgeConstraintMap, operation analysis.StatTrackedOperation[analysis.CreatePostRelationshipJob], tenant *graph.Node, tenantContainsServicePrincipalRelationships []*graph.Relationship) error { +func createAZMGGroupReadWriteAllEdges(ctx context.Context, db graph.Database, operation analysis.StatTrackedOperation[analysis.CreatePostRelationshipJob], tenant *graph.Node, tenantContainsServicePrincipalRelationships []*graph.Relationship) error { if err := db.ReadTransaction(ctx, func(tx graph.Transaction) error { if sourceNodes, err := aggregateSourceReadWriteServicePrincipals(tx, tenantContainsServicePrincipalRelationships, azure.GroupReadWriteAll); err != nil { return err @@ -382,7 +380,7 @@ func createAZMGGroupReadWriteAllEdges(ctx context.Context, db graph.Database, ed Kind: azure.AZMGAddMember, } - if edgeConstraintMap.TrackCreatePostRelationshipJob(AZMGAddMemberRelationship) && !channels.Submit(ctx, outC, AZMGAddMemberRelationship) { + if !channels.Submit(ctx, outC, AZMGAddMemberRelationship) { return nil } @@ -392,7 +390,7 @@ func createAZMGGroupReadWriteAllEdges(ctx context.Context, db graph.Database, ed Kind: azure.AZMGAddOwner, } - if edgeConstraintMap.TrackCreatePostRelationshipJob(AZMGAddOwnerRelationship) && !channels.Submit(ctx, outC, AZMGAddOwnerRelationship) { + if !channels.Submit(ctx, outC, AZMGAddOwnerRelationship) { return nil } } @@ -407,7 +405,7 @@ func createAZMGGroupReadWriteAllEdges(ctx context.Context, db graph.Database, ed } } -func createAZMGGroupMemberReadWriteAllEdges(ctx context.Context, db graph.Database, edgeConstraintMap analysis.EdgeConstraintMap, operation analysis.StatTrackedOperation[analysis.CreatePostRelationshipJob], tenant *graph.Node, tenantContainsServicePrincipalRelationships []*graph.Relationship) error { +func createAZMGGroupMemberReadWriteAllEdges(ctx context.Context, db graph.Database, operation analysis.StatTrackedOperation[analysis.CreatePostRelationshipJob], tenant *graph.Node, tenantContainsServicePrincipalRelationships []*graph.Relationship) error { if err := db.ReadTransaction(ctx, func(tx graph.Transaction) error { if sourceNodes, err := aggregateSourceReadWriteServicePrincipals(tx, tenantContainsServicePrincipalRelationships, azure.GroupMemberReadWriteAll); err != nil { return err @@ -423,7 +421,7 @@ func createAZMGGroupMemberReadWriteAllEdges(ctx context.Context, db graph.Databa Kind: azure.AZMGAddMember, } - if edgeConstraintMap.TrackCreatePostRelationshipJob(AZMGAddMemberRelationship) && !channels.Submit(ctx, outC, AZMGAddMemberRelationship) { + if !channels.Submit(ctx, outC, AZMGAddMemberRelationship) { return nil } } @@ -438,7 +436,7 @@ func createAZMGGroupMemberReadWriteAllEdges(ctx context.Context, db graph.Databa } } -func createAZMGRoleManagementReadWriteDirectoryEdgesPart1(ctx context.Context, db graph.Database, edgeConstraintMap analysis.EdgeConstraintMap, operation analysis.StatTrackedOperation[analysis.CreatePostRelationshipJob], tenant *graph.Node, tenantContainsServicePrincipalRelationships []*graph.Relationship) error { +func createAZMGRoleManagementReadWriteDirectoryEdgesPart1(ctx context.Context, db graph.Database, operation analysis.StatTrackedOperation[analysis.CreatePostRelationshipJob], tenant *graph.Node, tenantContainsServicePrincipalRelationships []*graph.Relationship) error { if err := db.ReadTransaction(ctx, func(tx graph.Transaction) error { if sourceNodes, err := aggregateSourceReadWriteServicePrincipals(tx, tenantContainsServicePrincipalRelationships, azure.RoleManagementReadWriteDirectory); err != nil { return err @@ -454,7 +452,7 @@ func createAZMGRoleManagementReadWriteDirectoryEdgesPart1(ctx context.Context, d Kind: azure.AZMGGrantAppRoles, } - if edgeConstraintMap.TrackCreatePostRelationshipJob(AZMGGrantAppRolesRelationship) && !channels.Submit(ctx, outC, AZMGGrantAppRolesRelationship) { + if !channels.Submit(ctx, outC, AZMGGrantAppRolesRelationship) { return nil } } @@ -469,7 +467,7 @@ func createAZMGRoleManagementReadWriteDirectoryEdgesPart1(ctx context.Context, d } } -func createAZMGRoleManagementReadWriteDirectoryEdgesPart2(ctx context.Context, db graph.Database, edgeConstraintMap analysis.EdgeConstraintMap, operation analysis.StatTrackedOperation[analysis.CreatePostRelationshipJob], tenant *graph.Node, tenantContainsServicePrincipalRelationships []*graph.Relationship) error { +func createAZMGRoleManagementReadWriteDirectoryEdgesPart2(ctx context.Context, db graph.Database, operation analysis.StatTrackedOperation[analysis.CreatePostRelationshipJob], tenant *graph.Node, tenantContainsServicePrincipalRelationships []*graph.Relationship) error { if err := db.ReadTransaction(ctx, func(tx graph.Transaction) error { if sourceNodes, err := aggregateSourceReadWriteServicePrincipals(tx, tenantContainsServicePrincipalRelationships, azure.RoleManagementReadWriteDirectory); err != nil { return err @@ -485,7 +483,7 @@ func createAZMGRoleManagementReadWriteDirectoryEdgesPart2(ctx context.Context, d Kind: azure.AZMGGrantRole, } - if edgeConstraintMap.TrackCreatePostRelationshipJob(AZMGGrantRoleRelationship) && !channels.Submit(ctx, outC, AZMGGrantRoleRelationship) { + if !channels.Submit(ctx, outC, AZMGGrantRoleRelationship) { return nil } } @@ -500,7 +498,7 @@ func createAZMGRoleManagementReadWriteDirectoryEdgesPart2(ctx context.Context, d } } -func createAZMGRoleManagementReadWriteDirectoryEdgesPart3(ctx context.Context, db graph.Database, edgeConstraintMap analysis.EdgeConstraintMap, operation analysis.StatTrackedOperation[analysis.CreatePostRelationshipJob], tenantContainsServicePrincipalRelationships []*graph.Relationship) error { +func createAZMGRoleManagementReadWriteDirectoryEdgesPart3(ctx context.Context, db graph.Database, operation analysis.StatTrackedOperation[analysis.CreatePostRelationshipJob], tenantContainsServicePrincipalRelationships []*graph.Relationship) error { if err := db.ReadTransaction(ctx, func(tx graph.Transaction) error { if sourceNodes, err := aggregateSourceReadWriteServicePrincipals(tx, tenantContainsServicePrincipalRelationships, azure.RoleManagementReadWriteDirectory); err != nil { return err @@ -514,7 +512,7 @@ func createAZMGRoleManagementReadWriteDirectoryEdgesPart3(ctx context.Context, d Kind: azure.AZMGAddSecret, } - if edgeConstraintMap.TrackCreatePostRelationshipJob(AZMGAddSecretRelationship) && !channels.Submit(ctx, outC, AZMGAddSecretRelationship) { + if !channels.Submit(ctx, outC, AZMGAddSecretRelationship) { return nil } @@ -524,7 +522,7 @@ func createAZMGRoleManagementReadWriteDirectoryEdgesPart3(ctx context.Context, d Kind: azure.AZMGAddOwner, } - if edgeConstraintMap.TrackCreatePostRelationshipJob(AZMGAddOwnerRelationship) && !channels.Submit(ctx, outC, AZMGAddOwnerRelationship) { + if !channels.Submit(ctx, outC, AZMGAddOwnerRelationship) { return nil } } @@ -540,7 +538,7 @@ func createAZMGRoleManagementReadWriteDirectoryEdgesPart3(ctx context.Context, d } } -func createAZMGRoleManagementReadWriteDirectoryEdgesPart4(ctx context.Context, db graph.Database, edgeConstraintMap analysis.EdgeConstraintMap, operation analysis.StatTrackedOperation[analysis.CreatePostRelationshipJob], tenant *graph.Node, tenantContainsServicePrincipalRelationships []*graph.Relationship) error { +func createAZMGRoleManagementReadWriteDirectoryEdgesPart4(ctx context.Context, db graph.Database, operation analysis.StatTrackedOperation[analysis.CreatePostRelationshipJob], tenant *graph.Node, tenantContainsServicePrincipalRelationships []*graph.Relationship) error { if err := db.ReadTransaction(ctx, func(tx graph.Transaction) error { if sourceNodes, err := aggregateSourceReadWriteServicePrincipals(tx, tenantContainsServicePrincipalRelationships, azure.RoleManagementReadWriteDirectory); err != nil { return err @@ -556,7 +554,7 @@ func createAZMGRoleManagementReadWriteDirectoryEdgesPart4(ctx context.Context, d Kind: azure.AZMGAddSecret, } - if edgeConstraintMap.TrackCreatePostRelationshipJob(AZMGAddSecretRelationship) && !channels.Submit(ctx, outC, AZMGAddSecretRelationship) { + if !channels.Submit(ctx, outC, AZMGAddSecretRelationship) { return nil } @@ -566,7 +564,7 @@ func createAZMGRoleManagementReadWriteDirectoryEdgesPart4(ctx context.Context, d Kind: azure.AZMGAddOwner, } - if edgeConstraintMap.TrackCreatePostRelationshipJob(AZMGAddOwnerRelationship) && !channels.Submit(ctx, outC, AZMGAddOwnerRelationship) { + if !channels.Submit(ctx, outC, AZMGAddOwnerRelationship) { return nil } } @@ -582,7 +580,7 @@ func createAZMGRoleManagementReadWriteDirectoryEdgesPart4(ctx context.Context, d } } -func createAZMGRoleManagementReadWriteDirectoryEdgesPart5(ctx context.Context, db graph.Database, edgeConstraintMap analysis.EdgeConstraintMap, operation analysis.StatTrackedOperation[analysis.CreatePostRelationshipJob], tenant *graph.Node, tenantContainsServicePrincipalRelationships []*graph.Relationship) error { +func createAZMGRoleManagementReadWriteDirectoryEdgesPart5(ctx context.Context, db graph.Database, operation analysis.StatTrackedOperation[analysis.CreatePostRelationshipJob], tenant *graph.Node, tenantContainsServicePrincipalRelationships []*graph.Relationship) error { if err := db.ReadTransaction(ctx, func(tx graph.Transaction) error { if sourceNodes, err := aggregateSourceReadWriteServicePrincipals(tx, tenantContainsServicePrincipalRelationships, azure.RoleManagementReadWriteDirectory); err != nil { return err @@ -598,7 +596,7 @@ func createAZMGRoleManagementReadWriteDirectoryEdgesPart5(ctx context.Context, d Kind: azure.AZMGAddMember, } - if edgeConstraintMap.TrackCreatePostRelationshipJob(AZMGAddMemberRelationship) && !channels.Submit(ctx, outC, AZMGAddMemberRelationship) { + if !channels.Submit(ctx, outC, AZMGAddMemberRelationship) { return nil } @@ -608,7 +606,7 @@ func createAZMGRoleManagementReadWriteDirectoryEdgesPart5(ctx context.Context, d Kind: azure.AZMGAddOwner, } - if edgeConstraintMap.TrackCreatePostRelationshipJob(AZMGAddOwnerRelationship) && !channels.Submit(ctx, outC, AZMGAddOwnerRelationship) { + if !channels.Submit(ctx, outC, AZMGAddOwnerRelationship) { return nil } } @@ -623,7 +621,7 @@ func createAZMGRoleManagementReadWriteDirectoryEdgesPart5(ctx context.Context, d } } -func createAZMGServicePrincipalEndpointReadWriteAllEdges(ctx context.Context, db graph.Database, edgeConstraintMap analysis.EdgeConstraintMap, operation analysis.StatTrackedOperation[analysis.CreatePostRelationshipJob], tenantContainsServicePrincipalRelationships []*graph.Relationship) error { +func createAZMGServicePrincipalEndpointReadWriteAllEdges(ctx context.Context, db graph.Database, operation analysis.StatTrackedOperation[analysis.CreatePostRelationshipJob], tenantContainsServicePrincipalRelationships []*graph.Relationship) error { if err := db.ReadTransaction(ctx, func(tx graph.Transaction) error { if sourceNodes, err := aggregateSourceReadWriteServicePrincipals(tx, tenantContainsServicePrincipalRelationships, azure.ServicePrincipalEndpointReadWriteAll); err != nil { return err @@ -637,7 +635,7 @@ func createAZMGServicePrincipalEndpointReadWriteAllEdges(ctx context.Context, db Kind: azure.AZMGAddOwner, } - if edgeConstraintMap.TrackCreatePostRelationshipJob(AZMGAddOwnerRelationship) && !channels.Submit(ctx, outC, AZMGAddOwnerRelationship) { + if !channels.Submit(ctx, outC, AZMGAddOwnerRelationship) { return nil } } @@ -653,7 +651,7 @@ func createAZMGServicePrincipalEndpointReadWriteAllEdges(ctx context.Context, db } } -func addSecret(edgeConstraintMap analysis.EdgeConstraintMap, operation analysis.StatTrackedOperation[analysis.CreatePostRelationshipJob], tenant *graph.Node) error { +func addSecret(operation analysis.StatTrackedOperation[analysis.CreatePostRelationshipJob], tenant *graph.Node) error { return operation.Operation.SubmitReader(func(ctx context.Context, tx graph.Transaction, outC chan<- analysis.CreatePostRelationshipJob) error { if addSecretRoles, err := TenantRoles(tx, tenant, AddSecretRoleIDs()...); err != nil { return err @@ -669,7 +667,7 @@ func addSecret(edgeConstraintMap analysis.EdgeConstraintMap, operation analysis. Kind: azure.AddSecret, } - if edgeConstraintMap.TrackCreatePostRelationshipJob(nextJob) && !channels.Submit(ctx, outC, nextJob) { + if !channels.Submit(ctx, outC, nextJob) { return nil } } @@ -796,7 +794,7 @@ func resetPasswordEndNodeBitmapForRole(role *graph.Node, roleAssignments RoleAss } } -func globalAdmins(roleAssignments RoleAssignments, edgeConstraintMap analysis.EdgeConstraintMap, tenant *graph.Node, operation analysis.StatTrackedOperation[analysis.CreatePostRelationshipJob]) { +func globalAdmins(roleAssignments RoleAssignments, tenant *graph.Node, operation analysis.StatTrackedOperation[analysis.CreatePostRelationshipJob]) { if err := operation.Operation.SubmitReader(func(ctx context.Context, tx graph.Transaction, outC chan<- analysis.CreatePostRelationshipJob) error { roleAssignments.PrincipalsWithRole(azure.CompanyAdministratorRole).Each(func(nextID uint64) bool { nextJob := analysis.CreatePostRelationshipJob{ @@ -805,7 +803,7 @@ func globalAdmins(roleAssignments RoleAssignments, edgeConstraintMap analysis.Ed Kind: azure.GlobalAdmin, } - if edgeConstraintMap.TrackCreatePostRelationshipJob(nextJob) && !channels.Submit(ctx, outC, nextJob) { + if !channels.Submit(ctx, outC, nextJob) { return false } @@ -818,7 +816,7 @@ func globalAdmins(roleAssignments RoleAssignments, edgeConstraintMap analysis.Ed } } -func privilegedRoleAdmins(roleAssignments RoleAssignments, edgeConstraintMap analysis.EdgeConstraintMap, tenant *graph.Node, operation analysis.StatTrackedOperation[analysis.CreatePostRelationshipJob]) { +func privilegedRoleAdmins(roleAssignments RoleAssignments, tenant *graph.Node, operation analysis.StatTrackedOperation[analysis.CreatePostRelationshipJob]) { if err := operation.Operation.SubmitReader(func(ctx context.Context, tx graph.Transaction, outC chan<- analysis.CreatePostRelationshipJob) error { roleAssignments.PrincipalsWithRole(azure.PrivilegedRoleAdministratorRole).Each(func(nextID uint64) bool { nextJob := analysis.CreatePostRelationshipJob{ @@ -827,7 +825,7 @@ func privilegedRoleAdmins(roleAssignments RoleAssignments, edgeConstraintMap ana Kind: azure.PrivilegedRoleAdmin, } - if edgeConstraintMap.TrackCreatePostRelationshipJob(nextJob) && !channels.Submit(ctx, outC, nextJob) { + if !channels.Submit(ctx, outC, nextJob) { return false } @@ -840,7 +838,7 @@ func privilegedRoleAdmins(roleAssignments RoleAssignments, edgeConstraintMap ana } } -func privilegedAuthAdmins(roleAssignments RoleAssignments, edgeConstraintMap analysis.EdgeConstraintMap, tenant *graph.Node, operation analysis.StatTrackedOperation[analysis.CreatePostRelationshipJob]) { +func privilegedAuthAdmins(roleAssignments RoleAssignments, tenant *graph.Node, operation analysis.StatTrackedOperation[analysis.CreatePostRelationshipJob]) { if err := operation.Operation.SubmitReader(func(ctx context.Context, tx graph.Transaction, outC chan<- analysis.CreatePostRelationshipJob) error { roleAssignments.PrincipalsWithRole(azure.PrivilegedAuthenticationAdministratorRole).Each(func(nextID uint64) bool { nextJob := analysis.CreatePostRelationshipJob{ @@ -849,7 +847,7 @@ func privilegedAuthAdmins(roleAssignments RoleAssignments, edgeConstraintMap ana Kind: azure.PrivilegedAuthAdmin, } - if edgeConstraintMap.TrackCreatePostRelationshipJob(nextJob) && !channels.Submit(ctx, outC, nextJob) { + if !channels.Submit(ctx, outC, nextJob) { return false } @@ -862,7 +860,7 @@ func privilegedAuthAdmins(roleAssignments RoleAssignments, edgeConstraintMap ana } } -func addMembers(roleAssignments RoleAssignments, edgeConstraintMap analysis.EdgeConstraintMap, operation analysis.StatTrackedOperation[analysis.CreatePostRelationshipJob]) { +func addMembers(roleAssignments RoleAssignments, operation analysis.StatTrackedOperation[analysis.CreatePostRelationshipJob]) { for tenantGroupID, tenantGroup := range roleAssignments.Principals.Get(azure.Group) { var ( innerGroupID = tenantGroupID @@ -877,7 +875,7 @@ func addMembers(roleAssignments RoleAssignments, edgeConstraintMap analysis.Edge Kind: azure.AddMembers, } - if edgeConstraintMap.TrackCreatePostRelationshipJob(nextJob) && !channels.Submit(ctx, outC, nextJob) { + if !channels.Submit(ctx, outC, nextJob) { return false } @@ -904,7 +902,7 @@ func addMembers(roleAssignments RoleAssignments, edgeConstraintMap analysis.Edge Kind: azure.AddMembers, } - if edgeConstraintMap.TrackCreatePostRelationshipJob(nextJob) && !channels.Submit(ctx, outC, nextJob) { + if !channels.Submit(ctx, outC, nextJob) { return false } @@ -923,10 +921,7 @@ func UserRoleAssignments(ctx context.Context, db graph.Database) (*analysis.Atom if tenantNodes, err := FetchTenants(ctx, db); err != nil { return &analysis.AtomicPostProcessingStats{}, err } else { - var ( - operation = analysis.NewPostRelationshipOperation(ctx, db, "Azure User Role Assignments Post Processing") - edgeConstraintMap = analysis.NewEdgeConstraintMap() - ) + operation := analysis.NewPostRelationshipOperation(ctx, db, "Azure User Role Assignments Post Processing") for _, tenant := range tenantNodes { if roleAssignments, err := TenantRoleAssignments(ctx, db, tenant); err != nil { @@ -943,10 +938,10 @@ func UserRoleAssignments(ctx context.Context, db graph.Database) (*analysis.Atom return &analysis.AtomicPostProcessingStats{}, err } else { - globalAdmins(roleAssignments, edgeConstraintMap, tenant, operation) - privilegedRoleAdmins(roleAssignments, edgeConstraintMap, tenant, operation) - privilegedAuthAdmins(roleAssignments, edgeConstraintMap, tenant, operation) - addMembers(roleAssignments, edgeConstraintMap, operation) + globalAdmins(roleAssignments, tenant, operation) + privilegedRoleAdmins(roleAssignments, tenant, operation) + privilegedAuthAdmins(roleAssignments, tenant, operation) + addMembers(roleAssignments, operation) } } } diff --git a/packages/go/analysis/post.go b/packages/go/analysis/post.go index cba34105ee..16f0a8de79 100644 --- a/packages/go/analysis/post.go +++ b/packages/go/analysis/post.go @@ -19,9 +19,6 @@ package analysis import ( "context" "sort" - "sync" - - "github.com/specterops/bloodhound/dawgs/cardinality" "github.com/specterops/bloodhound/dawgs/graph" "github.com/specterops/bloodhound/dawgs/ops" @@ -123,57 +120,6 @@ type DeleteRelationshipJob struct { ID graph.ID } -// EdgeConstraintMap is a thread safe tracker for post-processed edges. It guarantees that only one edge of a given -// post-processed kind may exist. This is useful either to create a batch of post-processed edges to insert or to -// guard against double insertion of the same post-processed edge. -type EdgeConstraintMap struct { - lock *sync.Mutex - adjacent map[graph.ID]map[graph.Kind]cardinality.Duplex[uint64] -} - -func NewEdgeConstraintMap() EdgeConstraintMap { - return EdgeConstraintMap{ - lock: &sync.Mutex{}, - adjacent: map[graph.ID]map[graph.Kind]cardinality.Duplex[uint64]{}, - } -} - -// TrackCreatePostRelationshipJob decomposes a CreatePostRelationshipJob type and returns the result of tracking it. -func (s EdgeConstraintMap) TrackCreatePostRelationshipJob(job CreatePostRelationshipJob) bool { - return s.Track(job.FromID, job.ToID, job.Kind) -} - -// Track will attempt to track creation of the given relationship arguments. This function returns false -// if the given relationship has already been tracked; true otherwise. -func (s EdgeConstraintMap) Track(start, end graph.ID, edgeKind graph.Kind) bool { - s.lock.Lock() - defer s.lock.Unlock() - - // Lookup what's adjacent outbound from the start ID - if startAdjacent, exists := s.adjacent[start]; !exists { - // If there's nothing adjacent for the start ID then this is the first outbound edge being created - // for it. - s.adjacent[start] = map[graph.Kind]cardinality.Duplex[uint64]{ - edgeKind: cardinality.NewBitmap64With(end.Uint64()), - } - } else if kindAdjacent, exists := startAdjacent[edgeKind]; !exists { - // If there's no bitmap representing outbound edges over the given edge kind then create a new bitmap - // and track it. - startAdjacent[edgeKind] = cardinality.NewBitmap64With(end.Uint64()) - } else if !kindAdjacent.CheckedAdd(end.Uint64()) { - // This Debugf statement is here to help engineers figure out where double-inserts are coming from in - // the post-processing logic. - log.Debugf("Duplicate post-processed edge: (%d)-[:%s]->(%d)", start, edgeKind, end) - - // If the CheckedAdd function returns false, the outbound nodes already contains an entry for this end - // ID, meaning that the edge has already been created. - return false - } - - // Getting here means we have added a new, unique edge. - return true -} - func DeleteTransitEdges(ctx context.Context, db graph.Database, baseKinds graph.Kinds, targetRelationships ...graph.Kind) (*AtomicPostProcessingStats, error) { defer log.Measure(log.LevelInfo, "Finished deleting transit edges")() diff --git a/packages/go/dawgs/drivers/pg/query/format.go b/packages/go/dawgs/drivers/pg/query/format.go index ebfc498298..ce7fcbe3bd 100644 --- a/packages/go/dawgs/drivers/pg/query/format.go +++ b/packages/go/dawgs/drivers/pg/query/format.go @@ -134,8 +134,7 @@ func FormatRelationshipPartitionUpsert(graphTarget model.Graph, identityProperti "(graph_id, start_id, end_id, kind_id, properties) ", "select $1::int4, unnest($2::int4[]), unnest($3::int4[]), unnest($4::int2[]), unnest($5::jsonb[]) ", formatConflictMatcher(identityProperties, "graph_id, start_id, end_id, kind_id"), - "do update set properties = e.properties || excluded.properties ", - "returning id;", + "do update set properties = e.properties || excluded.properties;", ) } diff --git a/packages/go/dawgs/drivers/pg/statements.go b/packages/go/dawgs/drivers/pg/statements.go index 6b2c5210d3..b7d3777f33 100644 --- a/packages/go/dawgs/drivers/pg/statements.go +++ b/packages/go/dawgs/drivers/pg/statements.go @@ -22,8 +22,13 @@ const ( createNodeWithIDBatchStatement = `insert into node (graph_id, id, kind_ids, properties) select $1, unnest($2::int4[]), unnest($3::text[])::int2[], unnest($4::jsonb[])` deleteNodeWithIDStatement = `delete from node where node.id = any($1)` - createEdgeStatement = `insert into edge (graph_id, start_id, end_id, kind_id, properties) values (@graph_id, @start_id, @end_id, @kind_id, @properties) returning (id, start_id, end_id, kind_id, properties)::edgeComposite;` - createEdgeBatchStatement = `insert into edge (graph_id, start_id, end_id, kind_id, properties) select $1::int4, unnest($2::int4[]), unnest($3::int4[]), unnest($4::int2[]), unnest($5::jsonb[]);` + createEdgeStatement = `insert into edge (graph_id, start_id, end_id, kind_id, properties) values (@graph_id, @start_id, @end_id, @kind_id, @properties) returning (id, start_id, end_id, kind_id, properties)::edgeComposite;` + + // TODO: The query below is not a pure creation statement as it contains an `on conflict` clause to dance around + // Azure post-processing. This was done because Azure post will submit the same creation request hundreds of + // times for the same edge. In PostgreSQL this results in a constraint violation. For now this is best-effort + // until Azure post-processing can be refactored. + createEdgeBatchStatement = `insert into edge as e (graph_id, start_id, end_id, kind_id, properties) select $1::int4, unnest($2::int4[]), unnest($3::int4[]), unnest($4::int2[]), unnest($5::jsonb[]) on conflict (graph_id, start_id, end_id, kind_id) do update set properties = e.properties || excluded.properties;` deleteEdgeWithIDStatement = `delete from edge as e where e.id = any($1)` edgePropertySetOnlyStatement = `update edge set properties = properties || $1::jsonb where edge.id = $2`