Better duplicate SID handling in multi forest/domain analysis, option…

… to not import CNF (conflict) objects from AD (default = don't import)

modules/engine/attributes.go

@@ -14,16 +14,23 @@ type AttributeGetFunc func(o *Object, a Attribute) (v AttributeValues, found boo
 type AttributeSetFunc func(o *Object, a Attribute, v AttributeValues) error
 
 type attributeinfo struct {
-	name      string
-	tags      []string
-	multi     bool // If true, this attribute can have multiple values
-	nonunique bool // Doing a Find on this attribute will return multiple results
-	merge     bool // If true, objects can be merged on this attribute
-	onset     AttributeSetFunc
-	onget     AttributeGetFunc
+	Name                         string
+	Tags                         []string
+	Multi                        bool // If true, this attribute can have multiple values
+	NonUnique                    bool // Doing a Find on this attribute will return multiple results
+	Merge                        bool // If true, objects can be merged on this attribute
+	DefaultF, DefaultM, DefaultL bool
+	onset                        AttributeSetFunc
+	onget                        AttributeGetFunc
 }
 
-var mergeapprovers []mergefunc
+type mergeapproverinfo struct {
+	mergefunc mergefunc
+	// priority  int
+	name string
+}
+
+var mergeapprovers []mergeapproverinfo
 var attributenums []attributeinfo
 
 var (
@@ -41,7 +48,7 @@ var (
 	ObjectSid             = NewAttribute("objectSid")
 	ObjectGUID            = NewAttribute("objectGUID")
 	NTSecurityDescriptor  = NewAttribute("nTSecurityDescriptor")
-	SchemaIDGUID          = NewAttribute("schemaIDGUID")
+	SchemaIDGUID          = NewAttribute("schemaIDGUID").NonUnique() // Dirty, needs proper FIXME for multi domain
 	RightsGUID            = NewAttribute("rightsGUID")
 	AttributeSecurityGUID = NewAttribute("attributeSecurityGUID")
 
@@ -82,7 +89,7 @@ var (
 )
 
 func init() {
-	AddMergeApprover(func(a, b *Object) (*Object, error) {
+	AddMergeApprover("Don't merge across UniqueSource", func(a, b *Object) (*Object, error) {
 		// Prevents objects from vastly different sources to join across them
 		if a.HasAttr(UniqueSource) && b.HasAttr(UniqueSource) {
 			for _, v := range a.Attr(UniqueSource).Slice() {
@@ -130,31 +137,36 @@ func NewAttribute(name string) Attribute {
 
 	newindex := Attribute(len(attributenames))
 	attributenames[lowername] = newindex
-	attributenums = append(attributenums, attributeinfo{name: name})
+	attributenums = append(attributenums, attributeinfo{
+		Name:     name,
+		DefaultF: true,
+		DefaultM: true,
+		DefaultL: true,
+	})
 	attributemutex.Unlock()
 
 	return Attribute(newindex)
 }
 
 func (a Attribute) String() string {
-	return attributenums[a].name
+	return attributenums[a].Name
 }
 
 func (a Attribute) Multi() Attribute {
 	ai := attributenums[a]
-	ai.multi = true
+	ai.Multi = true
 	attributenums[a] = ai
 	return a
 }
 
 func (a Attribute) IsNonUnique() bool {
 	ai := attributenums[a]
-	return ai.nonunique
+	return ai.NonUnique
 }
 
 func (a Attribute) NonUnique() Attribute {
 	ai := attributenums[a]
-	ai.nonunique = true
+	ai.NonUnique = true
 	attributenums[a] = ai
 	return a
 }
@@ -170,18 +182,21 @@ func StandardMerge(attr Attribute, a, b *Object) (*Object, error) {
 
 func (a Attribute) Merge() Attribute {
 	ai := attributenums[a]
-	ai.merge = true
+	ai.Merge = true
 	attributenums[a] = ai
 	return a
 }
 
-func AddMergeApprover(mf mergefunc) {
-	mergeapprovers = append(mergeapprovers, mf)
+func AddMergeApprover(name string, mf mergefunc) {
+	mergeapprovers = append(mergeapprovers, mergeapproverinfo{
+		name:      name,
+		mergefunc: mf,
+	})
 }
 
 func (a Attribute) Tag(t string) Attribute {
 	ai := attributenums[a]
-	ai.tags = append(ai.tags, t)
+	ai.Tags = append(ai.Tags, t)
 	attributenums[a] = ai
 	return a
 }

modules/engine/object.go

@@ -71,7 +71,7 @@ var IgnoreBlanks = "_IGNOREBLANKS_"
 func NewObject(flexinit ...interface{}) *Object {
 	var result Object
 	result.init()
-	result.SetFlex(flexinit...)
+	result.setFlex(flexinit...)
 
 	return &result
 }
@@ -600,10 +600,13 @@ func (o *Object) SetValues(a Attribute, values ...AttributeValue) {
 }
 
 func (o *Object) SetFlex(flexinit ...interface{}) {
-	var ignoreblanks bool
-
 	o.lock()
-	defer o.unlock()
+	o.setFlex(flexinit...)
+	o.unlock()
+}
+
+func (o *Object) setFlex(flexinit ...interface{}) {
+	var ignoreblanks bool
 
 	var attribute Attribute
 	data := make(AttributeValueSlice, 0, 1)
@@ -613,6 +616,11 @@ func (o *Object) SetFlex(flexinit ...interface{}) {
 			continue
 		}
 		switch v := i.(type) {
+		case windowssecurity.SID:
+			if ignoreblanks && v.IsNull() {
+				continue
+			}
+			data = append(data, AttributeValueSID(v))
 		case *[]string:
 			if v == nil {
 				continue
@@ -783,7 +791,7 @@ func (o *Object) StringNoACL() string {
 		if attr == NTSecurityDescriptor {
 			continue
 		}
-		result += "  " + attributenums[attr].name + ":\n"
+		result += "  " + attributenums[attr].Name + ":\n"
 		for _, value := range values.Slice() {
 			cleanval := stringsx.Clean(value.String())
 			if cleanval != value.String() {
@@ -931,7 +939,10 @@ func (o *Object) Dedup() {
 func (o *Object) ChildOf(parent *Object) {
 	o.lock()
 	if o.parent != nil {
+		// Unlock, as we call thing that lock in the debug message
+		o.unlock()
 		log.Debug().Msgf("Object already %v has %v as parent, so I'm not assigning %v as parent", o.Label(), o.parent.Label(), parent.Label())
+		o.lock()
 		// panic("objects can only have one parent")
 	}
 	o.parent = parent

modules/engine/objects.go

@@ -139,7 +139,7 @@ func (os *Objects) ReindexObject(o *Object) {
 func (os *Objects) updateIndex(o *Object, warn bool) {
 	for attribute := range os.uniqueindex {
 		values := o.Attr(attribute)
-		if values.Len() > 1 && !attributenums[attribute].multi {
+		if values.Len() > 1 && !attributenums[attribute].Multi {
 			log.Warn().Msgf("Encountered multiple values on attribute %v, but is not declared as multival", attribute.String())
 			log.Debug().Msgf("Object dump:\n%s", o.String(os))
 		}
@@ -163,7 +163,7 @@ func (os *Objects) updateIndex(o *Object, warn bool) {
 
 	for attribute := range os.multiindex {
 		values := o.Attr(attribute)
-		if values.Len() > 1 && !attributenums[attribute].multi {
+		if values.Len() > 1 && !attributenums[attribute].Multi {
 			log.Warn().Msgf("Encountered multiple values on attribute %v, but is not declared as multival", attribute.String())
 			log.Debug().Msgf("Object dump:\n%s", o.String(os))
 		}
@@ -201,7 +201,7 @@ func (os *Objects) AddMerge(attrtomerge []Attribute, obs ...*Object) {
 func (os *Objects) AddNew(flexinit ...interface{}) *Object {
 	o := NewObject(flexinit...)
 	if os.DefaultValues != nil {
-		o.SetFlex(os.DefaultValues...)
+		o.setFlex(os.DefaultValues...)
 	}
 	os.lock()
 	os.addmerge(nil, o)
@@ -242,22 +242,23 @@ func (os *Objects) merge(attrtomerge []Attribute, o *Object) bool {
 
 				targetloop:
 					for _, mergetarget := range mergetargets {
-						for _, mf := range mergeapprovers {
-							if mf != nil {
-								res, err := mf(o, mergetarget)
-								switch err {
-								case ErrDontMerge:
-									continue targetloop
-								case ErrMergeOnThis, nil:
-									// Let the code below do the merge
-								default:
-									log.Fatal().Msgf("Error merging %v: %v", o.Label(), err)
-								}
-								if res != nil {
-									// Custom merge - how do we handle this?
-									log.Fatal().Msgf("Custom merge function not supported yet")
-									return false
+						for _, mfi := range mergeapprovers {
+							res, err := mfi.mergefunc(o, mergetarget)
+							switch err {
+							case ErrDontMerge:
+								if !strings.HasPrefix(mfi.name, "QUIET") {
+									log.Debug().Msgf("Not merging %v with %v on %v, because %v said so", o.Label(), mergetarget.Label(), mergeattr.String(), mfi.name)
 								}
+								continue targetloop
+							case ErrMergeOnThis, nil:
+								// Let the code below do the merge
+							default:
+								log.Fatal().Msgf("Error merging %v: %v", o.Label(), err)
+							}
+							if res != nil {
+								// Custom merge - how do we handle this?
+								log.Fatal().Msgf("Custom merge function not supported yet")
+								return false
 							}
 						}
 						log.Trace().Msgf("Merging %v with %v on attribute %v", o.Label(), mergetarget.Label(), mergeattr.String())
@@ -275,7 +276,7 @@ func (os *Objects) merge(attrtomerge []Attribute, o *Object) bool {
 func (os *Objects) add(o *Object) {
 	// Add this to the iterator array
 	if os.DefaultValues != nil {
-		o.SetFlex(os.DefaultValues...)
+		o.setFlex(os.DefaultValues...)
 	}
 
 	// Do chunked extensions for speed

modules/engine/processing.go

@@ -42,7 +42,7 @@ func Merge(aos []*Objects) (*Objects, error) {
 	// Find all the attributes that can be merged objects on
 	var mergeon []Attribute
 	for i, ai := range attributenums {
-		if ai.merge {
+		if ai.Merge {
 			mergeon = append(mergeon, Attribute(i))
 		}
 	}

modules/engine/pwn.go

@@ -92,7 +92,16 @@ type PwnMethod int
 
 var pwnmutex sync.RWMutex
 var pwnnames = make(map[string]PwnMethod)
-var pwnnums []string
+var pwnnums []pwninfo
+
+type pwninfo struct {
+	name                         string
+	tags                         []string
+	multi                        bool // If true, this attribute can have multiple values
+	nonunique                    bool // Doing a Find on this attribute will return multiple results
+	merge                        bool // If true, objects can be merged on this attribute
+	defaultf, defaultm, defaultl bool
+}
 
 func NewPwn(name string) PwnMethod {
 	// Lowercase it, everything is case insensitive
@@ -112,7 +121,12 @@ func NewPwn(name string) PwnMethod {
 	}
 
 	newindex := PwnMethod(len(pwnnums))
-	pwnnums = append(pwnnums, name)
+	pwnnums = append(pwnnums, pwninfo{
+		name:     name,
+		defaultf: true,
+		defaultm: true,
+		defaultl: true,
+	})
 	pwnnames[lowername] = newindex
 	pwnmutex.Unlock()
 
@@ -123,7 +137,7 @@ func (p PwnMethod) String() string {
 	if p == 10000 {
 		return "NOT A PWN METHOD. DIVISION BY ZORRO ERROR."
 	}
-	return pwnnums[p]
+	return pwnnums[p].name
 }
 
 func LookupPwnMethod(name string) PwnMethod {

modules/integrations/activedirectory/analyze/adloader.go

@@ -22,6 +22,7 @@ import (
 var (
 	// FIXME - NEEDS TO ME TRUE AT THE MOMENT AS MAX_IMPORT IS BROKEN4
 	importall = analyze.Command.Flags().Bool("importall", true, "Load all attributes from dump (expands search options, but at the cost of memory")
+	importcnf = analyze.Command.Flags().Bool("importcnf", false, "Import CNF (conflict) objects (experimental)")
 
 	adsource = engine.AttributeValueString("Active Directory dumps")
 	Loader   = engine.AddLoader(&ADLoader{})
@@ -40,6 +41,7 @@ type ADLoader struct {
 	gpofiletoprocess chan string
 	// domains          []domaininfo
 	importall bool
+	importcnf bool
 }
 
 type domaininfo struct {
@@ -53,6 +55,7 @@ func (ld *ADLoader) Name() string {
 
 func (ld *ADLoader) Init() error {
 	ld.importall = *importall
+	ld.importcnf = *importcnf
 
 	ld.dco = make(map[string]*engine.Objects)
 	ld.objectstoconvert = make(chan convertqueueitem, 8192)
@@ -66,6 +69,9 @@ func (ld *ADLoader) Init() error {
 			for item := range ld.objectstoconvert {
 				o := item.object.ToObject(ld.importall)
 
+				if !ld.importcnf && strings.Contains(o.DN(), "\\0ACNF:") {
+					continue // skip conflict object
+				}
 				// Here's a quirky workaround that will bite me later
 				// Legacy well known objects in ForeignSecurityPrincipals gives us trouble with duplicate SIDs - skip them
 				if strings.Count(o.OneAttrString(engine.ObjectSid), "-") == 3 && strings.Contains(o.OneAttrString(engine.DistinguishedName), "CN=ForeignSecurityPrincipals") {

modules/integrations/activedirectory/analyze/gpoimport.go

@@ -36,12 +36,20 @@ var (
 )
 
 func init() {
-	engine.AddMergeApprover(func(a, b *engine.Object) (*engine.Object, error) {
+	engine.AddMergeApprover("Don't merge differing relative paths from GPOs", func(a, b *engine.Object) (*engine.Object, error) {
 		if a.HasAttr(RelativePath) || b.HasAttr(RelativePath) {
 			return nil, engine.ErrDontMerge
 		}
-		return nil, engine.ErrMergeOnThis
+		return nil, nil
 	})
+
+	// engine.AddMergeApprover(func(a, b *engine.Object) (*engine.Object, error) {
+	// 	if a.HasAttr(engine.ObjectSid) && b.HasAttr(engine.ObjectSid) && a.OneAttr(engine.ObjectSid).String() == b.OneAttr(engine.ObjectSid).String() {
+	// 		return nil, engine.ErrDontMerge
+	// 	}
+	// 	return nil, engine.ErrMergeOnThis
+	// })
+
 }
 
 var cpasswordusername = regexp.MustCompile(`(?i)cpassword="(?P<password>[^"]+)[^>]+(runAs|userName)="(?P<username>[^"]+)"`)

modules/integrations/activedirectory/attributes.go

@@ -89,7 +89,7 @@ var (
 )
 
 func init() {
-	engine.AddMergeApprover(func(a, b *engine.Object) (result *engine.Object, err error) {
+	engine.AddMergeApprover("Don't merge differing distinguishedName", func(a, b *engine.Object) (result *engine.Object, err error) {
 		if a.HasAttr(engine.DistinguishedName) && b.HasAttr(engine.DistinguishedName) && a.DN() != b.DN() {
 			// Yes, it happens we have objects that have different DNs but the same merge attributes (objectSID, etc)
 			return nil, engine.ErrDontMerge