many various improvements

engine/plugins/api/gleif/fuzzy.go

@@ -133,7 +133,7 @@ func (fc *fuzzyCompletions) query(e *et.Event, orgent *dbt.Entity) *dbt.Entity {
 
 				leiList = append(leiList, &general.Identifier{
 					UniqueID: fmt.Sprintf("%s:%s", general.LEICode, id),
-					EntityID: id,
+					ID:       id,
 					Type:     general.LEICode,
 				})
 			}

engine/plugins/api/gleif/lei_record.go

@@ -20,7 +20,7 @@ import (
 func (g *gleif) getLEIRecord(id *general.Identifier) (*leiRecord, error) {
 	g.rlimit.Take()
 
-	u := "https://api.gleif.org/api/v1/lei-records/" + id.EntityID
+	u := "https://api.gleif.org/api/v1/lei-records/" + id.ID
 	resp, err := http.RequestWebPage(context.TODO(), &http.Request{URL: u})
 	if err != nil || resp.StatusCode != 200 || resp.Body == "" {
 		return nil, err
@@ -29,7 +29,7 @@ func (g *gleif) getLEIRecord(id *general.Identifier) (*leiRecord, error) {
 	var result singleResponse
 	if err := json.Unmarshal([]byte(resp.Body), &result); err != nil {
 		return nil, err
-	} else if result.Data.Type != "lei-records" || result.Data.ID != id.EntityID {
+	} else if result.Data.Type != "lei-records" || result.Data.ID != id.ID {
 		return nil, errors.New("failed to find the LEI record")
 	}
 	return &result.Data, nil
@@ -38,7 +38,7 @@ func (g *gleif) getLEIRecord(id *general.Identifier) (*leiRecord, error) {
 func (g *gleif) getDirectParentRecord(id *general.Identifier) (*leiRecord, error) {
 	g.rlimit.Take()
 
-	u := "https://api.gleif.org/api/v1/lei-records/" + id.EntityID + "/direct-parent"
+	u := "https://api.gleif.org/api/v1/lei-records/" + id.ID + "/direct-parent"
 	resp, err := http.RequestWebPage(context.TODO(), &http.Request{URL: u})
 	if err != nil || resp.StatusCode != 200 || resp.Body == "" {
 		return nil, err
@@ -57,7 +57,7 @@ func (g *gleif) getDirectChildrenRecords(id *general.Identifier) ([]*leiRecord,
 	var children []*leiRecord
 
 	last := 1
-	link := "https://api.gleif.org/api/v1/lei-records/" + id.EntityID + "/direct-children"
+	link := "https://api.gleif.org/api/v1/lei-records/" + id.ID + "/direct-children"
 	for i := 1; i <= last && link != ""; i++ {
 		g.rlimit.Take()
 
@@ -106,7 +106,7 @@ func (g *gleif) createLEIIdentifier(session et.Session, orgent *dbt.Entity, lei
 func (g *gleif) createLEIFromRecord(e *et.Event, orgent *dbt.Entity, lei *leiRecord) (*dbt.Entity, error) {
 	return g.createLEIIdentifier(e.Session, orgent, &general.Identifier{
 		UniqueID:       fmt.Sprintf("%s:%s", general.LEICode, lei.ID),
-		EntityID:       lei.ID,
+		ID:             lei.ID,
 		Type:           general.LEICode,
 		Status:         lei.Attributes.Registration.Status,
 		CreationDate:   lei.Attributes.Registration.InitialRegistrationDate,

engine/plugins/api/gleif/lei_record_test.go

@@ -19,7 +19,7 @@ func TestGetLEIRecord(t *testing.T) {
 	lei := "ZXTILKJKG63JELOEG630"
 	id := &general.Identifier{
 		UniqueID: fmt.Sprintf("%s:%s", general.LEICode, lei),
-		EntityID: lei,
+		ID:       lei,
 		Type:     general.LEICode,
 	}
 
@@ -36,7 +36,7 @@ func TestGetDirectParentRecord(t *testing.T) {
 	lei := "25490065U2GR0UPXFY63"
 	id := &general.Identifier{
 		UniqueID: fmt.Sprintf("%s:%s", general.LEICode, lei),
-		EntityID: lei,
+		ID:       lei,
 		Type:     general.LEICode,
 	}
 
@@ -53,7 +53,7 @@ func TestGetDirectChildrenRecord(t *testing.T) {
 	lei := "ZXTILKJKG63JELOEG630"
 	id := &general.Identifier{
 		UniqueID: fmt.Sprintf("%s:%s", general.LEICode, lei),
-		EntityID: lei,
+		ID:       lei,
 		Type:     general.LEICode,
 	}
 

engine/plugins/api/gleif/org_lei.go

@@ -51,7 +51,7 @@ func (g *gleif) updateOrgFromLEIRecord(e *et.Event, orgent *dbt.Entity, lei *lei
 	// check if the org entity already has a LEI identifier
 	if leient := g.orgEntityToLEI(e, orgent); leient != nil {
 		// check if the LEI identifier is the same as the one we are processing
-		if id, ok := leient.Asset.(*general.Identifier); ok && id.EntityID != lei.ID {
+		if id, ok := leient.Asset.(*general.Identifier); ok && id.ID != lei.ID {
 			return
 		}
 	}
@@ -136,7 +136,7 @@ func (g *gleif) addIdentifiersToOrg(e *et.Event, orgent *dbt.Entity, idtype stri
 
 		oamid := &general.Identifier{
 			UniqueID: fmt.Sprintf("%s:%s", idtype, id),
-			EntityID: id,
+			ID:       id,
 			Type:     idtype,
 		}
 

engine/plugins/api/rdap/netblock.go

@@ -68,12 +68,11 @@ func (nb *netblock) lookup(e *et.Event, cidr string, since time.Time) *dbt.Entit
 func (nb *netblock) query(e *et.Event, asset *dbt.Entity) (*dbt.Entity, *rdap.IPNetwork) {
 	n := asset.Asset.(*network.Netblock)
 
-	var req *rdap.Request
 	_, ipnet, err := net.ParseCIDR(n.CIDR.String())
 	if err != nil {
 		return nil, nil
 	}
-	req = rdap.NewIPNetRequest(ipnet)
+	req := rdap.NewIPNetRequest(ipnet)
 
 	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
 	defer cancel()
@@ -89,10 +88,10 @@ func (nb *netblock) query(e *et.Event, asset *dbt.Entity) (*dbt.Entity, *rdap.IP
 	if !ok {
 		return nil, nil
 	}
-	return nb.store(e, record, asset, nb.plugin.source), record
+	return nb.store(e, record, asset), record
 }
 
-func (nb *netblock) store(e *et.Event, resp *rdap.IPNetwork, asset *dbt.Entity, src *et.Source) *dbt.Entity {
+func (nb *netblock) store(e *et.Event, resp *rdap.IPNetwork, asset *dbt.Entity) *dbt.Entity {
 	n := asset.Asset.(*network.Netblock)
 	ipnetrec := &oamreg.IPNetRecord{
 		CIDR:         n.CIDR,
@@ -128,14 +127,19 @@ func (nb *netblock) store(e *et.Event, resp *rdap.IPNetwork, asset *dbt.Entity,
 
 	record, err := e.Session.Cache().CreateAsset(ipnetrec)
 	if err == nil && record != nil {
+		_, _ = e.Session.Cache().CreateEntityProperty(record, &general.SourceProperty{
+			Source:     nb.plugin.source.Name,
+			Confidence: nb.plugin.source.Confidence,
+		})
+
 		if edge, err := e.Session.Cache().CreateEdge(&dbt.Edge{
 			Relation:   &general.SimpleRelation{Name: "registration"},
 			FromEntity: asset,
 			ToEntity:   record,
 		}); err == nil && edge != nil {
 			_, _ = e.Session.Cache().CreateEdgeProperty(edge, &general.SourceProperty{
-				Source:     src.Name,
-				Confidence: src.Confidence,
+				Source:     nb.plugin.source.Name,
+				Confidence: nb.plugin.source.Confidence,
 			})
 		}
 	}

engine/plugins/api/rdap/plugin.go

@@ -240,7 +240,7 @@ func (rd *rdapPlugin) storeEntity(e *et.Event, level int, entity *rdap.Entity, a
 	if email := strings.ToLower(v.Email()); m.IsMatch(string(oam.Identifier)) && email != "" {
 		if a, err := e.Session.Cache().CreateAsset(&general.Identifier{
 			UniqueID: fmt.Sprintf("%s:%s", general.EmailAddress, email),
-			EntityID: email,
+			ID:       email,
 			Type:     general.EmailAddress,
 		}); err == nil && a != nil {
 			_ = rd.createContactEdge(e.Session, cr, a, &general.SimpleRelation{Name: "id"}, src)

engine/plugins/enrich/banner_url.go

@@ -7,12 +7,15 @@ package enrich
 import (
 	"errors"
 	"log/slog"
+	"net/netip"
 
 	"github.com/owasp-amass/amass/v4/engine/plugins/support"
 	et "github.com/owasp-amass/amass/v4/engine/types"
 	dbt "github.com/owasp-amass/asset-db/types"
 	oam "github.com/owasp-amass/open-asset-model"
+	oamdns "github.com/owasp-amass/open-asset-model/dns"
 	"github.com/owasp-amass/open-asset-model/general"
+	oamnet "github.com/owasp-amass/open-asset-model/network"
 	"github.com/owasp-amass/open-asset-model/platform"
 	oamurl "github.com/owasp-amass/open-asset-model/url"
 )
@@ -89,11 +92,23 @@ func (bu *bannerURLs) query(e *et.Event, asset *dbt.Entity) []*dbt.Entity {
 		return nil
 	}
 
-	var results []*dbt.Entity
+	var results []*oamurl.URL
+	// TODO: in the future, further investigation of out of scope URLs may be needed
 	if urls := support.ExtractURLsFromString(serv.Output); len(urls) > 0 {
-		results = append(results, bu.store(e, urls)...)
+		for _, u := range urls {
+			if addr, err := netip.ParseAddr(u.Host); err == nil {
+				if _, conf := e.Session.Scope().IsAssetInScope(&oamnet.IPAddress{Address: addr}, 0); conf > 0 {
+					results = append(results, u)
+				}
+			} else {
+				if _, conf := e.Session.Scope().IsAssetInScope(&oamdns.FQDN{Name: u.Host}, 0); conf > 0 {
+					results = append(results, u)
+				}
+			}
+		}
 	}
-	return results
+
+	return bu.store(e, results)
 }
 
 func (bu *bannerURLs) store(e *et.Event, urls []*oamurl.URL) []*dbt.Entity {

engine/plugins/enrich/email.go

@@ -60,7 +60,7 @@ func (ee *emailexpand) Stop() {
 
 func (ee *emailexpand) check(e *et.Event) error {
 	if id, ok := e.Entity.Asset.(*general.Identifier); !ok ||
-		id == nil || id.Type != general.EmailAddress || id.EntityID == "" {
+		id == nil || id.Type != general.EmailAddress || id.ID == "" {
 		return nil
 	}
 
@@ -74,7 +74,7 @@ func (ee *emailexpand) store(e *et.Event, asset *dbt.Entity) []*support.Finding
 	var findings []*support.Finding
 	oame := asset.Asset.(*general.Identifier)
 
-	parts := strings.Split(oame.EntityID, "@")
+	parts := strings.Split(oame.ID, "@")
 	if len(parts) != 2 {
 		return findings
 	}

engine/plugins/enrich/tls_cert.go

@@ -218,7 +218,7 @@ func (te *tlsexpand) store(e *et.Event, cert *x509.Certificate, asset *dbt.Entit
 
 			if a, err := e.Session.Cache().CreateAsset(&general.Identifier{
 				UniqueID: fmt.Sprintf("%s:%s", general.EmailAddress, email),
-				EntityID: email,
+				ID:       email,
 				Type:     general.EmailAddress,
 			}); err == nil && a != nil {
 				findings = append(findings, &support.Finding{