Apply configured include/ignore lists to GitLab unit scans (#4592)

Previously, the GitLab include and ignore lists were only applied during repository enumeration, which meant that they would be ignored after enumeration completed. For large environments, post-enumeration scanning can take days, and it was awkward that the include/ignore lists could effectively not be modified during that time. This PR changes things such that repositories can be configured to be skipped even post-enumeration.

Importantly, repositories cannot be "un-ignored" post-enumeration. This is unfortunate, but this PR still represents improvement on the status quo.

Author: rosecodym

pkg/sources/gitlab/gitlab.go

@@ -39,13 +39,18 @@ type Source struct {
 	jobID    sources.JobID
 	verify   bool
 
-	authMethod   string
-	user         string
-	password     string
-	token        string
-	url          string
-	repos        []string
-	groupIds     []string
+	authMethod string
+	user       string
+	password   string
+	token      string
+	url        string
+	repos      []string
+	groupIds   []string
+
+	// These lists are checked both during enumeration and when ChunkUnit is called. This means that if they're modified
+	// between enumeration and individual unit scans, units will be scanned only if they pass the filter during
+	// enumeration and also if they pass the filter during unit scanning. This means that units can be "removed" from
+	// the enumerated list post-enumeration by modifying the filters, but they can never be added post-enumeration.
 	ignoreRepos  []string
 	includeRepos []string
 
@@ -1086,6 +1091,14 @@ func (s *Source) Enumerate(ctx context.Context, reporter sources.UnitReporter) e
 func (s *Source) ChunkUnit(ctx context.Context, unit sources.SourceUnit, reporter sources.ChunkReporter) error {
 	repoURL, _ := unit.SourceUnitID()
 
+	ignoreRepo := buildIgnorer(s.includeRepos, s.ignoreRepos, func(err error, pattern string) {
+		ctx.Logger().Error(err, "could not compile include/exclude repo glob", "glob", pattern)
+	})
+	if ignoreRepo(repoURL) {
+		ctx.Logger().V(3).Info("skipping project", "reason", "ignored in config")
+		return nil
+	}
+
 	var path string
 	var repo *gogit.Repository
 	var err error

pkg/sources/gitlab/gitlab_integration_test.go

@@ -11,6 +11,7 @@ import (
 
 	"github.com/kylelemons/godebug/pretty"
 	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
 	"google.golang.org/protobuf/types/known/anypb"
 
 	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
@@ -498,6 +499,79 @@ func TestSource_Chunks_TargetedScan(t *testing.T) {
 	}
 }
 
+func TestSource_ChunkUnit_RepoFiltersRespected(t *testing.T) {
+	ctx := context.Background()
+
+	// Arrange: Get test environment token
+	secret, err := common.GetTestSecret(ctx)
+	if err != nil {
+		t.Fatal(fmt.Errorf("failed to access secret: %v", err))
+	}
+	token := secret.MustGetField("GITLAB_TOKEN")
+
+	// Arrange: Build a unit to scan
+	unit := sources.CommonSourceUnit{
+		Kind: "repo",
+		ID:   "https://gitlab.com/testermctestface/testy",
+	}
+
+	tests := []struct {
+		name          string
+		includeRepos  []string
+		ignoreRepos   []string
+		wantAnyChunks bool
+	}{
+		{
+			name:          "empty include, empty ignore",
+			wantAnyChunks: true,
+		},
+		{
+			name:          "unit matches include",
+			includeRepos:  []string{"https://gitlab.com/testermctestface/testy"},
+			wantAnyChunks: true,
+		},
+		{
+			name:          "unit does not match include",
+			includeRepos:  []string{"https://gitlab.com/testermctestface/something-else"},
+			wantAnyChunks: false,
+		},
+		{
+			name:          "unit matches ignore",
+			ignoreRepos:   []string{"https://gitlab.com/testermctestface/testy"},
+			wantAnyChunks: false,
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			// Arrange: Create the connection
+			typedConn := &sourcespb.GitLab{
+				Credential: &sourcespb.GitLab_Token{
+					Token: token,
+				},
+				IncludeRepos: tt.includeRepos,
+				IgnoreRepos:  tt.ignoreRepos,
+			}
+			conn, err := anypb.New(typedConn)
+			require.NoError(t, err)
+
+			// Arrange: Instantiate and initialize the source
+			s := &Source{}
+			require.NoError(t, s.Init(ctx, "test source", 1, 1, false, conn, 1))
+
+			// Arrange: Build the chunk reporter
+			chunksChan := make(chan *sources.Chunk, 1024)
+			chunkReporter := sources.ChanReporter{chunksChan}
+
+			// Act: Scan the unit
+			require.NoError(t, s.ChunkUnit(ctx, unit, chunkReporter))
+
+			// Assert: Verify that chunk production was correct
+			assert.Equal(t, tt.wantAnyChunks, len(chunksChan) > 0)
+		})
+	}
+}
+
 func TestSource_InclusionGlobbing(t *testing.T) {
 	ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
 	defer cancel()