`BeforeEachFeature` and `AfterEachFeature` are not skipped even though feature labels indicate so

[pmalek]

What happened?

Given the reproduction code below I'd expect the feature setup and teardown (set using BeforeEachFeature and AfterEachFeature) to not execute yet those do execute:

 go test -v -count 1 . -args -labels=type=simple
=== RUN   TestABC1
=== RUN   TestABC1/pod_list_1
    abc_test.go:16: test setup 1
=== RUN   TestABC1/pod_list_1/assess_1
=== NAME  TestABC1
    main_test.go:34: feat teardown
    main_test.go:29: feat setup
=== RUN   TestABC1/pod_list_2
    env.go:438: Skipping feature "pod list 2": unmatched labels "[type=[complex]]"
=== NAME  TestABC1
    main_test.go:34: feat teardown
--- PASS: TestABC1 (0.00s)
    --- PASS: TestABC1/pod_list_1 (0.00s)
        --- PASS: TestABC1/pod_list_1/assess_1 (0.00s)
    --- SKIP: TestABC1/pod_list_2 (0.00s)
=== RUN   TestABC2
    main_test.go:29: feat setup
=== RUN   TestABC2/pod_list_2
    env.go:438: Skipping feature "pod list 2": unmatched labels "[type=[complex]]"
=== NAME  TestABC2
    main_test.go:34: feat teardown
--- PASS: TestABC2 (0.00s)
    --- SKIP: TestABC2/pod_list_2 (0.00s)
PASS
ok  	e2e-test-features	0.019s

Only the setup set via features.New("").Setup(...) is skipped.

What did you expect to happen?

When filters indicate a skipped feature, both setup and teardown set with BeforeEachFeature and AfterEachFeature are skipped.

How can we reproduce it (as minimally and precisely as possible)?

package main

import (
	"context"
	"os"
	"testing"

	"sigs.k8s.io/e2e-framework/pkg/env"
	"sigs.k8s.io/e2e-framework/pkg/envconf"
	"sigs.k8s.io/e2e-framework/pkg/features"
)

var tenv env.Environment

func TestMain(m *testing.M) {
	cfg, err := envconf.NewFromFlags()
	if err != nil {
		panic(err)
	}
	tenv = env.NewWithConfig(cfg)

	tenv.BeforeEachFeature(featSetup)
	tenv.AfterEachFeature(featTeardown)

	os.Exit(tenv.Run(m))
}

func featSetup(ctx context.Context, c *envconf.Config, t *testing.T, f features.Feature) (context.Context, error) {
	t.Log("feat setup")
	return ctx, nil
}

func featTeardown(ctx context.Context, c *envconf.Config, t *testing.T, f features.Feature) (context.Context, error) {
	t.Log("feat teardown")
	return ctx, nil
}

package main

import (
	"context"
	"testing"

	"sigs.k8s.io/e2e-framework/pkg/envconf"
	"sigs.k8s.io/e2e-framework/pkg/features"
)

func TestABC1(t *testing.T) {
	f1 := features.
		New("pod list 1").
		WithLabel("type", "simple").
		Setup(func(ctx context.Context, t *testing.T, c *envconf.Config) context.Context {
			t.Log("test setup 1")
			return ctx
		}).
		Assess("assess 1", func(ctx context.Context, t *testing.T, cfg *envconf.Config) context.Context {
			return ctx
		})

	f2 := features.
		New("pod list 2").
		WithLabel("type", "complex").
		Setup(func(ctx context.Context, t *testing.T, c *envconf.Config) context.Context {
			t.Log("test setup 2")
			return ctx
		}).
		Assess("assess 2", func(ctx context.Context, t *testing.T, cfg *envconf.Config) context.Context {
			return ctx
		})

	tenv.Test(t, f1.Feature(), f2.Feature())
}

func TestABC2(t *testing.T) {
	f := features.
		New("pod list 2").
		WithLabel("type", "complex").
		Setup(func(ctx context.Context, t *testing.T, c *envconf.Config) context.Context {
			t.Log("test setup 2")
			return ctx
		}).
		Assess("assess 2", func(ctx context.Context, t *testing.T, cfg *envconf.Config) context.Context {
			return ctx
		})

	tenv.Test(t, f.Feature())
}

Anything elese we need to know?

No response

E2E Provider Used

kind

e2e-framework Version

v0.3.0

OS version

darwin, 13.5.2, go 1.21.2

[pmalek]

@matrus2 Thanks for fixing this!