Skip to content

TRT-2471: Consider mass failures during CR query #3285

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
xueqzhan wants to merge 2 commits into
base: main
Choose a base branch
from
+457 −41
Open

TRT-2471: Consider mass failures during CR query #3285

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
36f26ac
Filter CR query by key tests
xueqzhan Jan 27, 2026
7c39f99
Prioritize key tests so that only one will be counted for regression …
xueqzhan Feb 19, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions cmd/sippy/component_readiness.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -186,6 +186,9 @@ func (f *ComponentReadinessFlags) runServerMode() error {
log.WithError(err).Warn("unable to initialize Jira client, bug filing will be disabled")
}

// Get exclusive test names for mass failure filtering
exclusiveTestNames := f.ComponentReadinessFlags.GetMassFailureTestNames()

server := sippyserver.NewServer(
sippyserver.ModeOpenShift,
f.APIFlags.ListenAddr,
Expand All @@ -206,6 +209,7 @@ func (f *ComponentReadinessFlags) runServerMode() error {
f.APIFlags.EnableWriteEndpoints,
"", // No chat API in Component Readiness
jiraClient,
exclusiveTestNames,
)

if f.APIFlags.MetricsAddr != "" {
Expand Down
1 change: 1 addition & 0 deletions cmd/sippy/serve.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -175,6 +175,7 @@ func NewServeCommand() *cobra.Command {
f.APIFlags.EnableWriteEndpoints,
f.APIFlags.ChatAPIURL,
jiraClient,
f.ComponentReadinessFlags.GetMassFailureTestNames(),
)

if f.APIFlags.MetricsAddr != "" {
Expand Down
4 changes: 2 additions & 2 deletions pkg/api/componentreadiness/middleware/releasefallback/releasefallback.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -541,7 +541,7 @@ func (f *fallbackTestQueryGenerator) getCacheKey() fallbackTestQueryGeneratorCac
func (f *fallbackTestQueryGenerator) getTestFallbackRelease(ctx context.Context) (bq.ReportTestStatus, []error) {
commonQuery, groupByQuery, queryParameters := query.BuildComponentReportQuery(
f.client, f.ReqOptions, f.allVariants, f.ReqOptions.VariantOption.IncludeVariants,
query.DefaultJunitTable, false)
query.DefaultJunitTable, false, f.ReqOptions.AdvancedOption.ExclusiveTestNames...)
before := time.Now()
log.Infof("Starting Fallback (%s) QueryTestStatus", f.BaseRelease)
errs := []error{}
Expand All @@ -564,7 +564,7 @@ func (f *fallbackTestQueryGenerator) getTestFallbackRelease(ctx context.Context)
},
}...)

baseStatus, baseErrs := query.FetchTestStatusResults(ctx, baseQuery)
baseStatus, baseErrs := query.FetchTestStatusResults(ctx, baseQuery, f.ReqOptions.AdvancedOption.ExclusiveTestNames)

if len(baseErrs) != 0 {
errs = append(errs, baseErrs...)
Expand Down
130 changes: 103 additions & 27 deletions pkg/api/componentreadiness/query/querygenerators.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -47,6 +47,7 @@ const (
// So, this sorts the data, partitioning by the 3-tuple of file_path/test_name/testsuite -
// preferring flakes, then successes, then failures, and we get the first row of each
// partition.
// partition.
dedupedJunitTable = `
Comment on lines +50 to 51
Copy link
Contributor

@coderabbitai coderabbitai bot Feb 19, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

⚠️ Potential issue | 🟡 Minor

Remove duplicate comment line.

Line 50 appears to be a duplicate of the comment ending on line 49 (// partition.). This looks like an editing artifact.

Proposed fix 
 	// So, this sorts the data, partitioning by the 3-tuple of file_path/test_name/testsuite -
 	// preferring flakes, then successes, then failures, and we get the first row of each
 	// partition.
-	// partition.
 	dedupedJunitTable = `
📝 Committable suggestion

‼️ IMPORTANT
Carefully review the code before committing. Ensure that it accurately replaces the highlighted code, contains no missing lines, and has no issues with indentation. Thoroughly test & benchmark the code to ensure it meets the requirements.

Suggested change
// partition.
dedupedJunitTable = `
// So, this sorts the data, partitioning by the 3-tuple of file_path/test_name/testsuite -
// preferring flakes, then successes, then failures, and we get the first row of each
// partition.
dedupedJunitTable = `
🤖 Prompt for AI Agents 
Verify each finding against the current code and only fix it if needed.

In `@pkg/api/componentreadiness/query/querygenerators.go` around lines 50 - 51,
Remove the duplicated comment line "// partition." that appears immediately
before the dedupedJunitTable declaration; locate the comment just above the
dedupedJunitTable variable in querygenerators.go and delete the extra duplicate
so only a single "// partition." comment remains.

WITH deduped_testcases AS (
SELECT
Expand Down Expand Up @@ -135,9 +136,8 @@ func NewBaseQueryGenerator(

func (b *baseQueryGenerator) QueryTestStatus(ctx context.Context) (bq.ReportTestStatus, []error) {

commonQuery, groupByQuery, queryParameters := BuildComponentReportQuery(b.client, b.ReqOptions, b.allVariants, b.ReqOptions.VariantOption.IncludeVariants, DefaultJunitTable, false)
commonQuery, groupByQuery, queryParameters := BuildComponentReportQuery(b.client, b.ReqOptions, b.allVariants, b.ReqOptions.VariantOption.IncludeVariants, DefaultJunitTable, false, b.ReqOptions.AdvancedOption.ExclusiveTestNames...)

before := time.Now()
errs := []error{}
baseString := commonQuery + ` AND jv_Release.variant_value = @BaseRelease`
baseQuery := b.client.Query(ctx, bqlabel.CRJunitBase, baseString+groupByQuery)
Expand All @@ -158,14 +158,12 @@ func (b *baseQueryGenerator) QueryTestStatus(ctx context.Context) (bq.ReportTest
},
}...)

baseStatus, baseErrs := FetchTestStatusResults(ctx, baseQuery)
baseStatus, baseErrs := FetchTestStatusResults(ctx, baseQuery, b.ReqOptions.AdvancedOption.ExclusiveTestNames)

if len(baseErrs) != 0 {
errs = append(errs, baseErrs...)
}

log.Infof("Base QueryTestStatus completed in %s with %d base results from db", time.Since(before), len(baseStatus))

return bq.ReportTestStatus{BaseStatus: baseStatus}, errs
}

Expand Down Expand Up @@ -207,9 +205,8 @@ func NewSampleQueryGenerator(
}

func (s *sampleQueryGenerator) QueryTestStatus(ctx context.Context) (bq.ReportTestStatus, []error) {
commonQuery, groupByQuery, queryParameters := BuildComponentReportQuery(s.client, s.ReqOptions, s.allVariants, s.IncludeVariants, s.JunitTable, true)
commonQuery, groupByQuery, queryParameters := BuildComponentReportQuery(s.client, s.ReqOptions, s.allVariants, s.IncludeVariants, s.JunitTable, true, s.ReqOptions.AdvancedOption.ExclusiveTestNames...)

before := time.Now()
errs := []error{}
sampleString := commonQuery
// Only set sample release when PR and payload options are not set
Expand Down Expand Up @@ -267,25 +264,41 @@ func (s *sampleQueryGenerator) QueryTestStatus(ctx context.Context) (bq.ReportTe
}...)
}

sampleStatus, sampleErrs := FetchTestStatusResults(ctx, sampleQuery)
sampleStatus, sampleErrs := FetchTestStatusResults(ctx, sampleQuery, s.ReqOptions.AdvancedOption.ExclusiveTestNames)

if len(sampleErrs) != 0 {
errs = append(errs, sampleErrs...)
}

log.Infof("Sample QueryTestStatus completed in %s with %d sample results db", time.Since(before), len(sampleStatus))

return bq.ReportTestStatus{SampleStatus: sampleStatus}, errs
}

// buildPriorityCaseStatement generates a SQL CASE statement that assigns priority based on test position in the list.
// Lower index = higher priority. This is used to ensure when multiple exclusive tests appear in the same job,
// only the highest priority one is counted.
func buildPriorityCaseStatement(exclusiveTestNames []string) string {
var caseStatements []string
for i, testName := range exclusiveTestNames {
// Escape single quotes in test name for SQL
escapedTestName := strings.ReplaceAll(testName, "'", "''")
caseStatements = append(caseStatements, fmt.Sprintf("WHEN test_name = '%s' THEN %d", escapedTestName, i))
}
// Add a default case to handle any unexpected tests (should not happen due to IN UNNEST filter)
caseStatements = append(caseStatements, fmt.Sprintf("ELSE %d", len(exclusiveTestNames)))
return strings.Join(caseStatements, "\n\t\t\t\t\t\t\t")
}

// BuildComponentReportQuery returns the common query for the higher level summary component summary.
// If exclusiveTestNames is provided, jobs containing tests from this set will have all other tests excluded,
// and only the highest priority (earliest in the list) exclusive test will be included.
func BuildComponentReportQuery(
client *bqcachedclient.Client,
reqOptions reqopts.RequestOptions,
allJobVariants crtest.JobVariants,
includeVariants map[string][]string,
junitTable string,
isSample bool,
exclusiveTestNames ...string,
) (string, string, []bigquery.QueryParameter) {
// Parts of the query, including the columns returned, are dynamic, based on the list of variants we're told to work with.
// Variants will be returned as columns with names like: variant_[VariantName]
Expand All @@ -294,7 +307,7 @@ func BuildComponentReportQuery(
joinVariants := ""
groupByVariants := ""
for _, v := range sortedKeys(allJobVariants.Variants) {
joinVariants += fmt.Sprintf("LEFT JOIN %s.job_variants jv_%s ON variant_registry_job_name = jv_%s.job_name AND jv_%s.variant_name = '%s'\n",
joinVariants += fmt.Sprintf("LEFT JOIN %s.job_variants jv_%s ON junit_data.variant_registry_job_name = jv_%s.job_name AND jv_%s.variant_name = '%s'\n",
client.Dataset, v, v, v, v)
}
for _, v := range reqOptions.VariantOption.DBGroupBy.List() {
Expand All @@ -313,33 +326,95 @@ func BuildComponentReportQuery(
// TODO: last_failure here explicitly uses success_val not adjusted_success_val, this ensures we
// show the last time the test failed, not flaked. if you enable the flakes as failures feature (which is
// non default today), the last failure time will be wrong which can impact things like failed fix detection.
queryString := fmt.Sprintf(`WITH latest_component_mapping AS (
SELECT *
FROM %s.component_mapping cm
WHERE created_at = (
SELECT MAX(created_at)
FROM %s.component_mapping))
// Build the WITH clause - add jobs_with_failed_exclusive_tests CTE if exclusiveTestNames is provided
withClause := ""
if len(exclusiveTestNames) > 0 {
// Create a CTE that identifies the highest priority (lowest index) exclusive test in each job
// This ensures when multiple exclusive tests appear in the same job, only the highest priority one is used
withClause = fmt.Sprintf(`WITH exclusive_test_priorities AS (
SELECT
prowjob_build_id,
test_name,
-- Find the index/priority of each test (lower index = higher priority)
CASE
%s
END AS test_priority
FROM %s.%s AS junit
WHERE modified_time >= DATETIME(@From)
AND modified_time < DATETIME(@To)
AND test_name IN UNNEST(@ExclusiveTestNames)
AND success_val = 0
),
jobs_with_highest_priority_test AS (
SELECT
prowjob_build_id,
test_name
FROM exclusive_test_priorities
WHERE test_priority = (
SELECT MIN(test_priority)
FROM exclusive_test_priorities ep2
WHERE ep2.prowjob_build_id = exclusive_test_priorities.prowjob_build_id
)
),
latest_component_mapping AS (
SELECT *
FROM %s.component_mapping cm
WHERE created_at = (
SELECT MAX(created_at)
FROM %s.component_mapping))`,
buildPriorityCaseStatement(exclusiveTestNames), client.Dataset, junitTable, client.Dataset, client.Dataset)
} else {
withClause = fmt.Sprintf(`WITH latest_component_mapping AS (
SELECT *
FROM %s.component_mapping cm
WHERE created_at = (
SELECT MAX(created_at)
FROM %s.component_mapping))`,
client.Dataset, client.Dataset)
}

queryString := fmt.Sprintf(`%s
SELECT
ANY_VALUE(test_name HAVING MAX prowjob_start) AS test_name,
ANY_VALUE(testsuite HAVING MAX prowjob_start) AS test_suite,
ANY_VALUE(junit_data.test_name HAVING MAX junit_data.prowjob_start) AS test_name,
ANY_VALUE(junit_data.testsuite HAVING MAX junit_data.prowjob_start) AS test_suite,
cm.id as test_id,
%s
COUNT(cm.id) AS total_count,
SUM(adjusted_success_val) AS success_count,
SUM(adjusted_flake_count) AS flake_count,
MAX(CASE WHEN success_val = 0 THEN prowjob_start ELSE NULL END) AS last_failure,
SUM(junit_data.adjusted_success_val) AS success_count,
SUM(junit_data.adjusted_flake_count) AS flake_count,
MAX(CASE WHEN junit_data.success_val = 0 THEN junit_data.prowjob_start ELSE NULL END) AS last_failure,
ANY_VALUE(cm.component) AS component,
ANY_VALUE(cm.capabilities) AS capabilities,
FROM (%s)
INNER JOIN latest_component_mapping cm ON testsuite = cm.suite AND test_name = cm.name
FROM (%s) AS junit_data
INNER JOIN latest_component_mapping cm ON junit_data.testsuite = cm.suite AND junit_data.test_name = cm.name
`,
client.Dataset, client.Dataset, selectVariants, fmt.Sprintf(dedupedJunitTable, jobNameQueryPortion, client.Dataset, junitTable, client.Dataset, client.Dataset, jobRunAnnotationToIgnore))
withClause, selectVariants, fmt.Sprintf(dedupedJunitTable, jobNameQueryPortion, client.Dataset, junitTable, client.Dataset, client.Dataset, jobRunAnnotationToIgnore))

queryString += joinVariants

queryString += `WHERE cm.staff_approved_obsolete = false AND
(variant_registry_job_name LIKE 'periodic-%%' OR variant_registry_job_name LIKE 'release-%%' OR variant_registry_job_name LIKE 'aggregator-%%')`
(junit_data.variant_registry_job_name LIKE 'periodic-%%' OR junit_data.variant_registry_job_name LIKE 'release-%%' OR junit_data.variant_registry_job_name LIKE 'aggregator-%%')`
commonParams := []bigquery.QueryParameter{}

// Add filtering logic for exclusive tests with priority
// Only include the highest priority test from each job, and exclude all other tests from those jobs
if len(exclusiveTestNames) > 0 {
queryString += `
AND (
-- Include tests from jobs that don't have any failed exclusive tests
junit_data.prowjob_build_id NOT IN (SELECT prowjob_build_id FROM jobs_with_highest_priority_test)
-- Or include only the highest priority exclusive test from jobs that have them
OR EXISTS (
SELECT 1 FROM jobs_with_highest_priority_test j
WHERE j.prowjob_build_id = junit_data.prowjob_build_id
AND j.test_name = junit_data.test_name
)
)`
commonParams = append(commonParams, bigquery.QueryParameter{
Name: "ExclusiveTestNames",
Value: exclusiveTestNames,
})
}
if reqOptions.AdvancedOption.IgnoreDisruption {
queryString += ` AND NOT 'Disruption' in UNNEST(capabilities)`
}
Expand Down Expand Up @@ -592,7 +667,7 @@ func filterByCrossCompareVariants(crossCompare []string, variantGroups map[strin
return
}

func FetchTestStatusResults(ctx context.Context, query *bigquery.Query) (map[string]bq.TestStatus, []error) {
func FetchTestStatusResults(ctx context.Context, query *bigquery.Query, exclusiveTestNames []string) (map[string]bq.TestStatus, []error) {
errs := []error{}
status := map[string]bq.TestStatus{}

Expand Down Expand Up @@ -627,6 +702,7 @@ func FetchTestStatusResults(ctx context.Context, query *bigquery.Query) (map[str

status[testIDStr] = testStatus
}

return status, errs
}

Expand Down
Loading