Skip to content

Commit

Permalink
fix check/SLO for Kubernetes conformance and CLI refs
Browse files Browse the repository at this point in the history
  • Loading branch information
@mtulio
mtulio committed Aug 12, 2024
1 parent 6cb02d8 commit 17b4784
Show file tree
Hide file tree
Showing 3 changed files with 114 additions and 174 deletions.
63 changes: 14 additions & 49 deletions internal/report/data.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -162,32 +162,31 @@ type ReportPlugin struct {
Tests map[string]*plugin.TestItem `json:"tests,omitempty"`

// Filters
TagsFailedPrio string `json:"tagsFailuresPriority"`
TestsFailedPrio []*ReportTestFailure `json:"testsFailuresPriority"`
TagsFlakeCI string `json:"tagsFlakeCI"`
TestsFlakeCI []*ReportTestFailure `json:"testsFlakeCI"`

// SuiteOnly
FailedFilter1 []*ReportTestFailure `json:"failedTestsFilter1"`
TagsFilter1 string `json:"tagsFailuresFilter1"`

// Baseline
// Filter: BaselineArchive
FailedFilter2 []*ReportTestFailure `json:"failedTestsFilter2"`
TagsFilter2 string `json:"tagsFailuresFilter2"`

// FlakeAPI
// Filter: FlakeAPI
FailedFilter3 []*ReportTestFailure `json:"failedTestsFilter3"`
TagsFilter3 string `json:"tagsFailuresFilter3"`

// Filter: BaselineAPI
FailedFilter4 []*ReportTestFailure `json:"failedTestsFilter4"`
TagsFilter4 string `json:"tagsFailuresFilter4"`

// Filter: KnownFailures
FailedFilter5 []*ReportTestFailure `json:"failedTestsFilter5"`
TagsFilter5 string `json:"tagsFailuresFilter5"`

// Filter: Replay
FailedFilter6 []*ReportTestFailure `json:"failedTestsFilter6"`
TagsFilter6 string `json:"tagsFailuresFilter6"`

// Final results after filters
FailedFiltered []*ReportTestFailure `json:"failedFiltered"`
TagsFiltered string `json:"tagsFailuresFiltered"`
}
Expand Down Expand Up @@ -229,6 +228,9 @@ func (rp *ReportPlugin) BuildFailedData(filterID string, dataFailures []string)
case "F4":
rp.FailedFilter4 = failures
rp.TagsFilter4 = tags.ShowSorted()
case "F5":
rp.FailedFilter5 = failures
rp.TagsFilter5 = tags.ShowSorted()
case "F6":
rp.FailedFilter6 = failures
rp.TagsFilter6 = tags.ShowSorted()
Expand All @@ -250,7 +252,7 @@ type ReportPluginStat struct {
FilterSuite int64 `json:"filter1Suite"`
Filter1Excluded int64 `json:"filter1Excluded"`

// Filter: Baseline (deprecated soon)
// Filter: BaselineArchive (deprecated soon)
FilterBaseline int64 `json:"filter2Baseline"`
Filter2Excluded int64 `json:"filter2Excluded"`

Expand Down Expand Up @@ -660,47 +662,7 @@ func (re *ReportData) populatePluginConformance(rs *summary.ResultSummary, reRes
}
}

// TODO move this filter to a dedicated function
noFlakes := make(map[string]struct{})
testTagsFailedPrio := plugin.NewTestTagsEmpty(len(pluginSum.FailedFiltered))
for _, test := range pluginSum.FailedFiltered {
noFlakes[test] = struct{}{}
testTagsFailedPrio.Add(&test)
testData := &ReportTestFailure{
Name: test,
ID: pluginSum.Tests[test].ID,
Documentation: pluginSum.Tests[test].Documentation,
}
if _, ok := pluginSum.Tests[test].ErrorCounters["total"]; ok {
testData.ErrorsCount = int64(pluginSum.Tests[test].ErrorCounters["total"])
}
reResult.Plugins[pluginID].FailedFiltered = append(reResult.Plugins[pluginID].FailedFiltered, testData)
}
reResult.Plugins[pluginID].TagsFailedPrio = testTagsFailedPrio.ShowSorted()
reResult.Plugins[pluginID].FailedFiltered = sortReportTestFailure(reResult.Plugins[pluginID].FailedFiltered)

flakes := reResult.Plugins[pluginID].TestsFlakeCI
testTagsFlakeCI := plugin.NewTestTagsEmpty(len(pluginSum.FailedFilter2))
for _, test := range pluginSum.FailedFilter2 {
if _, ok := noFlakes[test]; ok {
continue
}
testData := &ReportTestFailure{Name: test, ID: pluginSum.Tests[test].ID}
if pluginSum.Tests[test].Flake != nil {
testData.FlakeCount = pluginSum.Tests[test].Flake.CurrentFlakes
testData.FlakePerc = pluginSum.Tests[test].Flake.CurrentFlakePerc
}
testTagsFlakeCI.Add(&test)
if _, ok := pluginSum.Tests[test].ErrorCounters["total"]; ok {
testData.ErrorsCount = int64(pluginSum.Tests[test].ErrorCounters["total"])
}
flakes = append(flakes, testData)
}
reResult.Plugins[pluginID].TestsFlakeCI = sortReportTestFailure(flakes)
reResult.Plugins[pluginID].TagsFlakeCI = testTagsFlakeCI.ShowSorted()

// Final failures
// testTagsFilteredFailures := plugin.NewTestTagsEmpty(len(pluginSum.FailedFiltered))
// Filter failures
// Final filters (results/priority)
reResult.Plugins[pluginID].BuildFailedData("final", pluginSum.FailedFiltered)

Expand All @@ -713,6 +675,9 @@ func (re *ReportData) populatePluginConformance(rs *summary.ResultSummary, reRes
// Filter Replay
reResult.Plugins[pluginID].BuildFailedData("F6", pluginSum.FailedExcludedFilter6)

// Filter KnownFailures
reResult.Plugins[pluginID].BuildFailedData("F5", pluginSum.FailedExcludedFilter5)

// Filter SuiteOnly
reResult.Plugins[pluginID].BuildFailedData("F1", pluginSum.FailedExcludedFilter1)

Expand Down
14 changes: 7 additions & 7 deletions internal/report/slo.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,10 +27,10 @@ const (
CheckIdEmptyValue string = "--"

// SLOs
CheckID001 string = "OPCT-001"
CheckID004 string = "OPCT-004"
CheckID005 string = "OPCT-005"
CheckID022 string = "OPCT-022"
CheckID001 string = "OPCT-001"
CheckID004 string = "OPCT-004"
CheckID005 string = "OPCT-005"
CheckID022 string = "OPCT-022"
CheckID023A string = "OPCT-023A"
CheckID023B string = "OPCT-023B"
)
Expand Down Expand Up @@ -183,9 +183,9 @@ func NewCheckSummary(re *ReportData) *CheckSummary {
log.Debugf("%s Runtime: Total and Failed counters are equals indicating execution failure", prefix)
return res
}
res.Actual = fmt.Sprintf("Priority==%d", len(p.TestsFailedPrio))
if len(p.TestsFailedPrio) > 0 {
log.Debugf("%s Acceptance criteria: TestsFailedPrio counter are greater than 0: %v", prefix, len(p.TestsFailedPrio))
res.Actual = fmt.Sprintf("Priority==%d", len(p.FailedFiltered))
if len(p.FailedFiltered) > 0 {
log.Debugf("%s Acceptance criteria: FailedFiltered counter is greater than 0: %v", prefix, len(p.FailedFiltered))
return res
}
res.Name = CheckResultNamePass
Expand Down
211 changes: 93 additions & 118 deletions pkg/cmd/report/report.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -605,144 +605,119 @@ func showSummaryPlugin(re *report.ReportResult, pluginName string, bProcessed bo
func showErrorDetails(re *report.ReportData, verbose bool) error {
fmt.Printf("\n==> Result details by conformance plugins: \n")

bProcessed := re.Provider.HasValidBaseline
showErrorDetailPlugin(re.Provider.Plugins[plugin.PluginNameKubernetesConformance], verbose, bProcessed)
showErrorDetailPlugin(re.Provider.Plugins[plugin.PluginNameOpenShiftConformance], verbose, bProcessed)
showErrorDetailPlugin(re.Provider.Plugins[plugin.PluginNameKubernetesConformance], verbose)
showErrorDetailPlugin(re.Provider.Plugins[plugin.PluginNameOpenShiftConformance], verbose)

return nil
}

// showErrorDetailPlugin Show failed e2e tests by filter, when verbose each filter will be shown.
func showErrorDetailPlugin(p *report.ReportPlugin, verbose bool, bProcessed bool) {
flakeCount := p.Stat.FilterBaseline - p.Stat.FilterFailedPrio
func showErrorDetailPlugin(p *report.ReportPlugin, verbose bool) {
if p == nil {
errlog.LogError(errors.New("unable to get plugin"))
return
}
fmt.Printf("==> %s - test failures:\n", p.Name)

// TODO(mtulio): migrate to new table format (go-table)
if verbose {
fmt.Printf("\n\n => %s: (%d failures, %d failures filtered, %d flakes)\n", p.Name, p.Stat.Failed, p.Stat.FilterBaseline, flakeCount)
fmt.Printf("\n --> [verbose] Failed tests detected on archive (without filters):\n")
if p.Stat.Failed == 0 {
fmt.Println("<empty>")
}
for _, test := range p.Tests {
if test.State == "failed" {
fmt.Println(test.Name)
}
}
// Plugin failures table - setup
st := table.StyleLight
st.Options.SeparateRows = true

fmt.Printf("\n --> [verbose] Failed tests detected on suite (Filter SuiteOnly):\n")
if p.Stat.FilterSuite == 0 {
fmt.Println("<empty>")
}
for _, test := range p.Tests {
if test.State == "filterSuiteOnly" {
fmt.Println(test.Name)
}
}
if bProcessed {
fmt.Printf("\n --> [verbose] Failed tests removing baseline (Filter Baseline):\n")
if p.Stat.FilterBaseline == 0 {
fmt.Println("<empty>")
}
for _, test := range p.Tests {
if test.State == "filterBaseline" {
fmt.Println(test.Name)
}
}
}
} else {
if p.Stat.FilterFailures == 0 && flakeCount == 0 {
log.Infof("No failures detected on %s", p.Name)
return
}
fmt.Printf("\n\n => %s: (%d failures, %d flakes)\n", p.Name, p.Stat.FilterFailures, flakeCount)
// Build table utility
defaultHeaderRow := table.Row{"#Err", "#Flake", "%Flake", "State", "Test Name"}
buildTable := func(title string, header table.Row) table.Writer {
tb := table.NewWriter()
tb.SetOutputMirror(os.Stdout)
tb.SetStyle(st)
tb.SetTitle(title)
tb.AppendHeader(header)
tb.SetColumnConfigs([]table.ColumnConfig{
{Number: 5, AlignHeader: tabletext.AlignCenter, WidthMax: 117},
})
return tb
}

// tables with go-table
rowsFail := []table.Row{}
tbFailTags := ""
tbFailSkip := false
noFlakes := make(map[string]struct{})
if p.Stat.FilterBaseline == flakeCount {
tbFailSkip = true
} else {
testTags := plugin.NewTestTagsEmpty(int(p.Stat.FilterFailures))
for _, test := range p.FailedFiltered {
noFlakes[test.Name] = struct{}{}
testTags.Add(&test.Name)
populateTable := func(tb table.Writer, failures []*report.ReportTestFailure, skipFlake bool) {
for _, failure := range failures {
test, ok := p.Tests[failure.Name]
if !ok {
errlog.LogError(errors.New(fmt.Sprintf("unable to get test %s", failure.Name)))
continue
}
errCount := 0
if _, ok := p.Tests[test.Name].ErrorCounters["total"]; ok {
errCount = p.Tests[test.Name].ErrorCounters["total"]
if _, ok := test.ErrorCounters["total"]; ok {
errCount = test.ErrorCounters["total"]
}
// testsWErrCnt = append(testsWErrCnt, fmt.Sprintf("%d\t%s", errCount, test.Name))
rowsFail = append(rowsFail, table.Row{errCount, test.Name})
}
// Failed tests grouped by tag (first value between '[]')
tbFailTags = testTags.ShowSorted()
}

rowsFlake := []table.Row{}
tbFlakeTags := ""
tbFlakeSkip := false
if p.Stat.FilterBaseline == 0 {
tbFlakeSkip = true
} else {
testTags := plugin.NewTestTagsEmpty(int(p.Stat.FilterBaseline))
for _, test := range p.TestsFlakeCI {
// preventing duplication when flake tests was already listed.
if _, ok := noFlakes[test.Name]; ok {
if skipFlake {
tb.AppendRow(table.Row{errCount, test.Name})
continue
}
// TODO: fix issues when retrieving flakes from Sippy API.
// Fallback to '--' when has issues.
if p.Tests[test.Name].Flake == nil {
rowsFlake = append(rowsFlake, table.Row{"--", "--", "--", test.Name})
} else if p.Tests[test.Name].Flake.CurrentFlakes != 0 {
errCount := 0
if _, ok := p.Tests[test.Name].ErrorCounters["total"]; ok {
errCount = p.Tests[test.Name].ErrorCounters["total"]
}
rowsFlake = append(rowsFlake, table.Row{
p.Tests[test.Name].Flake.CurrentFlakes,
fmt.Sprintf("%.3f %%", p.Tests[test.Name].Flake.CurrentFlakePerc),
errCount, test.Name})
if test.Flake == nil {
tb.AppendRow(table.Row{errCount, "--", "--", test.State, test.Name})
continue
}
testTags.Add(&test.Name)
tb.AppendRow(table.Row{errCount, test.Flake.CurrentFlakes, fmt.Sprintf("%.2f", test.Flake.CurrentFlakePerc), test.State, test.Name})
}
tbFlakeTags = testTags.ShowSorted()
tb.Render()
}

// Table style
st := table.StyleLight
st.Options.SeparateRows = true
// Table for Priority
if len(p.FailedFiltered) > 0 {
tb := table.NewWriter()
tb.SetOutputMirror(os.Stdout)
tb.SetStyle(st)
tb.AppendHeader(table.Row{"#Err", "Test Name"})
tb.AppendFooter(table.Row{"", p.TagsFiltered})
tb.SetColumnConfigs([]table.ColumnConfig{
{Number: 2, AlignHeader: tabletext.AlignCenter, WidthMax: 150},
})
title := fmt.Sprintf("==> %s \n%s ACTION REQUIRED: Failed tests to review", p.Name, iconsCollor["alert"])
tb.SetTitle(title)
populateTable(tb, p.FailedFiltered, true)
}

// Table for Flakes
tbFail := table.NewWriter()
tbFail.SetOutputMirror(os.Stdout)
tbFail.SetStyle(st)
tbFail.SetTitle("==> %s \n%s ACTION REQUIRED: Failed tests to review", p.Name, iconsCollor["alert"])
tbFail.AppendHeader(table.Row{"Err Log", "Test Name"})
tbFail.AppendRows(rowsFail)
tbFail.AppendFooter(table.Row{"", tbFailTags})
tbFail.SetColumnConfigs([]table.ColumnConfig{
{Number: 2, AlignHeader: tabletext.AlignCenter, WidthMax: 150},
})
if !tbFailSkip {
tbFail.Render()
if len(p.FailedFilter3) > 0 {
filterName := "FlakeAPI"
title := fmt.Sprintf("\n==> %s\n Failed tests excluded in %q filter (%d)", p.Name, filterName, len(p.FailedFilter3))
tb := buildTable(title, defaultHeaderRow)
populateTable(tb, p.FailedFilter3, false)
}

// Table for Flakes
tbFlake := table.NewWriter()
tbFlake.SetOutputMirror(os.Stdout)
tbFlake.SetStyle(st)
tbFlake.SetTitle("==> %s \nFailed tests with flake occurrences (on OpenShift CI)", p.Name)
tbFlake.AppendHeader(table.Row{"Flake #", "%", "Err Log", "Test Name"})
tbFlake.AppendRows(rowsFlake)
tbFlake.AppendFooter(table.Row{"", "", "", tbFlakeTags})
tbFlake.SetColumnConfigs([]table.ColumnConfig{
{Number: 4, AlignHeader: tabletext.AlignCenter, WidthMax: 129},
})
if !tbFlakeSkip {
tbFlake.Render()
if verbose {
fmt.Printf("\n Show failures by filter (verbose mode): %v\n", verbose)
skipFlake := false
titlePrefix := fmt.Sprintf("==> %s\n Failed tests excluded in", p.Name)
// Show tests removed in filter: Replay
if len(p.FailedFilter6) > 0 {
filterName := "Replay"
title := fmt.Sprintf("%s %q filter (%d)", titlePrefix, filterName, len(p.FailedFilter6))
tb := buildTable(title, defaultHeaderRow)
populateTable(tb, p.FailedFilter6, skipFlake)
}

// Show tests removed in filter: KnownFailures
if len(p.FailedFilter5) > 0 {
filterName := "KnownFailures"
title := fmt.Sprintf("%s %q filter (%d)", titlePrefix, filterName, len(p.FailedFilter5))
tb := buildTable(title, defaultHeaderRow)
populateTable(tb, p.FailedFilter5, skipFlake)
}

// Show tests removed in filter: BaselineArchive
if len(p.FailedFilter2) > 0 {
filterName := "BaselineArchive"
title := fmt.Sprintf("%s %q filter (%d)", titlePrefix, filterName, len(p.FailedFilter2))
tb := buildTable(title, defaultHeaderRow)
populateTable(tb, p.FailedFilter2, skipFlake)
}

// Show tests removed in filter: SuiteOnly
if len(p.FailedFilter1) > 0 {
filterName := "SuiteOnly"
title := fmt.Sprintf("%s %q filter (%d)", titlePrefix, filterName, len(p.FailedFilter1))
tb := buildTable(title, defaultHeaderRow)
populateTable(tb, p.FailedFilter1, skipFlake)
}
}
}

Expand Down

0 comments on commit 17b4784

Please sign in to comment.