report.py

最後變更在這個檔案是 106061,由 vboxsync 提交於 6 月前
Copyright year updates by scm.
屬性 svn:eol-style 設為 `native` 屬性 svn:keywords 設為 `Author Date Id Revision`
檔案大小: 59.3 KB

行
1	# -- coding: utf-8 --
2	# $Id: report.py 106061 2024-09-16 14:03:52Z vboxsync $
3
4	"""
5	Test Manager - Report models.
6	"""
7
8	__copyright__ = \
9	"""
10	Copyright (C) 2012-2024 Oracle and/or its affiliates.
11
12	This file is part of VirtualBox base platform packages, as
13	available from https://www.alldomusa.eu.org.
14
15	This program is free software; you can redistribute it and/or
16	modify it under the terms of the GNU General Public License
17	as published by the Free Software Foundation, in version 3 of the
18	License.
19
20	This program is distributed in the hope that it will be useful, but
21	WITHOUT ANY WARRANTY; without even the implied warranty of
22	MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
23	General Public License for more details.
24
25	You should have received a copy of the GNU General Public License
26	along with this program; if not, see <https://www.gnu.org/licenses>.
27
28	The contents of this file may alternatively be used under the terms
29	of the Common Development and Distribution License Version 1.0
30	(CDDL), a copy of it is provided in the "COPYING.CDDL" file included
31	in the VirtualBox distribution, in which case the provisions of the
32	CDDL are applicable instead of those of the GPL.
33
34	You may elect to license modified versions of this file under the
35	terms and conditions of either the GPL or the CDDL or both.
36
37	SPDX-License-Identifier: GPL-3.0-only OR CDDL-1.0
38	"""
39	__version__ = "$Revision: 106061 $"
40
41
42	# Standard Python imports.
43	import sys;
44
45	# Validation Kit imports.
46	from testmanager.core.base import ModelLogicBase, TMExceptionBase;
47	from testmanager.core.build import BuildCategoryData;
48	from testmanager.core.dbobjcache import DatabaseObjCache;
49	from testmanager.core.failurereason import FailureReasonLogic;
50	from testmanager.core.testbox import TestBoxLogic, TestBoxData;
51	from testmanager.core.testcase import TestCaseLogic;
52	from testmanager.core.testcaseargs import TestCaseArgsLogic;
53	from testmanager.core.testresults import TestResultLogic, TestResultFilter;
54	from common import constants;
55
56	# Python 3 hacks:
57	if sys.version_info[0] >= 3:
58	xrange = range; # pylint: disable=redefined-builtin,invalid-name
59
60
61
62	class ReportFilter(TestResultFilter):
63	"""
64	Same as TestResultFilter for now.
65	"""
66
67	def __init__(self):
68	TestResultFilter.__init__(self);
69
70
71
72	class ReportModelBase(ModelLogicBase): # pylint: disable=too-few-public-methods
73	"""
74	Something all report logic(/miner) classes inherit from.
75	"""
76
77	## @name Report subjects
78	## @{
79	ksSubEverything = 'Everything';
80	ksSubSchedGroup = 'SchedGroup';
81	ksSubTestGroup = 'TestGroup';
82	ksSubTestCase = 'TestCase';
83	ksSubTestCaseArgs = 'TestCaseArgs';
84	ksSubTestBox = 'TestBox';
85	ksSubBuild = 'Build';
86	## @}
87	kasSubjects = [ ksSubEverything, ksSubSchedGroup, ksSubTestGroup, ksSubTestCase, ksSubTestBox, ksSubBuild, ];
88
89
90	## @name TestStatus_T
91	# @{
92	ksTestStatus_Running = 'running';
93	ksTestStatus_Success = 'success';
94	ksTestStatus_Skipped = 'skipped';
95	ksTestStatus_BadTestBox = 'bad-testbox';
96	ksTestStatus_Aborted = 'aborted';
97	ksTestStatus_Failure = 'failure';
98	ksTestStatus_TimedOut = 'timed-out';
99	ksTestStatus_Rebooted = 'rebooted';
100	## @}
101
102
103	def __init__(self, oDb, tsNow, cPeriods, cHoursPerPeriod, sSubject, aidSubjects, oFilter):
104	ModelLogicBase.__init__(self, oDb);
105	# Public so the report generator can easily access them.
106	self.tsNow = tsNow; # (Can be None.)
107	self.__tsNowDateTime = None;
108	self.cPeriods = cPeriods;
109	self.cHoursPerPeriod = cHoursPerPeriod;
110	self.sSubject = sSubject;
111	self.aidSubjects = aidSubjects;
112	self.oFilter = oFilter;
113	if self.oFilter is None:
114	class DummyFilter(object):
115	""" Dummy """
116	def getTableJoins(self, sExtraIndent = '', iOmit = -1, dOmitTables = None):
117	""" Dummy """
118	_ = sExtraIndent; _ = iOmit; _ = dOmitTables; # pylint: disable=redefined-variable-type
119	return '';
120	def getWhereConditions(self, sExtraIndent = '', iOmit = -1):
121	""" Dummy """
122	_ = sExtraIndent; _ = iOmit; # pylint: disable=redefined-variable-type
123	return '';
124	def isJoiningWithTable(self, sTable):
125	""" Dummy """;
126	_ = sTable;
127	return False;
128	self.oFilter = DummyFilter();
129
130	def getExtraSubjectTables(self):
131	"""
132	Returns a list of additional tables needed by the subject.
133	"""
134	return [];
135
136	def getExtraSubjectWhereExpr(self):
137	"""
138	Returns additional WHERE expression relating to the report subject. It starts
139	with an AND so that it can simply be appended to the WHERE clause.
140	"""
141	if self.sSubject == self.ksSubEverything:
142	return '';
143
144	if self.sSubject == self.ksSubSchedGroup:
145	sWhere = ' AND TestSets.idSchedGroup';
146	elif self.sSubject == self.ksSubTestGroup:
147	sWhere = ' AND TestSets.idTestGroup';
148	elif self.sSubject == self.ksSubTestCase:
149	sWhere = ' AND TestSets.idTestCase';
150	elif self.sSubject == self.ksSubTestCaseArgs:
151	sWhere = ' AND TestSets.idTestCaseArgs';
152	elif self.sSubject == self.ksSubTestBox:
153	sWhere = ' AND TestSets.idTestBox';
154	elif self.sSubject == self.ksSubBuild:
155	sWhere = ' AND TestSets.idBuild';
156	else:
157	raise TMExceptionBase(self.sSubject);
158
159	if len(self.aidSubjects) == 1:
160	sWhere += self._oDb.formatBindArgs(' = %s\n', (self.aidSubjects[0],));
161	else:
162	assert self.aidSubjects;
163	sWhere += self._oDb.formatBindArgs(' IN (%s', (self.aidSubjects[0],));
164	for i in range(1, len(self.aidSubjects)):
165	sWhere += self._oDb.formatBindArgs(', %s', (self.aidSubjects[i],));
166	sWhere += ')\n';
167
168	return sWhere;
169
170	def getNowAsDateTime(self):
171	""" Returns a datetime instance corresponding to tsNow. """
172	if self.__tsNowDateTime is None:
173	if self.tsNow is None:
174	self.__tsNowDateTime = self._oDb.getCurrentTimestamp();
175	else:
176	self._oDb.execute('SELECT %s::TIMESTAMP WITH TIME ZONE', (self.tsNow,));
177	self.__tsNowDateTime = self._oDb.fetchOne()[0];
178	return self.__tsNowDateTime;
179
180	def getPeriodStart(self, iPeriod):
181	""" Gets the python timestamp for the start of the given period. """
182	from datetime import timedelta;
183	cHoursStart = (self.cPeriods - iPeriod ) * self.cHoursPerPeriod;
184	return self.getNowAsDateTime() - timedelta(hours = cHoursStart);
185
186	def getPeriodEnd(self, iPeriod):
187	""" Gets the python timestamp for the end of the given period. """
188	from datetime import timedelta;
189	cHoursEnd = (self.cPeriods - iPeriod - 1) * self.cHoursPerPeriod;
190	return self.getNowAsDateTime() - timedelta(hours = cHoursEnd);
191
192	def getExtraWhereExprForPeriod(self, iPeriod):
193	"""
194	Returns additional WHERE expression for getting test sets for the
195	specified period. It starts with an AND so that it can simply be
196	appended to the WHERE clause.
197	"""
198	if self.tsNow is None:
199	sNow = 'CURRENT_TIMESTAMP';
200	else:
201	sNow = self._oDb.formatBindArgs('%s::TIMESTAMP', (self.tsNow,));
202
203	cHoursStart = (self.cPeriods - iPeriod ) * self.cHoursPerPeriod;
204	cHoursEnd = (self.cPeriods - iPeriod - 1) * self.cHoursPerPeriod;
205	if cHoursEnd == 0:
206	return ' AND TestSets.tsDone >= (%s - interval \'%u hours\')\n' \
207	' AND TestSets.tsDone < %s\n' \
208	% (sNow, cHoursStart, sNow);
209	return ' AND TestSets.tsDone >= (%s - interval \'%u hours\')\n' \
210	' AND TestSets.tsDone < (%s - interval \'%u hours\')\n' \
211	% (sNow, cHoursStart, sNow, cHoursEnd);
212
213	def getPeriodDesc(self, iPeriod):
214	"""
215	Returns the period description, usually for graph data.
216	"""
217	if iPeriod == 0:
218	return 'now' if self.tsNow is None else 'then';
219	sTerm = 'ago' if self.tsNow is None else 'earlier';
220	if self.cHoursPerPeriod == 24:
221	return '%dd %s' % (iPeriod, sTerm, );
222	if (iPeriod * self.cHoursPerPeriod) % 24 == 0:
223	return '%dd %s' % (iPeriod * self.cHoursPerPeriod / 24, sTerm, );
224	return '%dh %s' % (iPeriod * self.cHoursPerPeriod, sTerm);
225
226	def getStraightPeriodDesc(self, iPeriod):
227	"""
228	Returns the period description, usually for graph data.
229	"""
230	iWickedPeriod = self.cPeriods - iPeriod - 1;
231	return self.getPeriodDesc(iWickedPeriod);
232
233
234	#
235	# Data structures produced and returned by the ReportLazyModel.
236	#
237
238	class ReportTransientBase(object):
239	""" Details on the test where a problem was first/last seen. """
240	def __init__(self, idBuild, iRevision, sRepository, idTestSet, idTestResult, tsDone, # pylint: disable=too-many-arguments
241	iPeriod, fEnter, idSubject, oSubject):
242	self.idBuild = idBuild; # Build ID.
243	self.iRevision = iRevision; # SVN revision for build.
244	self.sRepository = sRepository; # SVN repository for build.
245	self.idTestSet = idTestSet; # Test set.
246	self.idTestResult = idTestResult; # Test result.
247	self.tsDone = tsDone; # When the test set was done.
248	self.iPeriod = iPeriod; # Data set period.
249	self.fEnter = fEnter; # True if enter event, False if leave event.
250	self.idSubject = idSubject;
251	self.oSubject = oSubject;
252
253	class ReportFailureReasonTransient(ReportTransientBase):
254	""" Details on the test where a failure reason was first/last seen. """
255	def __init__(self, idBuild, iRevision, sRepository, idTestSet, idTestResult, tsDone, # pylint: disable=too-many-arguments
256	iPeriod, fEnter, oReason):
257	ReportTransientBase.__init__(self, idBuild, iRevision, sRepository, idTestSet, idTestResult, tsDone, iPeriod, fEnter,
258	oReason.idFailureReason, oReason);
259	self.oReason = oReason; # FailureReasonDataEx
260
261
262	class ReportHitRowBase(object):
263	""" A row in a period. """
264	def __init__(self, idSubject, oSubject, cHits, tsMin = None, tsMax = None):
265	self.idSubject = idSubject;
266	self.oSubject = oSubject;
267	self.cHits = cHits;
268	self.tsMin = tsMin;
269	self.tsMax = tsMax;
270
271	class ReportHitRowWithTotalBase(ReportHitRowBase):
272	""" A row in a period. """
273	def __init__(self, idSubject, oSubject, cHits, cTotal, tsMin = None, tsMax = None):
274	ReportHitRowBase.__init__(self, idSubject, oSubject, cHits, tsMin, tsMax)
275	self.cTotal = cTotal;
276	self.uPct = cHits * 100 / cTotal;
277
278	class ReportFailureReasonRow(ReportHitRowBase):
279	""" The account of one failure reason for a period. """
280	def __init__(self, aoRow, oReason):
281	ReportHitRowBase.__init__(self, aoRow[0], oReason, aoRow[1], aoRow[2], aoRow[3]);
282	self.idFailureReason = aoRow[0];
283	self.oReason = oReason; # FailureReasonDataEx
284
285
286	class ReportPeriodBase(object):
287	""" A period in ReportFailureReasonSet. """
288	def __init__(self, oSet, iPeriod, sDesc, tsFrom, tsTo):
289	self.oSet = oSet # Reference to the parent ReportSetBase derived object.
290	self.iPeriod = iPeriod; # Period number in the set.
291	self.sDesc = sDesc; # Short period description.
292	self.tsStart = tsFrom; # Start of the period.
293	self.tsEnd = tsTo; # End of the period (exclusive).
294	self.tsMin = tsTo; # The earlierst hit of the period (only valid for cHits > 0).
295	self.tsMax = tsFrom; # The latest hit of the period (only valid for cHits > 0).
296	self.aoRows = []; # Rows in order the database returned them (ReportHitRowBase descendant).
297	self.dRowsById = {}; # Same as aoRows but indexed by object ID (see ReportSetBase::sIdAttr).
298	self.dFirst = {}; # The subjects seen for the first time - data object, keyed by ID.
299	self.dLast = {}; # The subjects seen for the last time - data object, keyed by ID.
300	self.cHits = 0; # Total number of hits in this period.
301	self.cMaxHits = 0; # Max hits in a row.
302	self.cMinHits = 99999999; # Min hits in a row (only valid for cHits > 0).
303
304	def appendRow(self, oRow, idRow, oData):
305	""" Adds a row. """
306	assert isinstance(oRow, ReportHitRowBase);
307	self.aoRows.append(oRow);
308	self.dRowsById[idRow] = oRow;
309	if idRow not in self.oSet.dSubjects:
310	self.oSet.dSubjects[idRow] = oData;
311	self._doStatsForRow(oRow, idRow, oData);
312
313	def _doStatsForRow(self, oRow, idRow, oData):
314	""" Does the statistics for a row. Helper for appendRow as well as helpRecalcStats. """
315	if oRow.tsMin is not None and oRow.tsMin < self.tsMin:
316	self.tsMin = oRow.tsMin;
317	if oRow.tsMax is not None and oRow.tsMax < self.tsMax:
318	self.tsMax = oRow.tsMax;
319
320	self.cHits += oRow.cHits;
321	if oRow.cHits > self.cMaxHits:
322	self.cMaxHits = oRow.cHits;
323	if oRow.cHits < self.cMinHits:
324	self.cMinHits = oRow.cHits;
325
326	if idRow in self.oSet.dcHitsPerId:
327	self.oSet.dcHitsPerId[idRow] += oRow.cHits;
328	else:
329	self.oSet.dcHitsPerId[idRow] = oRow.cHits;
330
331	if oRow.cHits > 0:
332	if idRow not in self.oSet.diPeriodFirst:
333	self.dFirst[idRow] = oData;
334	self.oSet.diPeriodFirst[idRow] = self.iPeriod;
335	self.oSet.diPeriodLast[idRow] = self.iPeriod;
336
337	def helperSetRecalcStats(self):
338	""" Recalc the statistics (do resetStats first on set). """
339	for idRow, oRow in self.dRowsById.items():
340	self._doStatsForRow(oRow, idRow, self.oSet.dSubjects[idRow]);
341
342	def helperSetResetStats(self):
343	""" Resets the statistics. """
344	self.tsMin = self.tsEnd;
345	self.tsMax = self.tsStart;
346	self.cHits = 0;
347	self.cMaxHits = 0;
348	self.cMinHits = 99999999;
349	self.dFirst = {};
350	self.dLast = {};
351
352	def helperSetDeleteKeyFromSet(self, idKey):
353	""" Helper for ReportPeriodSetBase::deleteKey """
354	if idKey in self.dRowsById:
355	oRow = self.dRowsById[idKey];
356	self.aoRows.remove(oRow);
357	del self.dRowsById[idKey]
358	self.cHits -= oRow.cHits;
359	if idKey in self.dFirst:
360	del self.dFirst[idKey];
361	if idKey in self.dLast:
362	del self.dLast[idKey];
363
364	class ReportPeriodWithTotalBase(ReportPeriodBase):
365	""" In addition to the cHits, we also have a total to relate it too. """
366	def __init__(self, oSet, iPeriod, sDesc, tsFrom, tsTo):
367	ReportPeriodBase.__init__(self, oSet, iPeriod, sDesc, tsFrom, tsTo);
368	self.cTotal = 0;
369	self.cMaxTotal = 0;
370	self.cMinTotal = 99999999;
371	self.uMaxPct = 0; # Max percentage in a row (100 = 100%).
372
373	def _doStatsForRow(self, oRow, idRow, oData):
374	assert isinstance(oRow, ReportHitRowWithTotalBase);
375	super(ReportPeriodWithTotalBase, self)._doStatsForRow(oRow, idRow, oData);
376	self.cTotal += oRow.cTotal;
377	if oRow.cTotal > self.cMaxTotal:
378	self.cMaxTotal = oRow.cTotal;
379	if oRow.cTotal < self.cMinTotal:
380	self.cMinTotal = oRow.cTotal;
381
382	if oRow.uPct > self.uMaxPct:
383	self.uMaxPct = oRow.uPct;
384
385	if idRow in self.oSet.dcTotalPerId:
386	self.oSet.dcTotalPerId[idRow] += oRow.cTotal;
387	else:
388	self.oSet.dcTotalPerId[idRow] = oRow.cTotal;
389
390	def helperSetResetStats(self):
391	super(ReportPeriodWithTotalBase, self).helperSetResetStats();
392	self.cTotal = 0;
393	self.cMaxTotal = 0;
394	self.cMinTotal = 99999999;
395	self.uMaxPct = 0;
396
397	class ReportFailureReasonPeriod(ReportPeriodBase):
398	""" A period in ReportFailureReasonSet. """
399	def __init__(self, oSet, iPeriod, sDesc, tsFrom, tsTo):
400	ReportPeriodBase.__init__(self, oSet, iPeriod, sDesc, tsFrom, tsTo);
401	self.cWithoutReason = 0; # Number of failed test sets without any assigned reason.
402
403
404
405	class ReportPeriodSetBase(object):
406	""" Period data set base class. """
407	def __init__(self, sIdAttr):
408	self.sIdAttr = sIdAttr; # The name of the key attribute. Mainly for documentation purposes.
409	self.aoPeriods = []; # Periods (ReportPeriodBase descendant) in ascending order (time wise).
410	self.dSubjects = {}; # The subject data objects, keyed by the subject ID.
411	self.dcHitsPerId = {}; # Sum hits per subject ID (key).
412	self.cHits = 0; # Sum number of hits in all periods and all reasons.
413	self.cMaxHits = 0; # Max hits in a row.
414	self.cMinHits = 99999999; # Min hits in a row.
415	self.cMaxRows = 0; # Max number of rows in a period.
416	self.cMinRows = 99999999; # Min number of rows in a period.
417	self.diPeriodFirst = {}; # The period number a reason was first seen (keyed by subject ID).
418	self.diPeriodLast = {}; # The period number a reason was last seen (keyed by subject ID).
419	self.aoEnterInfo = []; # Array of ReportTransientBase children order by iRevision. Excludes
420	# the first period of course. (Child class populates this.)
421	self.aoLeaveInfo = []; # Array of ReportTransientBase children order in descending order by
422	# iRevision. Excludes the last priod. (Child class populates this.)
423
424	def appendPeriod(self, oPeriod):
425	""" Appends a period to the set. """
426	assert isinstance(oPeriod, ReportPeriodBase);
427	self.aoPeriods.append(oPeriod);
428	self._doStatsForPeriod(oPeriod);
429
430	def _doStatsForPeriod(self, oPeriod):
431	""" Worker for appendPeriod and recalcStats. """
432	self.cHits += oPeriod.cHits;
433	if oPeriod.cMaxHits > self.cMaxHits:
434	self.cMaxHits = oPeriod.cMaxHits;
435	if oPeriod.cMinHits < self.cMinHits:
436	self.cMinHits = oPeriod.cMinHits;
437
438	if len(oPeriod.aoRows) > self.cMaxRows:
439	self.cMaxRows = len(oPeriod.aoRows);
440	if len(oPeriod.aoRows) < self.cMinRows:
441	self.cMinRows = len(oPeriod.aoRows);
442
443	def recalcStats(self):
444	""" Recalculates the statistics. ASSUMES finalizePass1 hasn't been done yet. """
445	self.cHits = 0;
446	self.cMaxHits = 0;
447	self.cMinHits = 99999999;
448	self.cMaxRows = 0;
449	self.cMinRows = 99999999;
450	self.diPeriodFirst = {};
451	self.diPeriodLast = {};
452	self.dcHitsPerId = {};
453	for oPeriod in self.aoPeriods:
454	oPeriod.helperSetResetStats();
455
456	for oPeriod in self.aoPeriods:
457	oPeriod.helperSetRecalcStats();
458	self._doStatsForPeriod(oPeriod);
459
460	def deleteKey(self, idKey):
461	""" Deletes a key from the set. May leave cMaxHits and cMinHits with outdated values. """
462	self.cHits -= self.dcHitsPerId[idKey];
463	del self.dcHitsPerId[idKey];
464	if idKey in self.diPeriodFirst:
465	del self.diPeriodFirst[idKey];
466	if idKey in self.diPeriodLast:
467	del self.diPeriodLast[idKey];
468	if idKey in self.aoEnterInfo:
469	del self.aoEnterInfo[idKey];
470	if idKey in self.aoLeaveInfo:
471	del self.aoLeaveInfo[idKey];
472	del self.dSubjects[idKey];
473	for oPeriod in self.aoPeriods:
474	oPeriod.helperSetDeleteKeyFromSet(idKey);
475
476	def pruneRowsWithZeroSumHits(self):
477	""" Discards rows with zero sum hits across all periods. Works around lazy selects counting both totals and hits. """
478	cDeleted = 0;
479	aidKeys = list(self.dcHitsPerId);
480	for idKey in aidKeys:
481	if self.dcHitsPerId[idKey] == 0:
482	self.deleteKey(idKey);
483	cDeleted += 1;
484	if cDeleted > 0:
485	self.recalcStats();
486	return cDeleted;
487
488	def finalizePass1(self):
489	""" Finished all but aoEnterInfo and aoLeaveInfo. """
490	# All we need to do here is to populate the dLast members.
491	for idKey, iPeriod in self.diPeriodLast.items():
492	self.aoPeriods[iPeriod].dLast[idKey] = self.dSubjects[idKey];
493	return self;
494
495	def finalizePass2(self):
496	""" Called after aoEnterInfo and aoLeaveInfo has been populated to sort them. """
497	self.aoEnterInfo = sorted(self.aoEnterInfo, key = lambda oTrans: oTrans.iRevision);
498	self.aoLeaveInfo = sorted(self.aoLeaveInfo, key = lambda oTrans: oTrans.iRevision, reverse = True);
499	return self;
500
501	class ReportPeriodSetWithTotalBase(ReportPeriodSetBase):
502	""" In addition to the cHits, we also have a total to relate it too. """
503	def __init__(self, sIdAttr):
504	ReportPeriodSetBase.__init__(self, sIdAttr);
505	self.dcTotalPerId = {}; # Sum total per subject ID (key).
506	self.cTotal = 0; # Sum number of total in all periods and all reasons.
507	self.cMaxTotal = 0; # Max total in a row.
508	self.cMinTotal = 0; # Min total in a row.
509	self.uMaxPct = 0; # Max percentage in a row (100 = 100%).
510
511	def _doStatsForPeriod(self, oPeriod):
512	assert isinstance(oPeriod, ReportPeriodWithTotalBase);
513	super(ReportPeriodSetWithTotalBase, self)._doStatsForPeriod(oPeriod);
514	self.cTotal += oPeriod.cTotal;
515	if oPeriod.cMaxTotal > self.cMaxTotal:
516	self.cMaxTotal = oPeriod.cMaxTotal;
517	if oPeriod.cMinTotal < self.cMinTotal:
518	self.cMinTotal = oPeriod.cMinTotal;
519
520	if oPeriod.uMaxPct > self.uMaxPct:
521	self.uMaxPct = oPeriod.uMaxPct;
522
523	def recalcStats(self):
524	self.dcTotalPerId = {};
525	self.cTotal = 0;
526	self.cMaxTotal = 0;
527	self.cMinTotal = 0;
528	self.uMaxPct = 0;
529	super(ReportPeriodSetWithTotalBase, self).recalcStats();
530
531	def deleteKey(self, idKey):
532	self.cTotal -= self.dcTotalPerId[idKey];
533	del self.dcTotalPerId[idKey];
534	super(ReportPeriodSetWithTotalBase, self).deleteKey(idKey);
535
536	class ReportFailureReasonSet(ReportPeriodSetBase):
537	""" What ReportLazyModel.getFailureReasons returns. """
538	def __init__(self):
539	ReportPeriodSetBase.__init__(self, 'idFailureReason');
540
541
542
543	class ReportLazyModel(ReportModelBase): # pylint: disable=too-few-public-methods
544	"""
545	The 'lazy bird' report model class.
546
547	We may want to have several classes, maybe one for each report even. But,
548	I'm thinking that's a bit overkill so we'll start with this and split it
549	if/when it becomes necessary.
550	"""
551
552	kdsStatusSimplificationMap = {
553	ReportModelBase.ksTestStatus_Running: ReportModelBase.ksTestStatus_Running,
554	ReportModelBase.ksTestStatus_Success: ReportModelBase.ksTestStatus_Success,
555	ReportModelBase.ksTestStatus_Skipped: ReportModelBase.ksTestStatus_Skipped,
556	ReportModelBase.ksTestStatus_BadTestBox: ReportModelBase.ksTestStatus_Skipped,
557	ReportModelBase.ksTestStatus_Aborted: ReportModelBase.ksTestStatus_Skipped,
558	ReportModelBase.ksTestStatus_Failure: ReportModelBase.ksTestStatus_Failure,
559	ReportModelBase.ksTestStatus_TimedOut: ReportModelBase.ksTestStatus_Failure,
560	ReportModelBase.ksTestStatus_Rebooted: ReportModelBase.ksTestStatus_Failure,
561	};
562
563	def getSuccessRates(self):
564	"""
565	Gets the success rates of the subject in the specified period.
566
567	Returns an array of data per period (0 is the oldes, self.cPeriods-1 is
568	the latest) where each entry is a status (TestStatus_T) dictionary with
569	the number of occurences of each final status (i.e. not running).
570	"""
571
572	sBaseQuery = 'SELECT TestSets.enmStatus, COUNT(TestSets.idTestSet)\n' \
573	'FROM TestSets\n' \
574	+ self.oFilter.getTableJoins();
575	for sTable in self.getExtraSubjectTables():
576	sBaseQuery = sBaseQuery[:-1] + ',\n ' + sTable + '\n';
577	sBaseQuery += 'WHERE enmStatus <> \'running\'\n' \
578	+ self.oFilter.getWhereConditions() \
579	+ self.getExtraSubjectWhereExpr();
580
581	adPeriods = [];
582	for iPeriod in xrange(self.cPeriods):
583	self._oDb.execute(sBaseQuery + self.getExtraWhereExprForPeriod(iPeriod) + 'GROUP BY enmStatus\n');
584
585	dRet = \
586	{
587	self.ksTestStatus_Skipped: 0,
588	self.ksTestStatus_Failure: 0,
589	self.ksTestStatus_Success: 0,
590	};
591
592	for aoRow in self._oDb.fetchAll():
593	sKey = self.kdsStatusSimplificationMap[aoRow[0]]
594	if sKey in dRet:
595	dRet[sKey] += aoRow[1];
596	else:
597	dRet[sKey] = aoRow[1];
598
599	assert len(dRet) == 3;
600
601	adPeriods.insert(0, dRet);
602
603	return adPeriods;
604
605
606	def getFailureReasons(self):
607	"""
608	Gets the failure reasons of the subject in the specified period.
609
610	Returns a ReportFailureReasonSet instance.
611	"""
612
613	oFailureReasonLogic = FailureReasonLogic(self._oDb);
614
615	#
616	# Create a temporary table
617	#
618	sTsNow = 'CURRENT_TIMESTAMP' if self.tsNow is None else self._oDb.formatBindArgs('%s::TIMESTAMP', (self.tsNow,));
619	sTsFirst = '(%s - interval \'%s hours\')' \
620	% (sTsNow, self.cHoursPerPeriod * self.cPeriods,);
621	sQuery = 'CREATE TEMPORARY TABLE TmpReasons ON COMMIT DROP AS\n' \
622	'SELECT TestResultFailures.idFailureReason AS idFailureReason,\n' \
623	' TestResultFailures.idTestResult AS idTestResult,\n' \
624	' TestSets.idTestSet AS idTestSet,\n' \
625	' TestSets.tsDone AS tsDone,\n' \
626	' TestSets.tsCreated AS tsCreated,\n' \
627	' TestSets.idBuild AS idBuild\n' \
628	'FROM TestResultFailures,\n' \
629	' TestResults,\n' \
630	' TestSets\n' \
631	+ self.oFilter.getTableJoins(dOmitTables = {'TestResults': True, 'TestResultFailures': True});
632	for sTable in self.getExtraSubjectTables():
633	if sTable not in [ 'TestResults', 'TestResultFailures' ] and not self.oFilter.isJoiningWithTable(sTable):
634	sQuery = sQuery[:-1] + ',\n ' + sTable + '\n';
635	sQuery += 'WHERE TestResultFailures.idTestResult = TestResults.idTestResult\n' \
636	' AND TestResultFailures.tsExpire = \'infinity\'::TIMESTAMP\n' \
637	' AND TestResultFailures.tsEffective >= ' + sTsFirst + '\n' \
638	' AND TestResults.enmStatus <> \'running\'\n' \
639	' AND TestResults.enmStatus <> \'success\'\n' \
640	' AND TestResults.tsCreated >= ' + sTsFirst + '\n' \
641	' AND TestResults.tsCreated < ' + sTsNow + '\n' \
642	' AND TestResults.idTestSet = TestSets.idTestSet\n' \
643	' AND TestSets.tsDone >= ' + sTsFirst + '\n' \
644	' AND TestSets.tsDone < ' + sTsNow + '\n' \
645	+ self.oFilter.getWhereConditions() \
646	+ self.getExtraSubjectWhereExpr();
647	self._oDb.execute(sQuery);
648	self._oDb.execute('SELECT idFailureReason FROM TmpReasons;');
649
650	#
651	# Retrieve the period results.
652	#
653	oSet = ReportFailureReasonSet();
654	for iPeriod in xrange(self.cPeriods):
655	self._oDb.execute('SELECT idFailureReason,\n'
656	' COUNT(idTestResult),\n'
657	' MIN(tsDone),\n'
658	' MAX(tsDone)\n'
659	'FROM TmpReasons\n'
660	'WHERE TRUE\n'
661	+ self.getExtraWhereExprForPeriod(iPeriod).replace('TestSets.', '') +
662	'GROUP BY idFailureReason\n');
663	aaoRows = self._oDb.fetchAll()
664
665	oPeriod = ReportFailureReasonPeriod(oSet, iPeriod, self.getStraightPeriodDesc(iPeriod),
666	self.getPeriodStart(iPeriod), self.getPeriodEnd(iPeriod));
667
668	for aoRow in aaoRows:
669	oReason = oFailureReasonLogic.cachedLookup(aoRow[0]);
670	oPeriodRow = ReportFailureReasonRow(aoRow, oReason);
671	oPeriod.appendRow(oPeriodRow, oReason.idFailureReason, oReason);
672
673	# Count how many test sets we've got without any reason associated with them.
674	self._oDb.execute('SELECT COUNT(TestSets.idTestSet)\n'
675	'FROM TestSets\n'
676	' LEFT OUTER JOIN TestResultFailures\n'
677	' ON TestSets.idTestSet = TestResultFailures.idTestSet\n'
678	' AND TestResultFailures.tsEffective = \'infinity\'::TIMESTAMP\n'
679	'WHERE TestSets.enmStatus <> \'running\'\n'
680	' AND TestSets.enmStatus <> \'success\'\n'
681	+ self.getExtraWhereExprForPeriod(iPeriod) +
682	' AND TestResultFailures.idTestSet IS NULL\n');
683	oPeriod.cWithoutReason = self._oDb.fetchOne()[0];
684
685	oSet.appendPeriod(oPeriod);
686
687
688	#
689	# For reasons entering after the first period, look up the build and
690	# test set it first occured with.
691	#
692	oSet.finalizePass1();
693
694	for iPeriod in xrange(1, self.cPeriods):
695	oPeriod = oSet.aoPeriods[iPeriod];
696	for oReason in oPeriod.dFirst.values():
697	oSet.aoEnterInfo.append(self._getEdgeFailureReasonOccurence(oReason, iPeriod, fEnter = True));
698
699	# Ditto for reasons leaving before the last.
700	for iPeriod in xrange(self.cPeriods - 1):
701	oPeriod = oSet.aoPeriods[iPeriod];
702	for oReason in oPeriod.dLast.values():
703	oSet.aoLeaveInfo.append(self._getEdgeFailureReasonOccurence(oReason, iPeriod, fEnter = False));
704
705	oSet.finalizePass2();
706
707	self._oDb.execute('DROP TABLE TmpReasons\n');
708	return oSet;
709
710
711	def _getEdgeFailureReasonOccurence(self, oReason, iPeriod, fEnter = True):
712	"""
713	Helper for the failure reason report that finds the oldest or newest build
714	(SVN rev) and test set (start time) it occured with.
715
716	If fEnter is set the oldest occurence is return, if fEnter clear the newest
717	is is returned.
718
719	Returns ReportFailureReasonTransient instant.
720
721	"""
722
723
724	sSorting = 'ASC' if fEnter else 'DESC';
725	self._oDb.execute('SELECT TmpReasons.idTestResult,\n'
726	' TmpReasons.idTestSet,\n'
727	' TmpReasons.tsDone,\n'
728	' TmpReasons.idBuild,\n'
729	' Builds.iRevision,\n'
730	' BuildCategories.sRepository\n'
731	'FROM TmpReasons,\n'
732	' Builds,\n'
733	' BuildCategories\n'
734	'WHERE TmpReasons.idFailureReason = %s\n'
735	' AND TmpReasons.idBuild = Builds.idBuild\n'
736	' AND Builds.tsExpire > TmpReasons.tsCreated\n'
737	' AND Builds.tsEffective <= TmpReasons.tsCreated\n'
738	' AND Builds.idBuildCategory = BuildCategories.idBuildCategory\n'
739	'ORDER BY Builds.iRevision ' + sSorting + ',\n'
740	' TmpReasons.tsCreated ' + sSorting + '\n'
741	'LIMIT 1\n'
742	, ( oReason.idFailureReason, ));
743	aoRow = self._oDb.fetchOne();
744	if aoRow is None:
745	return ReportFailureReasonTransient(-1, -1, 'internal-error', -1, -1, self._oDb.getCurrentTimestamp(),
746	iPeriod, fEnter, oReason);
747	return ReportFailureReasonTransient(idBuild = aoRow[3], iRevision = aoRow[4], sRepository = aoRow[5],
748	idTestSet = aoRow[1], idTestResult = aoRow[0], tsDone = aoRow[2],
749	iPeriod = iPeriod, fEnter = fEnter, oReason = oReason);
750
751
752	def getTestCaseFailures(self):
753	"""
754	Gets the test case failures of the subject in the specified period.
755
756	Returns a ReportPeriodSetWithTotalBase instance.
757
758	"""
759	return self._getSimpleFailures('idTestCase', TestCaseLogic);
760
761
762	def getTestCaseVariationFailures(self):
763	"""
764	Gets the test case failures of the subject in the specified period.
765
766	Returns a ReportPeriodSetWithTotalBase instance.
767
768	"""
769	return self._getSimpleFailures('idTestCaseArgs', TestCaseArgsLogic);
770
771
772	def getTestBoxFailures(self):
773	"""
774	Gets the test box failures of the subject in the specified period.
775
776	Returns a ReportPeriodSetWithTotalBase instance.
777
778	"""
779	return self._getSimpleFailures('idTestBox', TestBoxLogic);
780
781
782	def _getSimpleFailures(self, sIdColumn, oCacheLogicType, sIdAttr = None):
783	"""
784	Gets the test box failures of the subject in the specified period.
785
786	Returns a ReportPeriodSetWithTotalBase instance.
787
788	"""
789
790	oLogic = oCacheLogicType(self._oDb);
791	oSet = ReportPeriodSetWithTotalBase(sIdColumn if sIdAttr is None else sIdAttr);
792
793	# Construct base query.
794	sBaseQuery = 'SELECT TestSets.' + sIdColumn + ',\n' \
795	' COUNT(CASE WHEN TestSets.enmStatus >= \'failure\' THEN 1 END),\n' \
796	' MIN(TestSets.tsDone),\n' \
797	' MAX(TestSets.tsDone),\n' \
798	' COUNT(TestSets.idTestResult)\n' \
799	'FROM TestSets\n' \
800	+ self.oFilter.getTableJoins();
801	for sTable in self.getExtraSubjectTables():
802	sBaseQuery = sBaseQuery[:-1] + ',\n ' + sTable + '\n';
803	sBaseQuery += 'WHERE TRUE\n' \
804	+ self.oFilter.getWhereConditions() \
805	+ self.getExtraSubjectWhereExpr() + '\n';
806
807	# Retrieve the period results.
808	for iPeriod in xrange(self.cPeriods):
809	self._oDb.execute(sBaseQuery + self.getExtraWhereExprForPeriod(iPeriod) + 'GROUP BY TestSets.' + sIdColumn + '\n');
810	aaoRows = self._oDb.fetchAll()
811
812	oPeriod = ReportPeriodWithTotalBase(oSet, iPeriod, self.getStraightPeriodDesc(iPeriod),
813	self.getPeriodStart(iPeriod), self.getPeriodEnd(iPeriod));
814
815	for aoRow in aaoRows:
816	oSubject = oLogic.cachedLookup(aoRow[0]);
817	oPeriodRow = ReportHitRowWithTotalBase(aoRow[0], oSubject, aoRow[1], aoRow[4], aoRow[2], aoRow[3]);
818	oPeriod.appendRow(oPeriodRow, aoRow[0], oSubject);
819
820	oSet.appendPeriod(oPeriod);
821	oSet.pruneRowsWithZeroSumHits();
822
823
824
825	#
826	# For reasons entering after the first period, look up the build and
827	# test set it first occured with.
828	#
829	oSet.finalizePass1();
830
831	for iPeriod in xrange(1, self.cPeriods):
832	oPeriod = oSet.aoPeriods[iPeriod];
833	for idSubject, oSubject in oPeriod.dFirst.items():
834	oSet.aoEnterInfo.append(self._getEdgeSimpleFailureOccurence(idSubject, sIdColumn, oSubject,
835	iPeriod, fEnter = True));
836
837	# Ditto for reasons leaving before the last.
838	for iPeriod in xrange(self.cPeriods - 1):
839	oPeriod = oSet.aoPeriods[iPeriod];
840	for idSubject, oSubject in oPeriod.dLast.items():
841	oSet.aoLeaveInfo.append(self._getEdgeSimpleFailureOccurence(idSubject, sIdColumn, oSubject,
842	iPeriod, fEnter = False));
843
844	oSet.finalizePass2();
845
846	return oSet;
847
848	def _getEdgeSimpleFailureOccurence(self, idSubject, sIdColumn, oSubject, iPeriod, fEnter = True):
849	"""
850	Helper for the failure reason report that finds the oldest or newest build
851	(SVN rev) and test set (start time) it occured with.
852
853	If fEnter is set the oldest occurence is return, if fEnter clear the newest
854	is is returned.
855
856	Returns ReportTransientBase instant.
857
858	"""
859	sSorting = 'ASC' if fEnter else 'DESC';
860	sQuery = 'SELECT TestSets.idTestResult,\n' \
861	' TestSets.idTestSet,\n' \
862	' TestSets.tsDone,\n' \
863	' TestSets.idBuild,\n' \
864	' Builds.iRevision,\n' \
865	' BuildCategories.sRepository\n' \
866	'FROM TestSets\n' \
867	+ self.oFilter.getTableJoins(dOmitTables = {'Builds': True, 'BuildCategories': True});
868	sQuery = sQuery[:-1] + ',\n' \
869	' Builds,\n' \
870	' BuildCategories\n';
871	for sTable in self.getExtraSubjectTables():
872	if sTable not in [ 'Builds', 'BuildCategories' ] and not self.oFilter.isJoiningWithTable(sTable):
873	sQuery = sQuery[:-1] + ',\n ' + sTable + '\n';
874	sQuery += 'WHERE TestSets.' + sIdColumn + ' = ' + str(idSubject) + '\n' \
875	' AND TestSets.idBuild = Builds.idBuild\n' \
876	' AND TestSets.enmStatus >= \'failure\'\n' \
877	+ self.getExtraWhereExprForPeriod(iPeriod) + \
878	' AND Builds.tsExpire > TestSets.tsCreated\n' \
879	' AND Builds.tsEffective <= TestSets.tsCreated\n' \
880	' AND Builds.idBuildCategory = BuildCategories.idBuildCategory\n' \
881	+ self.oFilter.getWhereConditions() \
882	+ self.getExtraSubjectWhereExpr() + '\n' \
883	'ORDER BY Builds.iRevision ' + sSorting + ',\n' \
884	' TestSets.tsCreated ' + sSorting + '\n' \
885	'LIMIT 1\n';
886	self._oDb.execute(sQuery);
887	aoRow = self._oDb.fetchOne();
888	if aoRow is None:
889	return ReportTransientBase(-1, -1, 'internal-error', -1, -1, self._oDb.getCurrentTimestamp(),
890	iPeriod, fEnter, idSubject, oSubject);
891	return ReportTransientBase(idBuild = aoRow[3], iRevision = aoRow[4], sRepository = aoRow[5],
892	idTestSet = aoRow[1], idTestResult = aoRow[0], tsDone = aoRow[2],
893	iPeriod = iPeriod, fEnter = fEnter, idSubject = idSubject, oSubject = oSubject);
894
895	def fetchPossibleFilterOptions(self, oFilter, tsNow, sPeriod):
896	"""
897	Fetches possible filtering options.
898	"""
899	return TestResultLogic(self._oDb).fetchPossibleFilterOptions(oFilter, tsNow, sPeriod, oReportModel = self);
900
901
902
903	class ReportGraphModel(ReportModelBase): # pylint: disable=too-few-public-methods
904	"""
905	Extended report model used when generating the more complicated graphs
906	detailing results, time elapsed and values over time.
907	"""
908
909	## @name Subject ID types.
910	## These prefix the values in the aidSubjects array. The prefix is
911	## followed by a colon and then a list of string IDs. Following the prefix
912	## is one or more string table IDs separated by colons. These are used to
913	## drill down the exact test result we're looking for, by matching against
914	## TestResult::idStrName (in the db).
915	## @{
916	ksTypeResult = 'result';
917	ksTypeElapsed = 'elapsed';
918	## The last string table ID gives the name of the value.
919	ksTypeValue = 'value';
920	## List of types.
921	kasTypes = (ksTypeResult, ksTypeElapsed, ksTypeValue);
922	## @}
923
924	class SampleSource(object):
925	""" A sample source. """
926	def __init__(self, sType, aidStrTests, idStrValue):
927	self.sType = sType;
928	self.aidStrTests = aidStrTests;
929	self.idStrValue = idStrValue;
930
931	def getTestResultTables(self):
932	""" Retrieves the list of TestResults tables to join with."""
933	sRet = '';
934	for i in range(len(self.aidStrTests)):
935	sRet += ' TestResults TR%u,\n' % (i,);
936	return sRet;
937
938	def getTestResultConditions(self):
939	""" Retrieves the join conditions for the TestResults tables."""
940	sRet = '';
941	cItems = len(self.aidStrTests);
942	for i in range(cItems - 1):
943	sRet += ' AND TR%u.idStrName = %u\n' \
944	' AND TR%u.idTestResultParent = TR%u.idTestResult\n' \
945	% ( i, self.aidStrTests[cItems - i - 1], i, i + 1 );
946	sRet += ' AND TR%u.idStrName = %u\n' % (cItems - 1, self.aidStrTests[0]);
947	return sRet;
948
949	class DataSeries(object):
950	""" A data series. """
951	def __init__(self, oCache, idBuildCategory, idTestBox, idTestCase, idTestCaseArgs, iUnit):
952	_ = oCache;
953	self.idBuildCategory = idBuildCategory;
954	self.oBuildCategory = oCache.getBuildCategory(idBuildCategory);
955	self.idTestBox = idTestBox;
956	self.oTestBox = oCache.getTestBox(idTestBox);
957	self.idTestCase = idTestCase;
958	self.idTestCaseArgs = idTestCaseArgs;
959	if idTestCase is not None:
960	self.oTestCase = oCache.getTestCase(idTestCase);
961	self.oTestCaseArgs = None;
962	else:
963	self.oTestCaseArgs = oCache.getTestCaseArgs(idTestCaseArgs);
964	self.oTestCase = oCache.getTestCase(self.oTestCaseArgs.idTestCase);
965	self.iUnit = iUnit;
966	# Six parallel arrays.
967	self.aiRevisions = []; # The X values.
968	self.aiValues = []; # The Y values.
969	self.aiErrorBarBelow = []; # The Y value minimum errorbars, relative to the Y value (positive).
970	self.aiErrorBarAbove = []; # The Y value maximum errorbars, relative to the Y value (positive).
971	self.acSamples = []; # The number of samples at this X value.
972	self.aoRevInfo = []; # VcsRevisionData objects for each revision. Empty/SQL-NULL objects if no info.
973
974	class DataSeriesCollection(object):
975	""" A collection of data series corresponding to one input sample source. """
976	def __init__(self, oSampleSrc, asTests, sValue = None):
977	self.sType = oSampleSrc.sType;
978	self.aidStrTests = oSampleSrc.aidStrTests;
979	self.asTests = list(asTests);
980	self.idStrValue = oSampleSrc.idStrValue;
981	self.sValue = sValue;
982	self.aoSeries = [];
983
984	def addDataSeries(self, oDataSeries):
985	""" Appends a data series to the collection. """
986	self.aoSeries.append(oDataSeries);
987	return oDataSeries;
988
989
990	def __init__(self, oDb, tsNow, cPeriods, cHoursPerPeriod, sSubject, aidSubjects, # pylint: disable=too-many-arguments
991	aidTestBoxes, aidBuildCats, aidTestCases, fSepTestVars):
992	assert(sSubject == self.ksSubEverything); # dummy
993	ReportModelBase.__init__(self, oDb, tsNow, cPeriods, cHoursPerPeriod, sSubject, aidSubjects, oFilter = None);
994	self.aidTestBoxes = aidTestBoxes;
995	self.aidBuildCats = aidBuildCats;
996	self.aidTestCases = aidTestCases;
997	self.fOnTestCase = not fSepTestVars; # (Separates testcase variations into separate data series.)
998	self.oCache = DatabaseObjCache(self._oDb, self.tsNow, None, self.cPeriods * self.cHoursPerPeriod);
999
1000
1001	# Quickly validate and convert the subject "IDs".
1002	self.aoLookups = [];
1003	for sCur in self.aidSubjects:
1004	asParts = sCur.split(':');
1005	if len(asParts) < 2:
1006	raise TMExceptionBase('Invalid graph value "%s"' % (sCur,));
1007
1008	sType = asParts[0];
1009	if sType not in ReportGraphModel.kasTypes:
1010	raise TMExceptionBase('Invalid graph value type "%s" (full: "%s")' % (sType, sCur,));
1011
1012	aidStrTests = [];
1013	for sIdStr in asParts[1:]:
1014	try: idStr = int(sIdStr);
1015	except: raise TMExceptionBase('Invalid graph value id "%s" (full: "%s")' % (sIdStr, sCur,));
1016	if idStr < 0:
1017	raise TMExceptionBase('Invalid graph value id "%u" (full: "%s")' % (idStr, sCur,));
1018	aidStrTests.append(idStr);
1019
1020	idStrValue = None;
1021	if sType == ReportGraphModel.ksTypeValue:
1022	idStrValue = aidStrTests.pop();
1023	self.aoLookups.append(ReportGraphModel.SampleSource(sType, aidStrTests, idStrValue));
1024
1025	# done
1026
1027
1028	def getExtraWhereExprForTotalPeriod(self, sTimestampField):
1029	"""
1030	Returns additional WHERE expression for getting test sets for the
1031	specified period. It starts with an AND so that it can simply be
1032	appended to the WHERE clause.
1033	"""
1034	return self.getExtraWhereExprForTotalPeriodEx(sTimestampField, sTimestampField, True);
1035
1036	def getExtraWhereExprForTotalPeriodEx(self, sStartField = 'tsCreated', sEndField = 'tsDone', fLeadingAnd = True):
1037	"""
1038	Returns additional WHERE expression for getting test sets for the
1039	specified period.
1040	"""
1041	if self.tsNow is None:
1042	sNow = 'CURRENT_TIMESTAMP';
1043	else:
1044	sNow = self._oDb.formatBindArgs('%s::TIMESTAMP', (self.tsNow,));
1045
1046	sRet = ' AND %s >= (%s - interval \'%u hours\')\n' \
1047	' AND %s <= %s\n' \
1048	% ( sStartField, sNow, self.cPeriods * self.cHoursPerPeriod,
1049	sEndField, sNow);
1050
1051	if not fLeadingAnd:
1052	assert sRet[8] == ' ' and sRet[7] == 'D';
1053	return sRet[9:];
1054	return sRet;
1055
1056	def _getEligibleTestSetPeriod(self, sPrefix = 'TestSets.', fLeadingAnd = False):
1057	"""
1058	Returns additional WHERE expression for getting TestSets rows
1059	potentially relevant for the selected period.
1060	"""
1061	if self.tsNow is None:
1062	sNow = 'CURRENT_TIMESTAMP';
1063	else:
1064	sNow = self._oDb.formatBindArgs('%s::TIMESTAMP', (self.tsNow,));
1065
1066	# The 2nd line is a performance hack on TestSets. It nudges postgresql
1067	# into useing the TestSetsCreatedDoneIdx index instead of doing a table
1068	# scan when we look for eligible bits there.
1069	# ASSUMES no relevant test runs longer than 7 days!
1070	sRet = ' AND %stsCreated <= %s\n' \
1071	' AND %stsCreated >= (%s - interval \'%u hours\' - interval \'%u days\')\n' \
1072	' AND %stsDone >= (%s - interval \'%u hours\')\n' \
1073	% ( sPrefix, sNow,
1074	sPrefix, sNow, self.cPeriods * self.cHoursPerPeriod, 7,
1075	sPrefix, sNow, self.cPeriods * self.cHoursPerPeriod, );
1076
1077	if not fLeadingAnd:
1078	assert sRet[8] == ' ' and sRet[7] == 'D';
1079	return sRet[9:];
1080	return sRet;
1081
1082
1083	def _getNameStrings(self, aidStrTests):
1084	""" Returns an array of names corresponding to the array of string table entries. """
1085	return [self.oCache.getTestResultString(idStr) for idStr in aidStrTests];
1086
1087	def fetchGraphData(self):
1088	""" returns data """
1089	sWantedTestCaseId = 'idTestCase' if self.fOnTestCase else 'idTestCaseArgs';
1090
1091	aoRet = [];
1092	for oLookup in self.aoLookups:
1093	#
1094	# Set up the result collection.
1095	#
1096	if oLookup.sType == self.ksTypeValue:
1097	oCollection = self.DataSeriesCollection(oLookup, self._getNameStrings(oLookup.aidStrTests),
1098	self.oCache.getTestResultString(oLookup.idStrValue));
1099	else:
1100	oCollection = self.DataSeriesCollection(oLookup, self._getNameStrings(oLookup.aidStrTests));
1101
1102	#
1103	# Construct the query.
1104	#
1105	sQuery = 'SELECT Builds.iRevision,\n' \
1106	' TestSets.idBuildCategory,\n' \
1107	' TestSets.idTestBox,\n' \
1108	' TestSets.' + sWantedTestCaseId + ',\n';
1109	if oLookup.sType == self.ksTypeValue:
1110	sQuery += ' TestResultValues.iUnit as iUnit,\n' \
1111	' MIN(TestResultValues.lValue),\n' \
1112	' CAST(ROUND(AVG(TestResultValues.lValue)) AS BIGINT),\n' \
1113	' MAX(TestResultValues.lValue),\n' \
1114	' COUNT(TestResultValues.lValue)\n';
1115	elif oLookup.sType == self.ksTypeElapsed:
1116	sQuery += ' %u as iUnit,\n' \
1117	' CAST((EXTRACT(EPOCH FROM MIN(TR0.tsElapsed)) * 1000) AS INTEGER),\n' \
1118	' CAST((EXTRACT(EPOCH FROM AVG(TR0.tsElapsed)) * 1000) AS INTEGER),\n' \
1119	' CAST((EXTRACT(EPOCH FROM MAX(TR0.tsElapsed)) * 1000) AS INTEGER),\n' \
1120	' COUNT(TR0.tsElapsed)\n' \
1121	% (constants.valueunit.MS,);
1122	else:
1123	sQuery += ' %u as iUnit,\n'\
1124	' MIN(TR0.cErrors),\n' \
1125	' CAST(ROUND(AVG(TR0.cErrors)) AS INTEGER),\n' \
1126	' MAX(TR0.cErrors),\n' \
1127	' COUNT(TR0.cErrors)\n' \
1128	% (constants.valueunit.OCCURRENCES,);
1129
1130	if oLookup.sType == self.ksTypeValue:
1131	sQuery += 'FROM TestResultValues,\n';
1132	sQuery += ' TestSets,\n'
1133	sQuery += oLookup.getTestResultTables();
1134	else:
1135	sQuery += 'FROM ' + oLookup.getTestResultTables().lstrip();
1136	sQuery += ' TestSets,\n';
1137	sQuery += ' Builds\n';
1138
1139	if oLookup.sType == self.ksTypeValue:
1140	sQuery += 'WHERE TestResultValues.idStrName = %u\n' % ( oLookup.idStrValue, );
1141	sQuery += self.getExtraWhereExprForTotalPeriod('TestResultValues.tsCreated');
1142	sQuery += ' AND TestResultValues.idTestSet = TestSets.idTestSet\n';
1143	sQuery += self._getEligibleTestSetPeriod(fLeadingAnd = True);
1144	else:
1145	sQuery += 'WHERE ' + (self.getExtraWhereExprForTotalPeriod('TR0.tsCreated').lstrip()[4:]).lstrip();
1146	sQuery += ' AND TR0.idTestSet = TestSets.idTestSet\n';
1147
1148	if len(self.aidTestBoxes) == 1:
1149	sQuery += ' AND TestSets.idTestBox = %u\n' % (self.aidTestBoxes[0],);
1150	elif self.aidTestBoxes:
1151	sQuery += ' AND TestSets.idTestBox IN (' + ','.join([str(i) for i in self.aidTestBoxes]) + ')\n';
1152
1153	if len(self.aidBuildCats) == 1:
1154	sQuery += ' AND TestSets.idBuildCategory = %u\n' % (self.aidBuildCats[0],);
1155	elif self.aidBuildCats:
1156	sQuery += ' AND TestSets.idBuildCategory IN (' + ','.join([str(i) for i in self.aidBuildCats]) + ')\n';
1157
1158	if len(self.aidTestCases) == 1:
1159	sQuery += ' AND TestSets.idTestCase = %u\n' % (self.aidTestCases[0],);
1160	elif self.aidTestCases:
1161	sQuery += ' AND TestSets.idTestCase IN (' + ','.join([str(i) for i in self.aidTestCases]) + ')\n';
1162
1163	if oLookup.sType == self.ksTypeElapsed:
1164	sQuery += ' AND TestSets.enmStatus = \'%s\'::TestStatus_T\n' % (self.ksTestStatus_Success,);
1165
1166	if oLookup.sType == self.ksTypeValue:
1167	sQuery += ' AND TestResultValues.idTestResult = TR0.idTestResult\n'
1168	sQuery += self.getExtraWhereExprForTotalPeriod('TR0.tsCreated'); # For better index matching in some cases.
1169
1170	if oLookup.sType != self.ksTypeResult:
1171	sQuery += ' AND TR0.enmStatus = \'%s\'::TestStatus_T\n' % (self.ksTestStatus_Success,);
1172
1173	sQuery += oLookup.getTestResultConditions();
1174	sQuery += ' AND TestSets.idBuild = Builds.idBuild\n';
1175
1176	sQuery += 'GROUP BY TestSets.idBuildCategory,\n' \
1177	' TestSets.idTestBox,\n' \
1178	' TestSets.' + sWantedTestCaseId + ',\n' \
1179	' iUnit,\n' \
1180	' Builds.iRevision\n';
1181	sQuery += 'ORDER BY TestSets.idBuildCategory,\n' \
1182	' TestSets.idTestBox,\n' \
1183	' TestSets.' + sWantedTestCaseId + ',\n' \
1184	' iUnit,\n' \
1185	' Builds.iRevision\n';
1186
1187	#
1188	# Execute it and collect the result.
1189	#
1190	sCurRepository = None;
1191	dRevisions = {};
1192	oLastSeries = None;
1193	idLastBuildCat = -1;
1194	idLastTestBox = -1;
1195	idLastTestCase = -1;
1196	iLastUnit = -1;
1197	self._oDb.execute(sQuery);
1198	for aoRow in self._oDb.fetchAll(): # Fetching all here so we can make cache queries below.
1199	if aoRow[1] != idLastBuildCat \
1200	or aoRow[2] != idLastTestBox \
1201	or aoRow[3] != idLastTestCase \
1202	or aoRow[4] != iLastUnit:
1203	idLastBuildCat = aoRow[1];
1204	idLastTestBox = aoRow[2];
1205	idLastTestCase = aoRow[3];
1206	iLastUnit = aoRow[4];
1207	if self.fOnTestCase:
1208	oLastSeries = self.DataSeries(self.oCache, idLastBuildCat, idLastTestBox,
1209	idLastTestCase, None, iLastUnit);
1210	else:
1211	oLastSeries = self.DataSeries(self.oCache, idLastBuildCat, idLastTestBox,
1212	None, idLastTestCase, iLastUnit);
1213	oCollection.addDataSeries(oLastSeries);
1214	if oLastSeries.oBuildCategory.sRepository != sCurRepository:
1215	if sCurRepository is not None:
1216	self.oCache.preloadVcsRevInfo(sCurRepository, dRevisions.keys());
1217	sCurRepository = oLastSeries.oBuildCategory.sRepository
1218	dRevisions = {};
1219	oLastSeries.aiRevisions.append(aoRow[0]);
1220	oLastSeries.aiValues.append(aoRow[6]);
1221	oLastSeries.aiErrorBarBelow.append(aoRow[6] - aoRow[5]);
1222	oLastSeries.aiErrorBarAbove.append(aoRow[7] - aoRow[6]);
1223	oLastSeries.acSamples.append(aoRow[8]);
1224	dRevisions[aoRow[0]] = 1;
1225
1226	if sCurRepository is not None:
1227	self.oCache.preloadVcsRevInfo(sCurRepository, dRevisions.keys());
1228	del dRevisions;
1229
1230	#
1231	# Look up the VCS revision details.
1232	#
1233	for oSeries in oCollection.aoSeries:
1234	for iRevision in oSeries.aiRevisions:
1235	oSeries.aoRevInfo.append(self.oCache.getVcsRevInfo(sCurRepository, iRevision));
1236	aoRet.append(oCollection);
1237
1238	return aoRet;
1239
1240	def getEligibleTestBoxes(self):
1241	"""
1242	Returns a list of TestBoxData objects with eligible testboxes for
1243	the total period of time defined for this graph.
1244	"""
1245
1246	# Taking the simple way out now, getting all active testboxes at the
1247	# time without filtering out on sample sources.
1248
1249	# 1. Collect the relevant testbox generation IDs.
1250	self._oDb.execute('SELECT DISTINCT idTestBox, idGenTestBox\n'
1251	'FROM TestSets\n'
1252	'WHERE ' + self._getEligibleTestSetPeriod(fLeadingAnd = False) +
1253	'ORDER BY idTestBox, idGenTestBox DESC');
1254	idPrevTestBox = -1;
1255	asIdGenTestBoxes = [];
1256	for _ in range(self._oDb.getRowCount()):
1257	aoRow = self._oDb.fetchOne();
1258	if aoRow[0] != idPrevTestBox:
1259	idPrevTestBox = aoRow[0];
1260	asIdGenTestBoxes.append(str(aoRow[1]));
1261
1262	# 2. Query all the testbox data in one go.
1263	aoRet = [];
1264	if asIdGenTestBoxes:
1265	self._oDb.execute('SELECT *\n'
1266	'FROM TestBoxesWithStrings\n'
1267	'WHERE idGenTestBox IN (' + ','.join(asIdGenTestBoxes) + ')\n'
1268	'ORDER BY sName');
1269	for _ in range(self._oDb.getRowCount()):
1270	aoRet.append(TestBoxData().initFromDbRow(self._oDb.fetchOne()));
1271
1272	return aoRet;
1273
1274	def getEligibleBuildCategories(self):
1275	"""
1276	Returns a list of BuildCategoryData objects with eligible build
1277	categories for the total period of time defined for this graph. In
1278	addition it will add any currently selected categories that aren't
1279	really relevant to the period, just to simplify the WUI code.
1280
1281	"""
1282
1283	# Taking the simple way out now, getting all used build cat without
1284	# any testbox or testcase filtering.
1285
1286	sSelectedBuildCats = '';
1287	if self.aidBuildCats:
1288	sSelectedBuildCats = ' OR idBuildCategory IN (' + ','.join([str(i) for i in self.aidBuildCats]) + ')\n';
1289
1290	self._oDb.execute('SELECT DISTINCT *\n'
1291	'FROM BuildCategories\n'
1292	'WHERE idBuildCategory IN (\n'
1293	' SELECT DISTINCT idBuildCategory\n'
1294	' FROM TestSets\n'
1295	' WHERE ' + self._getEligibleTestSetPeriod(fLeadingAnd = False) +
1296	')\n'
1297	+ sSelectedBuildCats +
1298	'ORDER BY sProduct,\n'
1299	' sBranch,\n'
1300	' asOsArches,\n'
1301	' sType\n');
1302	aoRet = [];
1303	for _ in range(self._oDb.getRowCount()):
1304	aoRet.append(BuildCategoryData().initFromDbRow(self._oDb.fetchOne()));
1305
1306	return aoRet;
1307

注意: 瀏覽 TracBrowser 來幫助您使用儲存庫瀏覽器

source: vbox/trunk/src/VBox/ValidationKit/testmanager/core/report.py

以其他格式下載: