[svn-commits] r189 - in branches/advisor/src: back/scf/scm front/misc/analyzedb
thial01 at ingres.com
thial01 at ingres.com
Tue Aug 12 11:46:33 PDT 2008
Author: thial01
Date: 2008-08-12 11:46:33 -0700 (Tue, 12 Aug 2008)
New Revision: 189
Modified:
branches/advisor/src/back/scf/scm/scmonitor.sc
branches/advisor/src/back/scf/scm/workloaddb.sql
branches/advisor/src/front/misc/analyzedb/Analyze.py
branches/advisor/src/front/misc/analyzedb/Containers.py
branches/advisor/src/front/misc/analyzedb/Present.py
Log:
Changing analyzer to respect new monitoring code - see #174
Modified: branches/advisor/src/back/scf/scm/scmonitor.sc
===================================================================
--- branches/advisor/src/back/scf/scm/scmonitor.sc 2008-08-12 12:33:05 UTC (rev 188)
+++ branches/advisor/src/back/scf/scm/scmonitor.sc 2008-08-12 18:46:33 UTC (rev 189)
@@ -78,7 +78,7 @@
/* How long do we sleep between wake-ups? */
#define SLEEP 30000
/* How often do we wake up until we store data in workload DB? */
-#define ROUNDS 5
+#define ROUNDS 10
/* How often do we store until we delete old entries from the workload DB? */
#define DELOLD 5
/* And how old are the entries we delete (in seconds) */
@@ -400,7 +400,9 @@
exec sql select count(server)
into :rowcount
from workload
- where idx = :workload.idx;
+ where idx = :workload.idx
+ and query_key = :workload.query_key
+ and time = :workload.time;
if (rowcount == 0)
{
@@ -1318,9 +1320,7 @@
exec sql delete from statements
where database = :db_name
- and query_key not in
- (select query_key from workload
- where database = :db_name);
+ and time < :time;
exec sql delete from tables
where database = :db_name
Modified: branches/advisor/src/back/scf/scm/workloaddb.sql
===================================================================
--- branches/advisor/src/back/scf/scm/workloaddb.sql 2008-08-12 12:33:05 UTC (rev 188)
+++ branches/advisor/src/back/scf/scm/workloaddb.sql 2008-08-12 18:46:33 UTC (rev 189)
@@ -29,14 +29,12 @@
pages_touched integer4,
time integer4,
wctime integer4,
- primary key (idx)
+ primary key (idx, query_key, time)
)
\p\g
modify workload to btree\p\g
-create index workload_time_idx on workload(time) with structure = btree\p\g
-
drop table tables\p\g
create table tables (
Modified: branches/advisor/src/front/misc/analyzedb/Analyze.py
===================================================================
--- branches/advisor/src/front/misc/analyzedb/Analyze.py 2008-08-12 12:33:05 UTC (rev 188)
+++ branches/advisor/src/front/misc/analyzedb/Analyze.py 2008-08-12 18:46:33 UTC (rev 189)
@@ -8,6 +8,7 @@
__recommendations = []
__statements = []
+ __workload = []
__indexes = []
__attributes = []
__tables = []
@@ -44,6 +45,7 @@
self.__load_indexes()
self.__load_attributes()
self.__load_tables()
+ self.__load_workload()
self.__load_statements()
for statement in self.__statements:
@@ -104,33 +106,54 @@
self.__tables.append(Table(row, self.__attributes))
- def __load_statements(self):
+ def __load_workload(self):
- """ Get statements - most frequent and expensive ones first """
+ """ Get workload history data """
results = self.__workloaddb.execute("select \
- query_key, query_text, frequency, opf_cpu, opf_dio, qef_cpu, qef_dio, \
- est_cpu, est_dio, pages_touched, time, wctime \
- from statements where database = ? order by (frequency * (opf_cpu + opf_dio + qef_cpu + qef_dio)) desc",
+ query_key, opf_cpu, opf_dio, qef_cpu, qef_dio, est_cpu, est_dio, pages_touched, time, wctime \
+ from workload where database = ?",
(self.__config.userdb,) )
+ for row in results:
+ self.__workload.append(Workload(row))
+
+
+ def __load_statements(self):
+
+ """ Get statements """
+ results = self.__workloaddb.execute("select \
+ query_key, query_text, frequency \
+ from statements where database = ?",
+ (self.__config.userdb,) )
+
self.__workloadsize = len(results)
- count = 1
for row in results:
-
- if self.__config.querylimit != -1 and count > self.__config.querylimit:
- debug("Limit of queries to process reached - stopping")
- break
-
""" Try to find out if this is a select """
if (row[1].lstrip().lower())[0:6] == "select":
- count += 1
- stmt = Statement(row)
- self.__load_references(stmt)
- self.__statements.append(stmt)
+ row1 = self.__workloaddb.execute("select first 1 \
+ opf_cpu, opf_dio, qef_cpu, qef_dio, est_cpu, est_dio, pages_touched, time, wctime \
+ from workload where query_key = ? order by time desc",
+ (row[0],))
+ if len(row) > 0:
+ self.__statements.append(Statement(row, row1[0]))
+
+ """ Sort statements - most frequent and expensive ones first """
+ self.__statements.sort(lambda x, y: cmp(
+ x.frequency * (x.opf_cpu+x.opf_dio+x.qef_cpu+x.qef_dio),
+ y.frequency * (y.opf_cpu+y.opf_dio+y.qef_cpu+y.qef_dio)),
+ reverse=True)
+ if self.__config.querylimit != -1:
+ """ Only keep the first n statements """
+ del self.__statements[self.__config.querylimit:]
+
+ """ Load references for the rest """
+ for stmt in self.__statements:
+ self.__load_references(stmt)
+
def __load_references(self, statement):
debug("\n\n## Statement %d: %s\n" % (statement.query_key, statement.query_text))
@@ -412,7 +435,7 @@
avg /= len(self.__avg_win)
debug(" Average win per statement: %d %%" % avg)
- return Results(self.__statements, self.__recommendations, overall, avg, self.__statistics, self.__time)
+ return Results(self.__statements, self.__workload, self.__recommendations, overall, avg, self.__statistics, self.__time)
@@ -494,8 +517,9 @@
avg_win = []
statistics = False
- def __init__(self, statements, recommendations, overall, avg, statistics, time ):
+ def __init__(self, statements, workload, recommendations, overall, avg, statistics, time ):
self.statements = statements
+ self.workload = workload
self.recommendations = recommendations
self.overall_win = overall
self.avg_win = avg
Modified: branches/advisor/src/front/misc/analyzedb/Containers.py
===================================================================
--- branches/advisor/src/front/misc/analyzedb/Containers.py 2008-08-12 12:33:05 UTC (rev 188)
+++ branches/advisor/src/front/misc/analyzedb/Containers.py 2008-08-12 18:46:33 UTC (rev 189)
@@ -2,20 +2,20 @@
""" Statement class """
class Statement():
- def __init__(self, values):
+ def __init__(self, values, values1):
self.query_key = values[0]
self.query_text = values[1].strip()
self.frequency = values[2]
- self.opf_cpu = values[3]
- self.opf_dio = values[4]
- self.qef_cpu = values[5]
- self.qef_dio = values[6]
- self.est_cpu = values[7]
- self.est_dio = values[8]
- self.pages_touched = values[9]
- self.time = values[10]
- self.wctime = values[11]
+ self.opf_cpu = values1[0]
+ self.opf_dio = values1[1]
+ self.qef_cpu = values1[2]
+ self.qef_dio = values1[3]
+ self.est_cpu = values1[4]
+ self.est_dio = values1[5]
+ self.pages_touched = values1[6]
+ self.time = values1[7]
+ self.wctime = values1[8]
self.newcost = -1
self.tables = []
self.indexes = []
@@ -26,6 +26,24 @@
self.indexes = indexes
+
+""" Workload class """
+class Workload():
+
+ def __init__(self, values):
+
+ self.query_key = values[0]
+ self.opf_cpu = values[1]
+ self.opf_dio = values[2]
+ self.qef_cpu = values[3]
+ self.qef_dio = values[4]
+ self.est_cpu = values[5]
+ self.est_dio = values[6]
+ self.pages_touched = values[7]
+ self.time = values[8]
+ self.wctime = values[9]
+
+
""" Table class """
class Table():
Modified: branches/advisor/src/front/misc/analyzedb/Present.py
===================================================================
--- branches/advisor/src/front/misc/analyzedb/Present.py 2008-08-12 12:33:05 UTC (rev 188)
+++ branches/advisor/src/front/misc/analyzedb/Present.py 2008-08-12 18:46:33 UTC (rev 189)
@@ -51,12 +51,13 @@
(Recorded between %s and %s)<br />
<br />
There are %d attributes in %d tables with %d indexes in this database.<br />
- The current workload counts %d distinct statements.<br />
+ The current workload counts %d distinct statements and %d selects were executed.<br />
Since the DBMS is up %d selects have been executed and returned %d rows in total.<br />
At most %d users were connected at the same time. On average %d users are connected.<br />
There were at most %d sessions at the same time and %d sessions on average.<br />
<br /><br />
- %s %s
+ %s %s<br />
+ %s
'''
__recommendations = '''
@@ -85,18 +86,25 @@
def __draw_graphs(self):
+
costs = []
for stmt in self.__results.statements:
costs.append([stmt.query_key,
stmt.opf_cpu+stmt.opf_dio+stmt.qef_cpu+stmt.qef_dio,
stmt.est_cpu+stmt.est_dio,
stmt.newcost])
- """"%d %%" % (100 - (100 * (stmt.newcost+1) / (stmt.est_cpu+stmt.est_dio+1)))])"""
+
+ runtime = []
+ coststime = []
+ for wkl in self.__results.workload:
+ runtime.append([wkl.time,
+ wkl.wctime])
+ coststime.append([wkl.time,
+ (wkl.opf_cpu+wkl.opf_dio+wkl.qef_cpu+wkl.qef_dio) / 1000])
+
+ runtime.sort(lambda x, y: cmp(x[0], y[0]))
+ coststime.sort(lambda x, y: cmp(x[0], y[0]))
- 'costs.sort(lambda x, y: cmp(x[1], y[1]), reverse=True)'
- 'act_costs.sort(lambda x, y: cmp(x[1], y[1]), reverse=True)'
- 'new_costs.sort(lambda x, y: cmp(x[1], y[1]), reverse=True)'
-
debug(" Drawing costs histogram")
histo1 = Histogram("Top 10 - Cost Comparison", "Query Key", "Cost", ("Actual Costs", "Old Cost Estimates", "New Cost Estimates"))
histo1.rows = costs[0:10]
@@ -118,8 +126,14 @@
curve2.add_line(self.__results.statistics.locks_per_tx, "Locks Per Transaction")
curve2.add_line(self.__results.statistics.max_locks, "Max Locks")
curve2.draw("locks")
+
+ debug(" Drawing workload curve")
+ curve3 = Curve("Workload", "Time", "Runtime", True)
+ curve3.add_curve(runtime, "Time to Execute")
+ curve3.add_curve(coststime, "Cost / 1000")
+ curve3.draw("runtime")
+
-
def __render(self):
debug(" Render HTML")
@@ -158,6 +172,7 @@
self.__results.statistics.counts[0],
self.__results.statistics.counts[2],
self.__results.statistics.workloadsize,
+ len(self.__results.workload),
self.__results.statistics.selects_processed,
self.__results.statistics.total_rows,
max_connections,
@@ -165,7 +180,8 @@
max_sessions,
avg_sessions,
self.__picture("connections"),
- self.__picture("locks")
+ self.__picture("locks"),
+ self.__picture("runtime")
)
self.__recommendations = self.__recommendations % (
@@ -221,7 +237,6 @@
stm_str = "<h3>Statements:</h3>"
for s in stm:
stm_str += "<b>Query %s:</b><br /> " % self.__querylink(s)
- stm[s].sort(lambda x, y: cmp(x.frequency, y.frequency), reverse=True)
for v in stm[s]:
stm_str += recom_str % (
v.frequency,
@@ -287,7 +302,7 @@
<b>Frequency:</b> %d<br />
<b>Time to Execute:</b> %d<br />
<b>Actual Costs:</b> %d | <b>Old Estimated Costs:</b> %d | <b>New Estimated Costs:</b> %d <br />
- <b>Estimated Win:</b> %d %%)
+ <b>Estimated Win:</b> %d %%
<br /><br />
''' % (
statement.query_key,
@@ -295,7 +310,7 @@
statement.query_text,
statement.frequency,
statement.wctime,
- statement.opf_cpu + statement.qef_cpu,
+ statement.opf_cpu + statement.opf_dio + statement.qef_cpu + statement.qef_dio,
statement.est_cpu + statement.est_dio,
statement.newcost,
(100 - (100 * (statement.newcost+1) / (statement.est_cpu + statement.est_dio+1)))
More information about the svn-commits
mailing list