modules/enterprise/server/jar/src/test/java/org/rhq/enterprise/server/performance/test/AvailabilityInsertPurgeTest.java | 212 ++++++++++ modules/enterprise/server/jar/src/test/java/org/rhq/enterprise/server/test/AbstractEJB3PerformanceTest.java | 101 ++++ modules/helpers/perftest-data-gen/install_files.sh | 6 modules/helpers/perftest-data-gen/pom.xml | 1 modules/helpers/perftest-data-gen/src/main/java/org/rhq/etc/perftestDataGen/DataGen.java | 19 modules/plugins/mysql/src/main/java/org/rhq/plugins/mysql/MySqlDiscoveryComponent.java | 25 - pom.xml | 1 7 files changed, 350 insertions(+), 15 deletions(-) New commits: commit 6a4c52d2c965cb68fbcb40b9ccfe5256cccd2c01 Author: Heiko W. Rupp <hwr(a)redhat.com&gt; Date: Fri Aug 20 14:38:25 2010 +0200 Prototypical work for the PerfTests diff --git a/modules/enterprise/server/jar/src/test/java/org/rhq/enterprise/server/performance/test/AvailabilityInsertPurgeTest.java b/modules/enterprise/server/jar/src/test/java/org/rhq/enterprise/server/performance/test/AvailabilityInsertPurgeTest.java new file mode 100644 index 0000000..58603ea --- /dev/null +++ b/modules/enterprise/server/jar/src/test/java/org/rhq/enterprise/server/performance/test/AvailabilityInsertPurgeTest.java @@ -0,0 +1,212 @@ +/* + * RHQ Management Platform + * Copyright (C) 2005-2010 Red Hat, Inc. + * All rights reserved. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation version 2 of the License. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. + */ +package org.rhq.enterprise.server.performance.test; + +import java.io.FileNotFoundException; +import java.io.FileReader; +import java.io.IOException; +import java.net.URL; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import au.com.bytecode.opencsv.CSVReader; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; + +import org.rhq.core.domain.plugin.Plugin; +import org.rhq.core.domain.resource.Agent; +import org.rhq.core.domain.resource.ResourceCategory; +import org.rhq.core.domain.resource.ResourceType; +import org.rhq.enterprise.server.test.AbstractEJB3PerformanceTest; + +import org.testng.annotations.Test; + +/** + * Performance test the availabilities subsystem + * + * @author Heiko W. Rupp + */ +@Test(groups = "PERF") +public class AvailabilityInsertPurgeTest extends AbstractEJB3PerformanceTest { + + private final Log log = LogFactory.getLog(AvailabilityInsertPurgeTest.class); + + /* + * we need to replace the ids in the csv files with the ids that we get back from the + * databse in relations. So store them as pair <csv-id,new entity-id> + */ + private Map<Integer,Integer> agentsTranslationTable = new HashMap<Integer,Integer>(); + private Map<Integer,Integer> pluginsTranslationTable = new HashMap<Integer,Integer>(); + private Map<Integer,Integer> resourceTypeTranslationTable = new HashMap<Integer,Integer>(); + + + public void testOne() throws Exception { + setup(); + startTiming(); + + Thread.sleep(1234); + + endTiming(); + + commitTimings(); + + } + + private void setup() { + setupAgents(); + setupPlugins(); + setupResourceTypes(); + // TODO set up resources + + } + + private void setupAgents() { + String descriptorFile = "perftest/agents.csv"; + URL descriptorUrl = this.getClass().getClassLoader().getResource(descriptorFile); + FileReader fr = null; + try { + String fileName = descriptorUrl.getFile(); + fr = new FileReader(fileName); + } catch (FileNotFoundException e) { + e.printStackTrace(); // TODO: Customise this generated block + return; + } + try { + CSVReader reader = new CSVReader(fr,',','"',1); // Skip 1st line, use " as quote char and , as separator + + List<String[]> lines = reader.readAll(); + System.out.println("# of lines: " + lines.size()); + for (String[] line : lines) { + if (line[0].startsWith("#")) + continue; // comment + + int originalId = Integer.parseInt(line[0]); + Agent agent = new Agent(line[1],line[2],Integer.parseInt(line[3]),line[5],line[4]);// TODO more information? + getEntityManager().persist(agent); + int id = agent.getId(); + + agentsTranslationTable.put(originalId,id); + + + } + } + catch (IOException ioe) { + ioe.printStackTrace(); + } + finally { + try { + fr.close(); + } catch (IOException e) { + e.printStackTrace(); // TODO: Customise this generated block + } + } + + } + + private void setupPlugins() { + String descriptorFile = "perftest/plugins.csv"; + URL descriptorUrl = this.getClass().getClassLoader().getResource(descriptorFile); + FileReader fr = null; + try { + String fileName = descriptorUrl.getFile(); + fr = new FileReader(fileName); + } catch (FileNotFoundException e) { + e.printStackTrace(); // TODO: Customise this generated block + return; + } + try { + CSVReader reader = new CSVReader(fr,',','"',1); + + List<String[]> lines = reader.readAll(); + System.out.println("# of lines: " + lines.size()); + for (String[] line : lines) { + if (line[0].startsWith("#")) + continue; // comment + + int originalId = Integer.parseInt(line[0]); + Plugin plugin = new Plugin(line[1],line[5],line[6]); + plugin.setDisplayName(line[2]); + plugin.setVersion(line[3]); + plugin.setAmpsVersion(line[4]); + getEntityManager().persist(plugin); + + int id = plugin.getId(); + pluginsTranslationTable.put(originalId,id); + } + } + catch (IOException ioe) { + ioe.printStackTrace(); + } + finally { + try { + fr.close(); + } catch (IOException e) { + e.printStackTrace(); // TODO: Customise this generated block + } + } + + } + private void setupResourceTypes() { + + // TODO first pull in the parentResourceTypes.csv file to get a mapping for them. + + String descriptorFile = "perftest/resourceTypes.csv"; + URL descriptorUrl = this.getClass().getClassLoader().getResource(descriptorFile); + FileReader fr = null; + try { + String fileName = descriptorUrl.getFile(); + fr = new FileReader(fileName); + } catch (FileNotFoundException e) { + e.printStackTrace(); // TODO: Customise this generated block + return; + } + try { + CSVReader reader = new CSVReader(fr,',','"',1); + + List<String[]> lines = reader.readAll(); + System.out.println("# of lines: " + lines.size()); + for (String[] line : lines) { + if (line[0].startsWith("#")) + continue; // comment + + int originalId = Integer.parseInt(line[0]); + ResourceType parentType = null; // TODO get from parent-child table + ResourceCategory category = ResourceCategory.valueOf(line[2]); + ResourceType rt = new ResourceType(line[1],line[3],category,parentType); + getEntityManager().persist(rt); + + int id = rt.getId(); + resourceTypeTranslationTable.put(originalId,id); + } + } + catch (IOException ioe) { + ioe.printStackTrace(); + } + finally { + try { + fr.close(); + } catch (IOException e) { + e.printStackTrace(); // TODO: Customise this generated block + } + } + + } +} diff --git a/modules/enterprise/server/jar/src/test/java/org/rhq/enterprise/server/test/AbstractEJB3PerformanceTest.java b/modules/enterprise/server/jar/src/test/java/org/rhq/enterprise/server/test/AbstractEJB3PerformanceTest.java new file mode 100644 index 0000000..05b3495 --- /dev/null +++ b/modules/enterprise/server/jar/src/test/java/org/rhq/enterprise/server/test/AbstractEJB3PerformanceTest.java @@ -0,0 +1,101 @@ +/* + * RHQ Management Platform + * Copyright (C) 2005-2010 Red Hat, Inc. + * All rights reserved. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation version 2 of the License. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. + */ +package org.rhq.enterprise.server.test; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; + +import java.util.HashMap; +import java.util.Map; +import java.util.Set; + +/** + * Helper that introduces timing functionality on top of the Abstract EJB tests. + * + * @author Heiko W. Rupp + */ +public class AbstractEJB3PerformanceTest extends AbstractEJB3Test { + + private static final Log log = LogFactory.getLog("TIMING_INFO"); + + + private static final String DEFAULT = "-default-"; + private Map<String,Long> timings = new HashMap<String, Long>(); + private Map<String,Long> startTime = new HashMap<String, Long>(); + + + protected void startTiming(String name) { + long now = System.currentTimeMillis(); + startTime.put(name,now); + + } + + protected void endTiming(String name) { + + boolean found = startTime.containsKey(name); + assert found : "No start time information for name [" + name + "] found - did you call startTiming()?"; + + + long now = System.currentTimeMillis(); + long start = startTime.get(name); + long duration = (now - start); + if (timings.containsKey(name)) { + long timing = timings.get(name); + timing+=duration; + timings.put(name,timing); + } + else { + timings.put(name,duration); + } + } + + protected void startTiming() { + startTiming(DEFAULT); + } + + protected void endTiming() { + endTiming(DEFAULT); + } + + + protected void commitTimings() { + + Set<Map.Entry<String,Long>> data = timings.entrySet(); + for (Map.Entry<String,Long> item : data) { + log.info(":| " + item.getKey() + " => " + item.getValue()); + } + timings.clear(); + startTime.clear(); + } + + protected void assertTiming(String name, long maxDuration) { + + boolean found = timings.containsKey(name); + assert found : "No timing information for name [" + name + "] found"; + + long duration = timings.get(name); + + assert duration < maxDuration : "Execution took longer than given max ( " + duration + " > " + maxDuration + ")"; + + } + + protected void assertTiming(long maxDuration) { + assertTiming(DEFAULT,maxDuration); + } +} commit 74d0107cf4038dc3e307a384f25efd54bea22b93 Author: Heiko W. Rupp <hwr(a)redhat.com&gt; Date: Fri Aug 20 14:37:57 2010 +0200 Create dir if not there. diff --git a/modules/helpers/perftest-data-gen/install_files.sh b/modules/helpers/perftest-data-gen/install_files.sh index 0a33154..3513612 100644 --- a/modules/helpers/perftest-data-gen/install_files.sh +++ b/modules/helpers/perftest-data-gen/install_files.sh @@ -4,4 +4,8 @@ set -x pwd -cp target/*.csv ../../enterprise/server/jar/src/test/resources/perftest/ \ No newline at end of file +if [ ! -d ../../enterprise/server/jar/src/test/resources/perftest/ ] +then + mkdir -p ../../enterprise/server/jar/src/test/resources/perftest/ +fi +cp target/*.csv ../../enterprise/server/jar/src/test/resources/perftest/ commit 5ecc0338f2129d34de47c80f670ac1478efe92f5 Author: Heiko W. Rupp <hwr(a)redhat.com&gt; Date: Fri Aug 20 14:23:41 2010 +0200 Add table and OrderBy column diff --git a/modules/helpers/perftest-data-gen/pom.xml b/modules/helpers/perftest-data-gen/pom.xml index dceeb22..b9dafce 100644 --- a/modules/helpers/perftest-data-gen/pom.xml +++ b/modules/helpers/perftest-data-gen/pom.xml @@ -40,5 +40,4 @@ </dependency> </dependencies> - </project> \ No newline at end of file diff --git a/modules/helpers/perftest-data-gen/src/main/java/org/rhq/etc/perftestDataGen/DataGen.java b/modules/helpers/perftest-data-gen/src/main/java/org/rhq/etc/perftestDataGen/DataGen.java index 4e0857e..f51973f 100644 --- a/modules/helpers/perftest-data-gen/src/main/java/org/rhq/etc/perftestDataGen/DataGen.java +++ b/modules/helpers/perftest-data-gen/src/main/java/org/rhq/etc/perftestDataGen/DataGen.java @@ -42,11 +42,13 @@ public class DataGen { private static final String DOTCSV = ".csv"; private static final String[][] props = {// - {"agents","RHQ_Agent","id,name,address,port,agenttoken,remote_endpoint"}, - {"plugins","RHQ_Plugin","id,name,display_name,version,amps_version"}, - {"resourceTypes","RHQ_resource_type","id,name,category,plugin"}, // TODO parent / child types? - {"resources","RHQ_resource","id,uuid,resource_key,name,resource_type_id,parent_resource_id"} // TODO child resources? - }; + // filename , table , columns , oderby column + {"agents","RHQ_Agent","id,name,address,port,agenttoken,remote_endpoint","id"}, + {"plugins","RHQ_Plugin","id,name,display_name,version,amps_version,path,md5","id"}, + {"resourceTypes","RHQ_resource_type","id,name,category,plugin","id"}, + {"parentResourceTypes","RHQ_RESOURCE_TYPE_PARENTS","resource_type_id,parent_resource_type_id",null}, + {"resources","RHQ_resource","id,uuid,resource_key,name,resource_type_id,parent_resource_id","id"} // TODO child resources? + }; public static void main(String[] args) { @@ -83,12 +85,17 @@ public class DataGen { String fileName = prop[0]; String tableName = prop[1]; String columns = prop[2]; + String orderBy = prop[3]; + File agents = new File(TARGET + fileName + DOTCSV); System.out.println("Writing file: " + agents.getAbsolutePath()); CSVWriter writer = new CSVWriter(new FileWriter(agents)); Statement stm = conn.createStatement(); - String query = "SELECT " + columns + " FROM " + tableName; + String query = "SELECT " + columns + " FROM " + tableName ; + if (orderBy!=null) { + query += " ORDER BY " + orderBy + " ASC"; + } System.out.println(" using query: [" + query + "]"); System.out.flush(); ResultSet rs = stm.executeQuery(query); commit fce93b5bcb146d746b35f786bebc0b35ab9bc7a2 Merge: 660a61e... 462e15f... Author: Heiko W. Rupp <hwr(a)redhat.com&gt; Date: Fri Aug 20 12:58:23 2010 +0200 Merge branch 'master' of ssh://git.fedorahosted.org/git/rhq/rhq commit 660a61e8f9e2796b69e39922406353bf227bd448 Author: Heiko W. Rupp <hwr(a)redhat.com&gt; Date: Fri Aug 20 12:57:56 2010 +0200 Be less loud on exceptions when talking to MySQL. diff --git a/modules/plugins/mysql/src/main/java/org/rhq/plugins/mysql/MySqlDiscoveryComponent.java b/modules/plugins/mysql/src/main/java/org/rhq/plugins/mysql/MySqlDiscoveryComponent.java index f50fbc0..4f1045e 100644 --- a/modules/plugins/mysql/src/main/java/org/rhq/plugins/mysql/MySqlDiscoveryComponent.java +++ b/modules/plugins/mysql/src/main/java/org/rhq/plugins/mysql/MySqlDiscoveryComponent.java @@ -62,7 +62,10 @@ public class MySqlDiscoveryComponent implements ResourceDiscoveryComponent, Manu ProcessInfo procInfo = result.getProcessInfo(); - servers.add(createResourceDetails(context,context.getDefaultPluginConfiguration(),procInfo)); + DiscoveredResourceDetails resourceDetails = createResourceDetails(context, context.getDefaultPluginConfiguration(), procInfo); + if (resourceDetails!=null) { + servers.add(resourceDetails); + } } return servers; @@ -78,14 +81,22 @@ public class MySqlDiscoveryComponent implements ResourceDiscoveryComponent, Manu } protected static DiscoveredResourceDetails createResourceDetails(ResourceDiscoveryContext discoveryContext, - Configuration pluginConfiguration, - ProcessInfo processInfo) { + Configuration pluginConfiguration, ProcessInfo processInfo) { + String key = buildUrl(pluginConfiguration); String db = pluginConfiguration.getSimple(DB_CONFIGURATION_PROPERTY).getStringValue(); String name = "MySql [" + db + "]"; - String version = getVersion(pluginConfiguration); - return new DiscoveredResourceDetails(discoveryContext.getResourceType(), key, name, version, - DEFAULT_RESOURCE_DESCRIPTION, pluginConfiguration, processInfo); + try { + String version = getVersion(pluginConfiguration); + return new DiscoveredResourceDetails(discoveryContext.getResourceType(), key, name, version, + DEFAULT_RESOURCE_DESCRIPTION, pluginConfiguration, processInfo); + } catch (Exception e) { + log.warn("Getting details failed: " + e.getMessage()); + if (e.getCause()!=null) { + log.warn(" caused by: " + e.getCause().getMessage()); + } + } + return null; } protected static String buildUrl(Configuration config) { @@ -104,7 +115,7 @@ public class MySqlDiscoveryComponent implements ResourceDiscoveryComponent, Manu version = conn.getMetaData().getDatabaseProductVersion(); } catch (SQLException e) { // TODO GH: How to put this back to the server while inventorying this resource in an unconfigured state - log.info("Exception detecting mysql instance version", e); + log.info("Exception detecting mysql instance version" + e.getMessage()); } return version; } commit 19c1364252e04afde403a0413e0259a9f68d990e Author: Heiko W. Rupp <hwr(a)redhat.com&gt; Date: Thu Aug 19 15:14:49 2010 +0200 Add John Sanda's blog. diff --git a/pom.xml b/pom.xml index 6271e7e..6f27b89 100644 --- a/pom.xml +++ b/pom.xml @@ -1206,6 +1206,7 @@ </developer> <developer> <name>John Sanda</name> + <url>http://johnsanda.blogspot.com</url> <timezone>-5</timezone> </developer> <developer>