modules/core/domain/src/main/java/org/rhq/core/domain/measurement/MeasurementBaseline.java | 16 ++ modules/enterprise/server/jar/src/main/java/org/rhq/enterprise/server/measurement/MeasurementBaselineManagerBean.java | 68 ++++------ modules/enterprise/server/jar/src/main/java/org/rhq/enterprise/server/measurement/MeasurementBaselineManagerLocal.java | 7 - modules/enterprise/server/server-metrics/src/main/java/org/rhq/server/metrics/MetricsBaselineCalculator.java | 21 +-- modules/enterprise/server/server-metrics/src/test/java/org/rhq/server/metrics/MetricsBaselineCalculatorTest.java | 48 +------ 5 files changed, 68 insertions(+), 92 deletions(-)
New commits: commit 3c8110a8abd138ec47eec6cd60d2ad61cad8b56d Author: Stefan Negrea snegrea@redhat.com Date: Sat Aug 17 19:25:18 2013 -0500
[BZ 993513] Baselines entries are no longer calculated and inserted into the SQL database if there is no data in the storage.
The sql query that returns the schedules that need recomputation has been updated to return only the ids of the schedules, reducing the amount of data requested from SQL to the minimum.
diff --git a/modules/core/domain/src/main/java/org/rhq/core/domain/measurement/MeasurementBaseline.java b/modules/core/domain/src/main/java/org/rhq/core/domain/measurement/MeasurementBaseline.java index 5bd0e8b..681914d 100644 --- a/modules/core/domain/src/main/java/org/rhq/core/domain/measurement/MeasurementBaseline.java +++ b/modules/core/domain/src/main/java/org/rhq/core/domain/measurement/MeasurementBaseline.java @@ -94,6 +94,8 @@ public class MeasurementBaseline implements Serializable { @OneToOne(fetch = FetchType.LAZY, optional = false) private MeasurementSchedule schedule;
+ @Column(name = "SCHEDULE_ID", insertable = false, updatable = false) + private int scheduleId;
public MeasurementBaseline() { computeTime = System.currentTimeMillis(); @@ -176,6 +178,20 @@ public class MeasurementBaseline implements Serializable { }
/** + * @return the scheduleId + */ + public int getScheduleId() { + return scheduleId; + } + + /** + * @param scheduleId the scheduleId to set + */ + public void setScheduleId(int scheduleId) { + this.scheduleId = scheduleId; + } + + /** * If <code>true</code>, it means a user manually entered the baseline values, as opposed to having them * automatically be calculated by examining past measurement data. * diff --git a/modules/enterprise/server/jar/src/main/java/org/rhq/enterprise/server/measurement/MeasurementBaselineManagerBean.java b/modules/enterprise/server/jar/src/main/java/org/rhq/enterprise/server/measurement/MeasurementBaselineManagerBean.java index 90ac6e7..ed8db3a 100644 --- a/modules/enterprise/server/jar/src/main/java/org/rhq/enterprise/server/measurement/MeasurementBaselineManagerBean.java +++ b/modules/enterprise/server/jar/src/main/java/org/rhq/enterprise/server/measurement/MeasurementBaselineManagerBean.java @@ -18,10 +18,9 @@ */ package org.rhq.enterprise.server.measurement;
+import java.util.ArrayList; import java.util.Arrays; import java.util.Date; -import java.util.Iterator; -import java.util.LinkedList; import java.util.List; import java.util.Properties;
@@ -48,10 +47,10 @@ import org.rhq.enterprise.server.RHQConstants; import org.rhq.enterprise.server.auth.SubjectManagerLocal; import org.rhq.enterprise.server.authz.AuthorizationManagerLocal; import org.rhq.enterprise.server.authz.PermissionException; -import org.rhq.enterprise.server.storage.StorageClientManagerBean; import org.rhq.enterprise.server.cloud.StatusManagerLocal; import org.rhq.enterprise.server.measurement.instrumentation.MeasurementMonitor; import org.rhq.enterprise.server.resource.ResourceManagerLocal; +import org.rhq.enterprise.server.storage.StorageClientManagerBean; import org.rhq.enterprise.server.system.SystemManagerLocal; import org.rhq.server.metrics.MetricsBaselineCalculator;
@@ -148,8 +147,6 @@ public class MeasurementBaselineManagerBean implements MeasurementBaselineManage + (System.currentTimeMillis() - now) + ")ms");
now = System.currentTimeMillis(); - int totalProcessed = 0; - while (true) { /* * each call is done in a separate xtn of at most 100K inserted rows; this helps to keep the xtn * shorter to avoid timeouts in scenarios where baseline calculations bunch together. the idea was that @@ -189,16 +186,22 @@ public class MeasurementBaselineManagerBean implements MeasurementBaselineManage * In any event, an appropriate chunking solution needs to be found, and that partitioning strategy * needs to replace the limits in the query today. */ - List<MeasurementSchedule> schedulesWithoutBaselines = - measurementBaselineManager.getSchedulesWithoutBaselines(); - measurementBaselineManager.calculateBaselines(schedulesWithoutBaselines, now, amountOfData); - totalProcessed += schedulesWithoutBaselines.size(); - - if (schedulesWithoutBaselines.size() < BASELINE_PROCESSING_LIMIT) { - break; + List<Integer> schedulesWithoutBaselines = measurementBaselineManager.getSchedulesWithoutBaselines(); + + List<Integer> accumulator = new ArrayList<Integer>(); + for (Integer value : schedulesWithoutBaselines) { + accumulator.add(value); + if (accumulator.size() == BASELINE_PROCESSING_LIMIT) { + measurementBaselineManager.calculateBaselines(accumulator, now, amountOfData); + accumulator.clear(); } } - log.info("Calculated and inserted [" + totalProcessed + "] new baselines. (" + if (!accumulator.isEmpty()) { + measurementBaselineManager.calculateBaselines(accumulator, now, amountOfData); + accumulator.clear(); + } + + log.info("Calculated and inserted [" + schedulesWithoutBaselines.size() + "] new baselines. (" + (System.currentTimeMillis() - now) + ")ms");
MeasurementMonitor.getMBean().incrementBaselineCalculationTime(System.currentTimeMillis() - computeTime); @@ -222,18 +225,17 @@ public class MeasurementBaselineManagerBean implements MeasurementBaselineManage
@SuppressWarnings("unchecked") @TransactionAttribute(TransactionAttributeType.REQUIRES_NEW) - public List<MeasurementSchedule> getSchedulesWithoutBaselines() { - String sql = - "SELECT s.* FROM rhq_measurement_sched s INNER JOIN rhq_measurement_def d ON s.definition = d.id " + - "LEFT JOIN rhq_measurement_bline b ON s.id = b.schedule_id WHERE b.schedule_id IS NULL AND d.numeric_type = 0"; - Query query = this.entityManager.createNativeQuery(sql, MeasurementSchedule.class); - query.setMaxResults(BASELINE_PROCESSING_LIMIT); + public List<Integer> getSchedulesWithoutBaselines() { + final String sql = + "SELECT s.id FROM rhq_measurement_sched s INNER JOIN rhq_measurement_def d ON s.definition = d.id " + + "LEFT JOIN rhq_measurement_bline b ON s.id = b.schedule_id WHERE s.enabled = true AND b.schedule_id IS NULL AND d.numeric_type = 0"; + Query query = this.entityManager.createNativeQuery(sql);
return query.getResultList(); }
@TransactionAttribute(TransactionAttributeType.NOT_SUPPORTED) - public void calculateBaselines(List<MeasurementSchedule> schedules, long olderThan, long amountOfData) { + public void calculateBaselines(List<Integer> schedules, long olderThan, long amountOfData) { long endTime = olderThan; long startTime = endTime - amountOfData;
@@ -242,30 +244,21 @@ public class MeasurementBaselineManagerBean implements MeasurementBaselineManage long calcStartTime = System.currentTimeMillis(); List<MeasurementBaseline> results = baselineCalculator.calculateBaselines(schedules, startTime, endTime); long calcEndTime = System.currentTimeMillis(); - int count = results.size();
if (log.isDebugEnabled()) { - log.debug("Finished computing " + count + " new baselines in " + (calcEndTime - calcStartTime) + " ms"); + log.debug("Finished computing " + results.size() + " new baselines in " + (calcEndTime - calcStartTime) + + " ms"); }
log.debug("Persisting baselines calculations"); long saveStartTime = System.currentTimeMillis(); - Iterator<MeasurementBaseline> iterator = results.iterator(); - List<MeasurementBaseline> queue = new LinkedList<MeasurementBaseline>(); - while (iterator.hasNext()) { - if (queue.size() == 10) { - measurementBaselineManager.saveNewBaselines(queue); - queue = new LinkedList<MeasurementBaseline>(); - } - queue.add(iterator.next()); - } - if (!queue.isEmpty()) { - measurementBaselineManager.saveNewBaselines(queue); - } + + measurementBaselineManager.saveNewBaselines(results);
long saveEndTime = System.currentTimeMillis(); if (log.isDebugEnabled()) { - log.debug("Finished persisting " + count + " baselines in " + (saveEndTime - saveStartTime) + " ms"); + log.debug("Finished persisting " + results.size() + " baselines in " + (saveEndTime - saveStartTime) + + " ms"); } }
@@ -273,7 +266,10 @@ public class MeasurementBaselineManagerBean implements MeasurementBaselineManage @TransactionAttribute(TransactionAttributeType.REQUIRES_NEW) public void saveNewBaselines(List<MeasurementBaseline> baselines) { for (MeasurementBaseline baseline : baselines) { - entityManager.merge(baseline); + MeasurementSchedule schedule = new MeasurementSchedule(); + schedule.setId(baseline.getScheduleId()); + baseline.setSchedule(schedule); + this.entityManager.merge(baseline); } }
diff --git a/modules/enterprise/server/jar/src/main/java/org/rhq/enterprise/server/measurement/MeasurementBaselineManagerLocal.java b/modules/enterprise/server/jar/src/main/java/org/rhq/enterprise/server/measurement/MeasurementBaselineManagerLocal.java index af6145a..35b8099 100644 --- a/modules/enterprise/server/jar/src/main/java/org/rhq/enterprise/server/measurement/MeasurementBaselineManagerLocal.java +++ b/modules/enterprise/server/jar/src/main/java/org/rhq/enterprise/server/measurement/MeasurementBaselineManagerLocal.java @@ -24,7 +24,6 @@ import javax.ejb.Local;
import org.rhq.core.domain.auth.Subject; import org.rhq.core.domain.measurement.MeasurementBaseline; -import org.rhq.core.domain.measurement.MeasurementSchedule; import org.rhq.core.domain.resource.Resource;
/** @@ -73,7 +72,7 @@ public interface MeasurementBaselineManagerLocal { * will be called repeatedly during baseline calculations to get all of the necessary * schedules. */ - List<MeasurementSchedule> getSchedulesWithoutBaselines(); + List<Integer> getSchedulesWithoutBaselines();
/** * Given a list of schedules, this method calculates and stores baselines using the @@ -87,7 +86,7 @@ public interface MeasurementBaselineManagerLocal { * is treated as a duration. For example, a value of 259200000 * would be treated as 3 days. */ - void calculateBaselines(List<MeasurementSchedule> schedules, long olderThan, long amountOfData); + void calculateBaselines(List<Integer> schedules, long olderThan, long amountOfData);
/** * Persists the newly calculated baselines. @@ -148,7 +147,7 @@ public interface MeasurementBaselineManagerLocal { /** * Return a list of {@link MeasurementBaseline} objects for the {@link Resource} represented by the given id. * - * @param subject the user request to view the baseline history for the given resource + * @param subject the user request to view the baseline history for the given resource * @param resourceId the id of the resource whose baselines are to be returned * * @return a list of baselines for all measurements scheduled on the given resource diff --git a/modules/enterprise/server/server-metrics/src/main/java/org/rhq/server/metrics/MetricsBaselineCalculator.java b/modules/enterprise/server/server-metrics/src/main/java/org/rhq/server/metrics/MetricsBaselineCalculator.java index 6242df6..ef7d092 100644 --- a/modules/enterprise/server/server-metrics/src/main/java/org/rhq/server/metrics/MetricsBaselineCalculator.java +++ b/modules/enterprise/server/server-metrics/src/main/java/org/rhq/server/metrics/MetricsBaselineCalculator.java @@ -31,7 +31,6 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory;
import org.rhq.core.domain.measurement.MeasurementBaseline; -import org.rhq.core.domain.measurement.MeasurementSchedule; import org.rhq.server.metrics.domain.AggregateSimpleNumericMetric; import org.rhq.server.metrics.domain.AggregateType;
@@ -48,12 +47,12 @@ public class MetricsBaselineCalculator { this.metricsDAO = metricsDAO; }
- public List<MeasurementBaseline> calculateBaselines(List<MeasurementSchedule> schedules, long startTime, + public List<MeasurementBaseline> calculateBaselines(List<Integer> schedules, long startTime, long endTime) { List<MeasurementBaseline> calculatedBaselines = new ArrayList<MeasurementBaseline>();
MeasurementBaseline measurementBaseline; - for (MeasurementSchedule schedule : schedules) { + for (Integer schedule : schedules) { measurementBaseline = this.calculateBaseline(schedule, startTime, endTime); if (measurementBaseline != null) { calculatedBaselines.add(measurementBaseline); @@ -63,9 +62,9 @@ public class MetricsBaselineCalculator { return calculatedBaselines; }
- private MeasurementBaseline calculateBaseline(MeasurementSchedule schedule, long startTime, long endTime) { - Iterable<AggregateSimpleNumericMetric> metrics = this.metricsDAO.findAggregatedSimpleOneHourMetric( - schedule.getId(), startTime, endTime); + private MeasurementBaseline calculateBaseline(Integer schedule, long startTime, long endTime) { + Iterable<AggregateSimpleNumericMetric> metrics = this.metricsDAO.findAggregatedSimpleOneHourMetric(schedule, + startTime, endTime);
if (metrics != null && metrics.iterator() != null && metrics.iterator().hasNext()) { ArithmeticMeanCalculator mean = new ArithmeticMeanCalculator(); @@ -104,7 +103,7 @@ public class MetricsBaselineCalculator { baseline.setMax(max); baseline.setMin(min); baseline.setMean(mean.getArithmeticMean()); - baseline.setSchedule(schedule); + baseline.setScheduleId(schedule);
if (log.isDebugEnabled()) { log.debug("Calculated baseline: " + baseline.toString()); @@ -113,12 +112,6 @@ public class MetricsBaselineCalculator { return baseline; }
- MeasurementBaseline baseline = new MeasurementBaseline(); - baseline.setMax(Double.NaN); - baseline.setMin(Double.NaN); - baseline.setMean(Double.NaN); - baseline.setSchedule(schedule); - - return baseline; + return null; } } diff --git a/modules/enterprise/server/server-metrics/src/test/java/org/rhq/server/metrics/MetricsBaselineCalculatorTest.java b/modules/enterprise/server/server-metrics/src/test/java/org/rhq/server/metrics/MetricsBaselineCalculatorTest.java index 2b58222..1137dca 100644 --- a/modules/enterprise/server/server-metrics/src/test/java/org/rhq/server/metrics/MetricsBaselineCalculatorTest.java +++ b/modules/enterprise/server/server-metrics/src/test/java/org/rhq/server/metrics/MetricsBaselineCalculatorTest.java @@ -27,7 +27,6 @@ import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.verifyNoMoreInteractions; import static org.mockito.Mockito.when; -import static org.rhq.test.AssertUtils.assertPropertiesMatch; import static org.testng.Assert.assertEquals;
import java.util.ArrayList; @@ -44,7 +43,6 @@ import org.testng.annotations.ObjectFactory; import org.testng.annotations.Test;
import org.rhq.core.domain.measurement.MeasurementBaseline; -import org.rhq.core.domain.measurement.MeasurementSchedule; import org.rhq.server.metrics.domain.AggregateSimpleNumericMetric; import org.rhq.server.metrics.domain.AggregateType;
@@ -66,7 +64,6 @@ public class MetricsBaselineCalculatorTest {
@Test public void noCalculationTest() throws Exception { - //tell the method story as it happens: mock dependencies and configure //those dependencies to get the method under test to completion. StorageSession mockSession = mock(StorageSession.class); @@ -74,34 +71,24 @@ public class MetricsBaselineCalculatorTest { PowerMockito.whenNew(MetricsDAO.class).withParameterTypes(StorageSession.class, MetricsConfiguration.class) .withArguments(eq(mockSession), eq(metricsConfiguration)).thenReturn(mockMetricsDAO);
- when(mockMetricsDAO.findAggregatedSimpleOneHourMetric(eq(1), eq(0), - eq(1))).thenReturn(new ArrayList<AggregateSimpleNumericMetric>()); + when(mockMetricsDAO.findAggregatedSimpleOneHourMetric(eq(1), eq(0), eq(1))).thenReturn( + new ArrayList<AggregateSimpleNumericMetric>());
- MeasurementSchedule mockSchedule = mock(MeasurementSchedule.class); - when(mockSchedule.getId()).thenReturn(0); + int expectedScheduleId = 2567;
//create object to test and inject required dependencies MetricsBaselineCalculator objectUnderTest = new MetricsBaselineCalculator(new MetricsDAO(mockSession, metricsConfiguration));
//run code under test - List<MeasurementBaseline> result = objectUnderTest.calculateBaselines(Arrays.asList(mockSchedule), 0, 1); + List<MeasurementBaseline> result = objectUnderTest.calculateBaselines(Arrays.asList(expectedScheduleId), 0, 1);
//verify the results (Assert and mock verification) - assertEquals(result.size(), 1, "Expected to get back one baseline"); - MeasurementBaseline expected = new MeasurementBaseline(); - expected.setSchedule(mockSchedule); - expected.setMax(Double.NaN); - expected.setMin(Double.NaN); - expected.setMean(Double.NaN); - - assertPropertiesMatch("", expected, result.get(0), "computeTime"); + assertEquals(result.size(), 0, "No baselines expected");
verify(mockMetricsDAO, times(1)).findAggregatedSimpleOneHourMetric(any(Integer.class), any(Integer.class), any(Integer.class)); verifyNoMoreInteractions(mockMetricsDAO); - - verify(mockSchedule, times(1)).getId(); }
@Test @@ -138,7 +125,7 @@ public class MetricsBaselineCalculatorTest { } }
- int expectedScheduleId= 567; + int expectedScheduleId = 1567; long expectedStartTime = 135; long expectedEndTime = 246; long beforeComputeTime = System.currentTimeMillis(); @@ -154,15 +141,12 @@ public class MetricsBaselineCalculatorTest { mockMetricsDAO.findAggregatedSimpleOneHourMetric(eq(expectedScheduleId), eq(expectedStartTime), eq(expectedEndTime))).thenReturn(randomData);
- MeasurementSchedule mockSchedule = mock(MeasurementSchedule.class); - when(mockSchedule.getId()).thenReturn(expectedScheduleId); - //create object to test and inject required dependencies MetricsBaselineCalculator objectUnderTest = new MetricsBaselineCalculator(new MetricsDAO(mockSession, metricsConfiguration));
//run code under test - List<MeasurementBaseline> result = objectUnderTest.calculateBaselines(Arrays.asList(mockSchedule), + List<MeasurementBaseline> result = objectUnderTest.calculateBaselines(Arrays.asList(expectedScheduleId), expectedStartTime, expectedEndTime);
//verify the results (Assert and mock verification) @@ -172,7 +156,7 @@ public class MetricsBaselineCalculatorTest { Assert.assertEquals(baselineResult.getMean(), average, TEST_PRECISION); Assert.assertEquals(baselineResult.getMax(), expectedMax, TEST_PRECISION); Assert.assertEquals(baselineResult.getMin(), expectedMin, TEST_PRECISION); - Assert.assertEquals(baselineResult.getSchedule(), mockSchedule); + Assert.assertEquals(baselineResult.getScheduleId(), expectedScheduleId); if (baselineResult.getComputeTime().getTime() > System.currentTimeMillis()) { Assert.fail("Back compute time, the computation was forward dated."); } @@ -183,10 +167,6 @@ public class MetricsBaselineCalculatorTest { verify(mockMetricsDAO, times(1)).findAggregatedSimpleOneHourMetric(eq(expectedScheduleId), eq(expectedStartTime), eq(expectedEndTime)); verifyNoMoreInteractions(mockMetricsDAO); - - verify(mockSchedule, times(2)).getId(); - verify(mockSchedule, times(1)).setBaseline(eq(baselineResult)); - verifyNoMoreInteractions(mockSchedule); }
@Test @@ -221,15 +201,12 @@ public class MetricsBaselineCalculatorTest { mockMetricsDAO.findAggregatedSimpleOneHourMetric(eq(expectedScheduleId), eq(expectedStartTime), eq(expectedEndTime))).thenReturn(randomData);
- MeasurementSchedule mockSchedule = mock(MeasurementSchedule.class); - when(mockSchedule.getId()).thenReturn(expectedScheduleId); - //create object to test and inject required dependencies MetricsBaselineCalculator objectUnderTest = new MetricsBaselineCalculator(new MetricsDAO(mockSession, metricsConfiguration));
//run code under test - List<MeasurementBaseline> result = objectUnderTest.calculateBaselines(Arrays.asList(mockSchedule), + List<MeasurementBaseline> result = objectUnderTest.calculateBaselines(Arrays.asList(expectedScheduleId), expectedStartTime, expectedEndTime);
//verify the results (Assert and mock verification) @@ -239,15 +216,10 @@ public class MetricsBaselineCalculatorTest { Assert.assertEquals(baselineResult.getMean(), average, TEST_PRECISION); Assert.assertEquals(baselineResult.getMax(), expectedMinMax); Assert.assertEquals(baselineResult.getMin(), expectedMinMax); - Assert.assertEquals(baselineResult.getSchedule(), mockSchedule); + Assert.assertEquals(baselineResult.getScheduleId(), expectedScheduleId);
verify(mockMetricsDAO, times(1)).findAggregatedSimpleOneHourMetric(eq(expectedScheduleId), eq(expectedStartTime), eq(expectedEndTime)); verifyNoMoreInteractions(mockMetricsDAO); - - verify(mockSchedule, times(2)).getId(); - verify(mockSchedule, times(1)).setBaseline(eq(baselineResult)); - verifyNoMoreInteractions(mockSchedule); } - }
rhq-commits@lists.fedorahosted.org