diff --git a/src/main/java/info/peper/vz/rest/FillAggregateTableMain3.java b/src/main/java/info/peper/vz/rest/FillAggregateTableMain3.java index b364a89..f40fe6d 100644 --- a/src/main/java/info/peper/vz/rest/FillAggregateTableMain3.java +++ b/src/main/java/info/peper/vz/rest/FillAggregateTableMain3.java @@ -23,40 +23,29 @@ import info.peper.vz.rest.bo.db.Aggregate2; public class FillAggregateTableMain3 { private static final DateTimeFormatter DTF = DateTimeFormatter.ISO_LOCAL_DATE_TIME.withZone(ZoneId.systemDefault()); - private static final List SAVELIST = new LinkedList(); public static void main(String[] args) throws Exception { try (final Connection con = DriverManager.getConnection( "jdbc:mariadb://mariadb.fritz.box/volkszaehler", "vz", getPassword())) { -// long startTimeStamp = getTimestamp("2025-01-01T09:00:00"); final long startTimeStamp = getTimestamp("2022-05-20T09:00:00"); // final long finalEndTimeStamp = getTimestamp("2022-05-20T09:02:00"); final long finalEndTimeStamp = getTimestamp("2025-03-01T00:00:00"); - for (int channelId : CHANNELS_TO_USE) { - processChannel(startTimeStamp, finalEndTimeStamp, 60*1000, channelId, con); - } - System.out.println("Saving... " + SAVELIST.size()); - saveValues(con); -// for (AggregateToSave ats : SAVELIST) { -// saveValues(con, ats.startTs, ats.endTs, ats.channelId, new long[] {ats.valuePos, ats.valueNeg}); -// System.out.println("####"); -// } + + final List aggregates = processHouse(1, startTimeStamp, finalEndTimeStamp, 60*1000, 4, 1, con); + saveValues(con, aggregates, 1); } } - private static void processHouse(final long startTimestamp, + private static List processHouse(final int houseId, + final long startTimestamp, final long endTimestamp, final long interval, final int channelIdSolar, final int channelIdMeter, final Connection con) throws SQLException { - long currentTimestamp = startTimestamp; - long intervalStartTimestamp = startTimestamp; - long intervalEndTimestamp = startTimestamp + interval; - long intervalPos = 0; - long intervalNeg = 0; + final List result = new LinkedList(); try (final PreparedStatement stmtSolar = con.prepareStatement("SELECT * FROM volkszaehler.data WHERE channel_id=? AND timestamp>? ORDER BY timestamp;"); final PreparedStatement stmtMeter = con.prepareStatement("SELECT * FROM volkszaehler.data WHERE channel_id=? AND timestamp>? ORDER BY timestamp;");) { stmtSolar.setInt(1, channelIdSolar); @@ -66,60 +55,102 @@ public class FillAggregateTableMain3 { stmtMeter.setLong(2, startTimestamp); stmtMeter.setFetchSize(1000); - - boolean noNextRecord = false; - - - - try (final ResultSet rs = stmtSolar.executeQuery()) { - long sumDiff = 0; - while (noNextRecord || (currentTimestamp <= endTimestamp && rs.next())) { - noNextRecord = false; - final long timestamp = rs.getLong("timestamp"); - final double value; - if (rs.getDouble("value") >= 0 && rs.getDouble("value") < 1) { - value = 0.0; - } else { - value = rs.getDouble("value"); - } - final long tsDiff; - if (timestamp < intervalEndTimestamp) { - tsDiff = timestamp - currentTimestamp; - } else { - tsDiff = intervalEndTimestamp - currentTimestamp; - } -// final long tsDiff = Math.min(timestamp - currentTimestamp, intervalEndTimestamp - currentTimestamp); - sumDiff += tsDiff; - currentTimestamp = timestamp; - if (value > 0) { - intervalPos += Math.round(value * tsDiff); - } else if (value < 0) { - intervalNeg += Math.round(-value * tsDiff); - } -// System.out.println("ts: " + currentTimestamp + " / diff: " + tsDiff + " / value: " + value + " / intervalPos: " + intervalPos); - if (timestamp >= intervalEndTimestamp) { - if (sumDiff != interval) { - System.err.println("sumDiff: " + sumDiff + " / interval: " + interval); + try (final ResultSet rsSolar = stmtSolar.executeQuery(); final ResultSet rsMeter = stmtMeter.executeQuery()) { + System.out.println("Starting..."); + long intervalProduced = 0; + long intervalObtained = 0; + long intervalInjected = 0; + + long currentTsSolar = startTimestamp; + long currentTsMeter = startTimestamp; + + long intervalStartTimestamp = startTimestamp; + long intervalEndTimestamp = startTimestamp + interval; + + boolean noNextRecordSolar = false; + boolean noNextRecordMeter = false; + + while (currentTsSolar <= endTimestamp || currentTsMeter <= endTimestamp) { + + System.out.print(intervalStartTimestamp + ": "); + + while (currentTsSolar < intervalEndTimestamp && (noNextRecordSolar || rsSolar.next())) { + if (noNextRecordSolar) { + noNextRecordSolar = false; } - sumDiff = 0; - if (intervalStartTimestamp % (86400000*10) == 0) { - System.out.println( - DTF.format(Instant.ofEpochMilli(intervalStartTimestamp)) + " - " + - DTF.format(Instant.ofEpochMilli(intervalEndTimestamp)) + " / " + - (intervalPos/3600) + " / " + - (intervalNeg/3600)); + + final long tsSolar = rsSolar.getLong("timestamp"); + final double value = rsSolar.getDouble("value") > 1.0 ? rsSolar.getDouble("value") : 0.0; + final long tsDiff = (tsSolar < intervalEndTimestamp) ? tsSolar - currentTsSolar : intervalEndTimestamp - currentTsSolar; + + currentTsSolar = tsSolar; + + intervalProduced += Math.round(value * tsDiff); + + if (currentTsSolar >= intervalEndTimestamp) { + noNextRecordSolar = true; } - SAVELIST.add(new AggregateToSave(intervalStartTimestamp, intervalEndTimestamp, Math.round(intervalPos), Math.round(intervalNeg), channelId)); - intervalStartTimestamp += interval; - intervalEndTimestamp += interval; - currentTimestamp = intervalStartTimestamp; - intervalPos = 0; - intervalNeg = 0; - noNextRecord = true; + System.out.print("."); } + + while (currentTsMeter < intervalEndTimestamp && (noNextRecordMeter || rsMeter.next())) { + if (noNextRecordMeter) { + noNextRecordMeter = false; + } + final long tsMeter = rsMeter.getLong("timestamp"); + final double value = rsMeter.getDouble("value"); + final long tsDiff = (tsMeter < intervalEndTimestamp) ? tsMeter - currentTsMeter : intervalEndTimestamp - currentTsMeter; + + currentTsMeter = tsMeter; + + if (value > 0.0) { + intervalObtained += Math.round(value * tsDiff); + } else { + intervalInjected += Math.abs(Math.round(value * tsDiff)); + } + + if (currentTsMeter >= intervalEndTimestamp) { + noNextRecordMeter = true; + } + System.out.print("#"); + } + + System.out.println(" " + intervalProduced + " / " + intervalObtained + " / " + intervalInjected); + + result.add(new Aggregate2(houseId, intervalStartTimestamp, intervalEndTimestamp, intervalProduced, intervalObtained, intervalInjected)); + + intervalStartTimestamp += interval; + intervalEndTimestamp += interval; + + intervalInjected = 0; + intervalObtained = 0; + intervalProduced = 0; } } } + return result; + } + + private static void saveValues(final Connection con, final List aggregates, final int houseId) throws SQLException { + System.out.println("Saving entries... : " + aggregates.size()); + try (final PreparedStatement stmt = con.prepareStatement("INSERT INTO tobias_aggregate2 (house_id, timestamp_start, timestamp_end, produced_energy, obtained_energy, injected_energy) VALUES (?, ?, ?, ?, ?, ?)")) { + int i = 0; + for (Aggregate2 ats : aggregates) { + stmt.setInt(1, houseId); + stmt.setLong(2, ats.getTimestampStart()); + stmt.setLong(3, ats.getTimestampEnd()); + stmt.setLong(4, ats.getProducedEnergy()); + stmt.setLong(5, ats.getObtainedEnergy()); + stmt.setLong(6, ats.getInjectedEnergy()); + stmt.addBatch(); + i++; + if (i % 100000 == 0) { + System.out.println(i); + stmt.executeBatch(); + } + } + stmt.executeBatch(); + } } private static String getPassword() throws IOException { @@ -135,38 +166,4 @@ public class FillAggregateTableMain3 { return Instant.EPOCH.until(instant, ChronoUnit.MILLIS); } - private static void saveValues(final Connection con, - final long startTimestamp, - final long endTimestamp, - final int channelId, - final long[] values) throws SQLException { - try (final PreparedStatement stmt = con.prepareStatement("INSERT INTO tobias_aggregate (channel_id, timestamp_start, timestamp_end, sum_positive, sum_negative) VALUES (?, ?, ?, ?, ?)")) { - stmt.setInt(1, channelId); - stmt.setLong(2, startTimestamp); - stmt.setLong(3, endTimestamp); - stmt.setLong(4, values[0]); - stmt.setLong(5, values[1]); - stmt.execute(); - } - } - - private static void saveValues(final Connection con) throws SQLException { - try (final PreparedStatement stmt = con.prepareStatement("INSERT INTO tobias_aggregate (channel_id, timestamp_start, timestamp_end, sum_positive, sum_negative) VALUES (?, ?, ?, ?, ?)")) { - int i = 0; - for (AggregateToSave ats : SAVELIST) { - stmt.setInt(1, ats.channelId); - stmt.setLong(2, ats.startTs); - stmt.setLong(3, ats.endTs); - stmt.setDouble(4, ats.valuePos); - stmt.setDouble(5, ats.valueNeg); - stmt.addBatch(); - i++; - if (i % 100000 == 0) { - System.out.println(i); - stmt.executeBatch(); - } - } - stmt.executeBatch(); - } - } } diff --git a/src/main/java/info/peper/vz/rest/VzRestController.java b/src/main/java/info/peper/vz/rest/VzRestController.java index 028a4fc..5b72f0e 100644 --- a/src/main/java/info/peper/vz/rest/VzRestController.java +++ b/src/main/java/info/peper/vz/rest/VzRestController.java @@ -19,7 +19,9 @@ import org.springframework.web.bind.annotation.RequestParam; import org.springframework.web.bind.annotation.RestController; import info.peper.vz.rest.bo.Sums; +import info.peper.vz.rest.bo.Sums2; import info.peper.vz.rest.bo.db.Aggregate; +import info.peper.vz.rest.bo.db.Aggregate2; import info.peper.vz.rest.bo.db.Data; import info.peper.vz.rest.bo.db.EnergyPrice; @@ -79,6 +81,46 @@ class VzRestController { return new Sums(Math.round(sumPos), Math.round(sumNeg)); } + @GetMapping("/data/sums2") + Sums2 getData2(@RequestParam("timestampStart")final long timestampStart, + @RequestParam("timestampEnd")final long timestampEnd, + @RequestParam("houseId")final int houseId) { + final List aggregates = jdbcTemplate.query( + "SELECT MIN(timestamp_start) AS min_ts_start, MAX(timestamp_end) AS max_ts_end, SUM(produced_energy) AS sum_produced, SUM(obtained_energy) AS sum_obtained, SUM(injected_energy) AS sum_injected FROM volkszaehler.tobias_aggregate2 WHERE house_id=? AND timestamp_start>=? AND timestamp_end<=?;", + (rs, rowNum) -> new Aggregate2(houseId, rs.getLong("min_ts_start"), rs.getLong("max_ts_end"), + rs.getLong("sum_produced"), rs.getLong("sum_obtained"), rs.getLong("sum_injected")), + houseId ,timestampStart, timestampEnd); + if (aggregates.size() != 1) { + throw new RuntimeException("Interal error in SQL query."); + } + final Aggregate2 aggregate = aggregates.iterator().next(); + + double sumPos = aggregate.getSumPositive(); + double sumNeg = aggregate.getSumNegative(); + log.debug("sumPos: " + sumPos + " / sumNeg: " + sumNeg); + log.debug("timestampStart: " + timestampStart); + log.debug("timestampEnd: " + timestampEnd); + log.debug("firstTimestamp: " + aggregate.getTimestampStart()); + log.debug("lastTimestamp: " + aggregate.getTimestampEnd()); + + if (timestampStart < aggregate.getTimestampStart() ) { + final Sums startSums = getSums(timestampStart, aggregate.getTimestampStart(), channelId); + sumPos += startSums.getSumPositive(); + sumNeg += startSums.getSumNegative(); + log.debug("Start: " + startSums.toString()); + } + if (timestampEnd > aggregate.getTimestampEnd()) { + final Sums endSums = getSums(aggregate.getTimestampEnd(), timestampEnd, channelId); + sumPos += endSums.getSumPositive(); + sumNeg += endSums.getSumNegative(); + log.debug("End: " + endSums.toString()); + + } + sumPos /= 3600; + sumNeg /= 3600; + return new Sums(Math.round(sumPos), Math.round(sumNeg)); + } + @GetMapping("/data/sumsWithFactor") Sums getDataWithFactor(@RequestParam("timestampStart")final long timestampStart, @RequestParam("timestampEnd")final long timestampEnd, @@ -128,9 +170,8 @@ class VzRestController { return new Sums(Math.round(sumPos), Math.round(sumNeg)); } - - @GetMapping("/data/sums2") - Sums getData2(@RequestParam("timestampStart")final long timestampStart, + @GetMapping("/data/sumsWithoutAggregate") + Sums getDataWithoutAggregate(@RequestParam("timestampStart")final long timestampStart, @RequestParam("timestampEnd")final long timestampEnd, @RequestParam("channelId")final int channelId) { final Sums startSums = getSums(timestampStart, timestampEnd, channelId); diff --git a/src/main/java/info/peper/vz/rest/bo/Sums2.java b/src/main/java/info/peper/vz/rest/bo/Sums2.java new file mode 100644 index 0000000..e9dd4aa --- /dev/null +++ b/src/main/java/info/peper/vz/rest/bo/Sums2.java @@ -0,0 +1,51 @@ +package info.peper.vz.rest.bo; + +import java.io.Serializable; + +public class Sums2 implements Serializable { + private static final long serialVersionUID = -1816023197422851264L; + private final long injected; + private final long obtained; + private final long produced; + public Sums2(long injected, long obtained, long produced) { + super(); + this.injected = injected; + this.obtained = obtained; + this.produced = produced; + } + public long getInjected() { + return injected; + } + public long getObtained() { + return obtained; + } + public long getProduced() { + return produced; + } + @Override + public int hashCode() { + final int prime = 31; + int result = 1; + result = prime * result + (int) (injected ^ (injected >>> 32)); + result = prime * result + (int) (obtained ^ (obtained >>> 32)); + result = prime * result + (int) (produced ^ (produced >>> 32)); + return result; + } + @Override + public boolean equals(Object obj) { + if (this == obj) + return true; + if (obj == null) + return false; + if (getClass() != obj.getClass()) + return false; + Sums2 other = (Sums2) obj; + if (injected != other.injected) + return false; + if (obtained != other.obtained) + return false; + if (produced != other.produced) + return false; + return true; + } +}