Wp 6

.gitignore

@@ -26,3 +26,4 @@ tripAttraction.csv
 tripProduction.csv
 
 src/main/java/META-INF/MANIFEST.MF
+.Rproj.user

extensions/matsim2silo/src/main/java/de/tum/bgu/msm/matsim/MatsimScenarioAssembler.java

@@ -2,15 +2,28 @@
 
 import de.tum.bgu.msm.data.Day;
 import de.tum.bgu.msm.data.travelTimes.TravelTimes;
+import org.matsim.api.core.v01.Id;
 import org.matsim.api.core.v01.Scenario;
 import org.matsim.core.config.Config;
 
+import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 
 public interface MatsimScenarioAssembler {
 
     Scenario assembleScenario(Config initialMatsimConfig, int year, TravelTimes travelTimes);
+
+    // used default identifier to allow for using following method in only one class that implements MatsimScenarioAssembler.
+    default Scenario assembleScenarioWithPreviousPlans(
+            Scenario matsimScenario,
+            int year,
+            TravelTimes travelTimes,
+            HashMap<Id<org.matsim.api.core.v01.population.Person>, org.matsim.api.core.v01.population.Person> previousYearPopulation) {
+
+        throw new UnsupportedOperationException(
+                "This MatsimScenarioAssembler does not support previous-year plans.");
+    }
     Map<Day, Scenario> assembleMultiScenarios(Config initialMatsimConfig, int year, TravelTimes travelTimes);
 
 }

extensions/matsim2silo/src/main/java/de/tum/bgu/msm/matsim/MatsimSkimCreator.java

@@ -13,6 +13,7 @@
 import org.matsim.api.core.v01.Id;
 import org.matsim.api.core.v01.network.Network;
 import org.matsim.api.core.v01.network.Node;
+import org.matsim.api.core.v01.network.Link;
 import org.matsim.api.core.v01.population.Leg;
 import org.matsim.api.core.v01.population.PlanElement;
 import org.matsim.core.network.NetworkUtils;
@@ -24,16 +25,36 @@
 import org.matsim.facilities.Facility;
 import org.matsim.pt.transitSchedule.api.TransitStopFacility;
 
+import java.io.BufferedWriter;
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
 import java.util.*;
 
 public class MatsimSkimCreator {
 
     private final static Logger logger = LogManager.getLogger(MatsimSkimCreator.class);
 
     private MatsimData matsimData;
+    private final Path csvOutputDirectory;
+    private final Integer csvOutputYear;
 
     public MatsimSkimCreator(MatsimData provider) {
+        this(provider, null, null);
+    }
+
+    /**
+     * Creates a skim creator that also writes every generated skim to CSV.
+     * The CSV files use the project standard FROM,TO,VALUE format.
+     */
+    public MatsimSkimCreator(MatsimData provider, Path csvOutputDirectory) {
+        this(provider, csvOutputDirectory, null);
+    }
+
+    public MatsimSkimCreator(MatsimData provider, Path csvOutputDirectory, Integer csvOutputYear) {
         this.matsimData = provider;
+        this.csvOutputDirectory = csvOutputDirectory;
+        this.csvOutputYear = csvOutputYear;
     }
 
     public IndexedDoubleMatrix2D createCarSkim(Collection<? extends de.tum.bgu.msm.data.Id> zones, int numberOfThreads, double peakHour_s) {
@@ -77,10 +98,29 @@ public IndexedDoubleMatrix2D createCarSkim(Collection<? extends de.tum.bgu.msm.d
             });
         }
         executor.execute();
+        logZeroCarSkimCells(skim);
         assignIntrazonals(5, Float.MAX_VALUE, 0.66f, skim);
+        writeSkimToCsv(skim, "car");
         return skim;
     }
 
+    private void logZeroCarSkimCells(IndexedDoubleMatrix2D skim) {
+        long zeroCells = 0;
+        long zeroOffDiagonalCells = 0;
+        for (int row = 0; row < skim.rows(); row++) {
+            int origin = skim.getIdForInternalRowIndex(row);
+            for (int column = 0; column < skim.columns(); column++) {
+                int destination = skim.getIdForInternalColumnIndex(column);
+                if (skim.getIndexed(origin, destination) == 0.) {
+                    zeroCells++;
+                    if (origin != destination) zeroOffDiagonalCells++;
+                }
+            }
+        }
+        logger.warn("Car skim contains " + zeroCells + " zero cells, including "
+                + zeroOffDiagonalCells + " zero off-diagonal cells, before intrazonal post-processing.");
+    }
+
     public IndexedDoubleMatrix2D createPtSkim(Collection<? extends de.tum.bgu.msm.data.Id> zones, int numberOfThreads,
                                               double peakHour_s) {
         final int partitionSize = (int) ((double) zones.size() / numberOfThreads) + 1;
@@ -160,6 +200,7 @@ public IndexedDoubleMatrix2D createPtSkim(Collection<? extends de.tum.bgu.msm.da
         }
         executor.execute();
         assignIntrazonals(5, Float.MAX_VALUE, 0.66f, skim);
+        writeSkimToCsv(skim, "pt");
         return skim;
     }
 
@@ -210,6 +251,7 @@ public IndexedDoubleMatrix2D createTeleportedSkim(Collection<? extends de.tum.bg
         }
         executor.execute();
         assignIntrazonals(5, Float.MAX_VALUE, 0.66f, skim);
+        writeSkimToCsv(skim, mode);
         return skim;
     }
 
@@ -259,21 +301,52 @@ public IndexedDoubleMatrix2D createFreeSpeedFactorSkim(Collection<? extends de.t
         }
         executor.execute();
         assignIntrazonals(5, Float.MAX_VALUE, 0.66f, skim);
+        writeSkimToCsv(skim, "freeSpeedFactor");
         return skim;
     }
 
+    /**
+     * Writes a skim in long CSV format with one row per origin-destination pair.
+     */
+    public void writeSkimToCsv(IndexedDoubleMatrix2D skim, String name) {
+        if (csvOutputDirectory == null) {
+            return;
+        }
+
+        String yearSuffix = csvOutputYear == null ? "" : "_" + csvOutputYear;
+        Path outputFile = csvOutputDirectory.resolve(name + "Skim" + yearSuffix + ".csv");
+        try {
+            Files.createDirectories(csvOutputDirectory);
+            try (BufferedWriter writer = Files.newBufferedWriter(outputFile)) {
+                writer.write("FROM,TO,VALUE");
+                writer.newLine();
+                for (int row = 0; row < skim.rows(); row++) {
+                    int origin = skim.getIdForInternalRowIndex(row);
+                    for (int column = 0; column < skim.columns(); column++) {
+                        int destination = skim.getIdForInternalColumnIndex(column);
+                        writer.write(origin + "," + destination + "," + skim.getIndexed(origin, destination));
+                        writer.newLine();
+                    }
+                }
+            }
+            logger.info("Wrote " + name + " skim to " + outputFile);
+        } catch (IOException e) {
+            throw new RuntimeException("Could not write skim CSV to " + outputFile, e);
+        }
+    }
+
     //TODO: copied from MITO car skim updater...maybe provide a utility function there
     private void assignIntrazonals(int numberOfNeighbours, float maximumMinutes, float proportionOfTime, IndexedDoubleMatrix2D skim) {
-        int nonIntrazonalCounter = 0;
-        for (int i = 1; i < skim.columns(); i++) {
+        for (int i = 0; i < skim.columns(); i++) {
             int i_id = skim.getIdForInternalColumnIndex(i);
             double[] minTimeValues = new double[numberOfNeighbours];
             for (int k = 0; k < numberOfNeighbours; k++) {
                 minTimeValues[k] = maximumMinutes;
             }
             //find the  n closest neighbors - the lower travel time values in the matrix column
-            for (int j = 1; j < skim.rows(); j++) {
+            for (int j = 0; j < skim.rows(); j++) {
                 int j_id = skim.getIdForInternalRowIndex(j);
+                if (i_id == j_id) continue;
                 int minimumPosition = 0;
                 while (minimumPosition < numberOfNeighbours) {
                     if (minTimeValues[minimumPosition] > skim.getIndexed(i_id, j_id) && skim.getIndexed(i_id, j_id) != 0) {
@@ -292,19 +365,12 @@ private void assignIntrazonals(int numberOfNeighbours, float maximumMinutes, flo
             }
             globalMinTime = globalMinTime / numberOfNeighbours * proportionOfTime;
 
-            //fill with the calculated value the cells with zero
-            for (int j = 1; j < skim.rows(); j++) {
-                int j_id = skim.getIdForInternalColumnIndex(j);
-                if (skim.getIndexed(i_id, j_id) == 0) {
-                    skim.setIndexed(i_id, j_id, globalMinTime);
-                    if (i != j) {
-                        nonIntrazonalCounter++;
-                    }
-                }
+            // Only the diagonal is intrazonal. Keep off-diagonal zero values
+            // visible because they indicate missing or unreachable routes.
+            if (skim.getIndexed(i_id, i_id) == 0) {
+                skim.setIndexed(i_id, i_id, globalMinTime);
             }
         }
         logger.info("Calculated intrazonal times and distances using the " + numberOfNeighbours + " nearest neighbours.");
-        logger.info("The calculation of intrazonals has also assigned values for cells with travel time equal to 0, that are not intrazonal: (" +
-                nonIntrazonalCounter + " cases).");
     }
-}
+}

extensions/matsim2silo/src/main/java/de/tum/bgu/msm/matsim/MatsimTransportModel.java

@@ -36,7 +36,6 @@
 import org.matsim.core.config.ConfigUtils;
 import org.matsim.core.controler.Controler;
 import org.matsim.core.controler.ControlerDefaults;
-import org.matsim.core.controler.OutputDirectoryHierarchy;
 import org.matsim.core.router.util.TravelDisutility;
 import org.matsim.core.router.util.TravelTime;
 import org.matsim.core.scenario.ScenarioUtils;
@@ -47,10 +46,7 @@
 
 import java.io.File;
 
-import java.util.Collections;
-import java.util.HashSet;
-import java.util.Objects;
-import java.util.Set;
+import java.util.*;
 
 import static de.tum.bgu.msm.properties.modules.TransportModelPropertiesModule.*;
 
@@ -71,6 +67,8 @@ public final class MatsimTransportModel implements TransportModel {
 
     private MatsimScenarioAssembler scenarioAssembler;
 
+    protected HashMap<Id<org.matsim.api.core.v01.population.Person>, org.matsim.api.core.v01.population.Person> previousYearPopulation = new HashMap<>();
+
     public MatsimTransportModel(DataContainer dataContainer, Config matsimConfig,
                                 Properties properties, MatsimScenarioAssembler scenarioAssembler,
                                 MatsimData matsimData) {
@@ -102,7 +100,7 @@ public void setup() {
             runTransportModel(properties.main.startYear);
         } else {
             String eventsFile = properties.main.baseDirectory + properties.transportModel.matsimInitialEventsFile;
-            replayFromEvents(eventsFile);
+            replayFromEvents(eventsFile, properties.main.startYear);
         }
         logger.warn("### About to leave MatsimTransportModel#setup()." );
     }
@@ -134,7 +132,7 @@ private void runTransportModel(int year) {
             //if using the SimpleCommuteModeChoiceScenarioAssembler, we need some initial travel times (this will use an unlodaded network)
             TravelTime myTravelTime = SiloMatsimUtils.getAnEmptyNetworkTravelTime();
             TravelDisutility myTravelDisutility = SiloMatsimUtils.getAnEmptyNetworkTravelDisutility();
-            updateTravelTimes(myTravelTime, myTravelDisutility);
+            updateTravelTimes(myTravelTime, myTravelDisutility, year);
         }
 		{
 			Set<Id<org.matsim.api.core.v01.population.Person>> toRemove = new HashSet<>( assembledScenario.getPopulation().getPersons().keySet() );
@@ -149,6 +147,7 @@ private void runTransportModel(int year) {
 			}
 		}
 
+        // no input plans file - we first add silo agents to matsim
         for (Household household: dataContainer.getHouseholdDataManager().getHouseholds()) {
             for (Person pp : household.getPersons().values()) {
                 final int noHHAUtos = (int) household.getVehicles().stream().filter( vv -> vv.getType().equals( de.tum.bgu.msm.data.vehicle.VehicleType.CAR ) ).count();
@@ -179,7 +178,28 @@ private void runTransportModel(int year) {
 
         // TODO remove config argument as it is duplicate (cf. above)
 		// yyyyyy the following replaces "assembledScenario"!!!
-        assembledScenario = scenarioAssembler.assembleScenario(initialMatsimConfig, year, travelTimes);
+//        assembledScenario = scenarioAssembler.assembleScenario(
+//                assembledScenario.getConfig(),
+//                year,
+//                travelTimes);
+
+//
+
+        if (year == properties.main.baseYear || properties.transportModel.matsimStartOption != MatsimStartOptionIdentifier.HOT) {
+            // this is where we generate plans for silo agents
+            assembledScenario = scenarioAssembler.assembleScenario(
+                    assembledScenario.getConfig(),
+                    year,
+                    travelTimes);
+        } else {
+            // this is what we call in subsequent years - tries to identify silo agents that already have previous plan
+            assembledScenario = scenarioAssembler.assembleScenarioWithPreviousPlans(
+                    assembledScenario,
+                    year,
+                    travelTimes,
+                    previousYearPopulation);
+        }
+
 
         finalizeConfig(assembledScenario.getConfig(), year, Integer.toString(year));
 
@@ -191,9 +211,21 @@ private void runTransportModel(int year) {
         // Get travel Times from MATSim
         logger.warn("###################################################");
         logger.warn("Using MATSim to compute travel times from zone to zone.");
+
         TravelTime travelTime = controler.getLinkTravelTimes();
         TravelDisutility travelDisutility = controler.getTravelDisutilityFactory().createTravelDisutility(travelTime);
-        updateTravelTimes(travelTime, travelDisutility);
+        updateTravelTimes(travelTime, travelDisutility, year);
+
+        //don't actually need to check if not base year, since we save this after the simulation of first year,
+        // which is correct to then save "previous" year's plans
+//        if (year != properties.main.baseYear && properties.transportModel.transportModelIdentifier == TransportModelIdentifier.MATSIM) {
+
+        // here is where we add our current year silo agents to a map (may run into memory issues, so design may change).
+        previousYearPopulation.clear();
+            for (org.matsim.api.core.v01.population.Person person : assembledScenario.getPopulation().getPersons().values()) {
+                previousYearPopulation.put(person.getId(), person);
+            }
+
     }
 
     private void finalizeConfig(Config config, int runId, String dir) {
@@ -213,16 +245,16 @@ private void finalizeConfig(Config config, int runId, String dir) {
     /**
      * @param eventsFile
      */
-    private void replayFromEvents(String eventsFile) {
+    private void replayFromEvents(String eventsFile, int year) {
         Scenario scenario = ScenarioUtils.loadScenario(initialMatsimConfig);
         TravelTime travelTime = TravelTimeUtils.createTravelTimesFromEvents(scenario.getNetwork(), scenario.getConfig(), eventsFile);
         TravelDisutility travelDisutility = ControlerDefaults.createDefaultTravelDisutilityFactory(scenario).createTravelDisutility(travelTime);
-        updateTravelTimes(travelTime, travelDisutility);
+        updateTravelTimes(travelTime, travelDisutility, year);
     }
 
-    private void updateTravelTimes(TravelTime travelTime, TravelDisutility disutility) {
+    private void updateTravelTimes(TravelTime travelTime, TravelDisutility disutility, int year) {
         matsimData.update(disutility, travelTime);
-        internalTravelTimes.update(matsimData);
+        internalTravelTimes.update(matsimData, year);
         final TravelTimes mainTravelTimes = dataContainer.getTravelTimes();
 
         if (mainTravelTimes != this.internalTravelTimes && mainTravelTimes instanceof SkimTravelTimes) {
@@ -234,4 +266,5 @@ private void updateTravelTimes(TravelTime travelTime, TravelDisutility disutilit
                     dataContainer.getGeoData().getZones().values());
     }
     }
+
 }

extensions/matsim2silo/src/main/java/de/tum/bgu/msm/matsim/MatsimTravelTimesAndCosts.java

@@ -38,6 +38,7 @@
 import org.matsim.vehicles.VehicleUtils;
 import org.matsim.vehicles.VehiclesFactory;
 
+import java.nio.file.Paths;
 import java.util.*;
 
 /**
@@ -59,6 +60,7 @@ public final class MatsimTravelTimesAndCosts implements TravelTimes {
     private Collection<Region> regions;
 
     private final Config config;
+    private Integer skimOutputYear;
 
     public MatsimTravelTimesAndCosts(Config config) {
         this.config = config;
@@ -71,7 +73,12 @@ public void initialize(GeoData geoData, MatsimData matsimData) {
     }
 
     public void update(MatsimData matsimData) {
+        update(matsimData, null);
+    }
+
+    public void update(MatsimData matsimData, Integer year) {
         this.matsimData = matsimData;
+        this.skimOutputYear = year;
         this.tripRouter = matsimData.createTripRouter();
         this.skimsByMode.clear();
         this.travelTimesFromRegion.clear();
@@ -282,7 +289,10 @@ public IndexedDoubleMatrix2D getPeakSkim(String mode) {
             logger.info("Calculating skim matrix for mode " + mode +
                     " using " + Properties.get().main.numberOfThreads + " threads.");
             IndexedDoubleMatrix2D skim;
-            final MatsimSkimCreator matsimSkimCreator = new MatsimSkimCreator(matsimData);
+            final String csvDirectory = Properties.get().main.baseDirectory + "scenOutput/"
+                    + Properties.get().main.scenarioName + "/matsim/skims";
+            final MatsimSkimCreator matsimSkimCreator = new MatsimSkimCreator(matsimData,
+                    Paths.get(csvDirectory), skimOutputYear);
             switch (mode) {
                 case TransportMode.car:
                     skim = matsimSkimCreator.createCarSkim(zones.values(), Properties.get().main.numberOfThreads,
@@ -318,6 +328,7 @@ public TravelTimes duplicate() {
         matsimTravelTimesAndCosts.zones = this.zones;
         matsimTravelTimesAndCosts.regions = this.regions;
         matsimTravelTimesAndCosts.matsimData = matsimData;
+        matsimTravelTimesAndCosts.skimOutputYear = skimOutputYear;
         matsimTravelTimesAndCosts.tripRouter = matsimData.createTripRouter();
         matsimTravelTimesAndCosts.skimsByMode.putAll(this.skimsByMode);
         matsimTravelTimesAndCosts.travelTimesFromRegion.putAll(travelTimesFromRegion);