Merge pull request #5035 from ibi-group/cleanup-gtfs-rt

Use HttpHeaders class for GTFS-RT, cleanup

Author: leonardehrenfried

docs/UpdaterConfig.md

@@ -44,7 +44,7 @@ The information is downloaded in a single HTTP request and polled regularly.
 | frequencySec              |    `integer`    | How often the URL should be fetched.                                         | *Optional* | `60`          |  1.5  |
 | fuzzyTripMatching         |    `boolean`    | Whether to match trips fuzzily.                                              | *Optional* | `false`       |  1.5  |
 | url                       |     `string`    | URL to fetch the GTFS-RT feed from.                                          | *Required* |               |  1.5  |
-| [headers](#u_0_headers)   | `map of string` | Extra headers to add to the HTTP request fetching the data.                  | *Optional* |               |  2.3  |
+| [headers](#u_0_headers)   | `map of string` | HTTP headers to add to the request. Any header key, value can be inserted.   | *Optional* |               |  2.3  |
 
 
 ##### Parameter details
@@ -54,7 +54,7 @@ The information is downloaded in a single HTTP request and polled regularly.
 **Since version:** `2.3` ∙ **Type:** `map of string` ∙ **Cardinality:** `Optional`   
 **Path:** /updaters/[0] 
 
-Extra headers to add to the HTTP request fetching the data.
+HTTP headers to add to the request. Any header key, value can be inserted.
 
 
 
@@ -89,15 +89,15 @@ The information is downloaded in a single HTTP request and polled regularly.
 <!-- stop-time-updater BEGIN -->
 <!-- NOTE! This section is auto-generated. Do not change, change doc in code instead. -->
 
-| Config Parameter                                                      |       Type      | Summary                                                     |  Req./Opt. | Default Value        | Since |
-|-----------------------------------------------------------------------|:---------------:|-------------------------------------------------------------|:----------:|----------------------|:-----:|
-| type = "stop-time-updater"                                            |      `enum`     | The type of the updater.                                    | *Required* |                      |  1.5  |
-| [backwardsDelayPropagationType](#u__5__backwardsDelayPropagationType) |      `enum`     | How backwards propagation should be handled.                | *Optional* | `"required-no-data"` |  2.2  |
-| feedId                                                                |     `string`    | Which feed the updates apply to.                            | *Optional* |                      |  1.5  |
-| frequencySec                                                          |    `integer`    | How often the data should be downloaded in seconds.         | *Optional* | `60`                 |  1.5  |
-| fuzzyTripMatching                                                     |    `boolean`    | If the trips should be matched fuzzily.                     | *Optional* | `false`              |  1.5  |
-| url                                                                   |     `string`    | The URL of the GTFS-RT resource.                            | *Required* |                      |  1.5  |
-| [headers](#u__5__headers)                                             | `map of string` | Extra headers to add to the HTTP request fetching the data. | *Optional* |                      |  2.3  |
+| Config Parameter                                                      |       Type      | Summary                                                                    |  Req./Opt. | Default Value        | Since |
+|-----------------------------------------------------------------------|:---------------:|----------------------------------------------------------------------------|:----------:|----------------------|:-----:|
+| type = "stop-time-updater"                                            |      `enum`     | The type of the updater.                                                   | *Required* |                      |  1.5  |
+| [backwardsDelayPropagationType](#u__5__backwardsDelayPropagationType) |      `enum`     | How backwards propagation should be handled.                               | *Optional* | `"required-no-data"` |  2.2  |
+| feedId                                                                |     `string`    | Which feed the updates apply to.                                           | *Optional* |                      |  1.5  |
+| frequencySec                                                          |    `integer`    | How often the data should be downloaded in seconds.                        | *Optional* | `60`                 |  1.5  |
+| fuzzyTripMatching                                                     |    `boolean`    | If the trips should be matched fuzzily.                                    | *Optional* | `false`              |  1.5  |
+| [url](#u__5__url)                                                     |     `string`    | The URL of the GTFS-RT resource.                                           | *Required* |                      |  1.5  |
+| [headers](#u__5__headers)                                             | `map of string` | HTTP headers to add to the request. Any header key, value can be inserted. | *Optional* |                      |  2.3  |
 
 
 ##### Parameter details
@@ -124,12 +124,21 @@ How backwards propagation should be handled.
   The updated times are exposed through APIs.
 
 
+<h4 id="u__5__url">url</h4>
+
+**Since version:** `1.5` ∙ **Type:** `string` ∙ **Cardinality:** `Required`   
+**Path:** /updaters/[5] 
+
+The URL of the GTFS-RT resource.
+
+`file:` URLs are also supported if you want to read a file from the local disk.
+
 <h4 id="u__5__headers">headers</h4>
 
 **Since version:** `2.3` ∙ **Type:** `map of string` ∙ **Cardinality:** `Optional`   
 **Path:** /updaters/[5] 
 
-Extra headers to add to the HTTP request fetching the data.
+HTTP headers to add to the request. Any header key, value can be inserted.
 
 
 
@@ -214,13 +223,13 @@ The information is downloaded in a single HTTP request and polled regularly.
 <!-- vehicle-positions BEGIN -->
 <!-- NOTE! This section is auto-generated. Do not change, change doc in code instead. -->
 
-| Config Parameter           |       Type      | Summary                                                                   |  Req./Opt. | Default Value | Since |
-|----------------------------|:---------------:|---------------------------------------------------------------------------|:----------:|---------------|:-----:|
-| type = "vehicle-positions" |      `enum`     | The type of the updater.                                                  | *Required* |               |  1.5  |
-| feedId                     |     `string`    | Feed ID to which the update should be applied.                            | *Required* |               |  2.2  |
-| frequencySec               |    `integer`    | How often the positions should be updated.                                | *Optional* | `60`          |  2.2  |
-| url                        |      `uri`      | The URL of GTFS-RT protobuf HTTP resource to download the positions from. | *Required* |               |  2.2  |
-| [headers](#u__6__headers)  | `map of string` | Extra headers to add to the HTTP request fetching the data.               | *Optional* |               |  2.3  |
+| Config Parameter           |       Type      | Summary                                                                    |  Req./Opt. | Default Value | Since |
+|----------------------------|:---------------:|----------------------------------------------------------------------------|:----------:|---------------|:-----:|
+| type = "vehicle-positions" |      `enum`     | The type of the updater.                                                   | *Required* |               |  1.5  |
+| feedId                     |     `string`    | Feed ID to which the update should be applied.                             | *Required* |               |  2.2  |
+| frequencySec               |    `integer`    | How often the positions should be updated.                                 | *Optional* | `60`          |  2.2  |
+| url                        |      `uri`      | The URL of GTFS-RT protobuf HTTP resource to download the positions from.  | *Required* |               |  2.2  |
+| [headers](#u__6__headers)  | `map of string` | HTTP headers to add to the request. Any header key, value can be inserted. | *Optional* |               |  2.3  |
 
 
 ##### Parameter details
@@ -230,7 +239,7 @@ The information is downloaded in a single HTTP request and polled regularly.
 **Since version:** `2.3` ∙ **Type:** `map of string` ∙ **Cardinality:** `Optional`   
 **Path:** /updaters/[6] 
 
-Extra headers to add to the HTTP request fetching the data.
+HTTP headers to add to the request. Any header key, value can be inserted.
 
 
 

src/ext/java/org/opentripplanner/ext/siri/updater/SiriETHttpTripUpdateSource.java

@@ -67,7 +67,7 @@ public Siri getUpdates() {
         url,
         etServiceRequest,
         timeout,
-        requestHeaders.headers()
+        requestHeaders.asMap()
       );
       if (is != null) {
         // Decode message

src/ext/java/org/opentripplanner/ext/siri/updater/SiriSXUpdater.java

@@ -138,7 +138,7 @@ private Siri getUpdates() throws IOException {
         url,
         sxServiceRequest,
         timeout,
-        requestHeaders.headers()
+        requestHeaders.asMap()
       );
 
       fetching = System.currentTimeMillis() - t1;

src/ext/java/org/opentripplanner/ext/siri/updater/azure/SiriAzureETUpdater.java

@@ -93,7 +93,7 @@ protected void initializeData(String url, Consumer<ServiceBusReceivedMessageCont
       HttpHeaders rh = HttpHeaders.of().acceptApplicationXML().build();
 
       final long t1 = System.currentTimeMillis();
-      final InputStream data = HttpUtils.getData(uri, Duration.ofMillis(timeout), rh.headers());
+      final InputStream data = HttpUtils.getData(uri, Duration.ofMillis(timeout), rh.asMap());
       final long t2 = System.currentTimeMillis();
 
       if (data == null) {

src/ext/java/org/opentripplanner/ext/siri/updater/azure/SiriAzureSXUpdater.java

@@ -93,7 +93,7 @@ protected void initializeData(String url, Consumer<ServiceBusReceivedMessageCont
       // Maybe put this in the config?
       HttpHeaders rh = HttpHeaders.of().acceptApplicationXML().build();
 
-      final InputStream data = HttpUtils.getData(uri, Duration.ofMillis(timeout), rh.headers());
+      final InputStream data = HttpUtils.getData(uri, Duration.ofMillis(timeout), rh.asMap());
       final long t2 = System.currentTimeMillis();
 
       if (data == null) {

src/main/java/org/opentripplanner/standalone/config/routerconfig/updaters/GtfsRealtimeAlertsUpdaterConfig.java

@@ -28,11 +28,7 @@ public static GtfsRealtimeAlertsUpdaterParameters create(String configRef, NodeA
         .summary("Whether to match trips fuzzily.")
         .asBoolean(false),
       c.of("frequencySec").since(V1_5).summary("How often the URL should be fetched.").asInt(60),
-      c
-        .of("headers")
-        .since(V2_3)
-        .summary("Extra headers to add to the HTTP request fetching the data.")
-        .asStringMap()
+      HttpHeadersConfig.headers(c, V2_3)
     );
   }
 }

src/main/java/org/opentripplanner/standalone/config/routerconfig/updaters/PollingTripUpdaterConfig.java

@@ -4,35 +4,23 @@
 import static org.opentripplanner.standalone.config.framework.json.OtpVersion.V2_2;
 import static org.opentripplanner.standalone.config.framework.json.OtpVersion.V2_3;
 
-import org.opentripplanner.framework.application.OtpAppException;
 import org.opentripplanner.standalone.config.framework.json.NodeAdapter;
 import org.opentripplanner.updater.trip.BackwardsDelayPropagationType;
 import org.opentripplanner.updater.trip.PollingTripUpdaterParameters;
 
 public class PollingTripUpdaterConfig {
 
   public static PollingTripUpdaterParameters create(String configRef, NodeAdapter c) {
-    String file = null;
-    String url = null;
+    var url = c
+      .of("url")
+      .since(V1_5)
+      .summary("The URL of the GTFS-RT resource.")
+      .description(
+        "`file:` URLs are also supported if you want to read a file from the local disk."
+      )
+      .asString();
 
-    if (c.exist("file")) {
-      file = c.of("file").since(V1_5).summary("The path of the GTFS-RT file.").asString();
-    } else if (c.exist("url")) {
-      url = c.of("url").since(V1_5).summary("The URL of the GTFS-RT resource.").asString();
-    } else {
-      throw new OtpAppException(
-        "Need either 'url' or 'file' properties to configure " +
-        configRef +
-        " but received: " +
-        c.asText()
-      );
-    }
-
-    var headers = c
-      .of("headers")
-      .since(V2_3)
-      .summary("Extra headers to add to the HTTP request fetching the data.")
-      .asStringMap();
+    var headers = HttpHeadersConfig.headers(c, V2_3);
 
     return new PollingTripUpdaterParameters(
       configRef,
@@ -69,7 +57,6 @@ public static PollingTripUpdaterParameters create(String configRef, NodeAdapter
         .asEnum(BackwardsDelayPropagationType.REQUIRED_NO_DATA),
       c.of("feedId").since(V1_5).summary("Which feed the updates apply to.").asString(null),
       url,
-      file,
       headers
     );
   }

src/main/java/org/opentripplanner/standalone/config/routerconfig/updaters/VehiclePositionsUpdaterConfig.java

@@ -24,11 +24,7 @@ public static VehiclePositionsUpdaterParameters create(String updaterRef, NodeAd
       .since(V2_2)
       .summary("The URL of GTFS-RT protobuf HTTP resource to download the positions from.")
       .asUri();
-    var headers = c
-      .of("headers")
-      .since(V2_3)
-      .summary("Extra headers to add to the HTTP request fetching the data.")
-      .asStringMap();
+    var headers = HttpHeadersConfig.headers(c, V2_3);
     return new VehiclePositionsUpdaterParameters(updaterRef, feedId, url, frequencySec, headers);
   }
 }

src/main/java/org/opentripplanner/updater/alert/GtfsRealtimeAlertsUpdater.java

@@ -3,45 +3,30 @@
 import com.google.transit.realtime.GtfsRealtime.FeedMessage;
 import java.io.InputStream;
 import java.net.URI;
-import java.util.Map;
-import org.opentripplanner.framework.collection.MapUtils;
 import org.opentripplanner.framework.io.HttpUtils;
 import org.opentripplanner.framework.tostring.ToStringBuilder;
 import org.opentripplanner.routing.impl.TransitAlertServiceImpl;
 import org.opentripplanner.routing.services.TransitAlertService;
 import org.opentripplanner.transit.service.DefaultTransitService;
 import org.opentripplanner.transit.service.TransitModel;
 import org.opentripplanner.updater.GtfsRealtimeFuzzyTripMatcher;
+import org.opentripplanner.updater.spi.HttpHeaders;
 import org.opentripplanner.updater.spi.PollingGraphUpdater;
 import org.opentripplanner.updater.spi.WriteToGraphCallback;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 /**
  * GTFS-RT alerts updater
- * <p>
- * Usage example:
- *
- * <pre>
- * myalert.type = real-time-alerts
- * myalert.frequencySec = 60
- * myalert.url = http://host.tld/path
- * myalert.earlyStartSec = 3600
- * myalert.feedId = TA
- * </pre>
  */
 public class GtfsRealtimeAlertsUpdater extends PollingGraphUpdater implements TransitAlertProvider {
 
   private static final Logger LOG = LoggerFactory.getLogger(GtfsRealtimeAlertsUpdater.class);
-  public static final Map<String, String> DEFAULT_HEADERS = Map.of(
-    "Accept",
-    "application/x-google-protobuf, application/x-protobuf, application/protobuf, application/octet-stream, */*"
-  );
 
   private final String url;
   private final AlertsUpdateHandler updateHandler;
   private final TransitAlertService transitAlertService;
-  private final Map<String, String> headers;
+  private final HttpHeaders headers;
   private WriteToGraphCallback saveResultOnGraph;
   private Long lastTimestamp = Long.MIN_VALUE;
 
@@ -51,7 +36,7 @@ public GtfsRealtimeAlertsUpdater(
   ) {
     super(config);
     this.url = config.url();
-    this.headers = MapUtils.combine(config.headers(), DEFAULT_HEADERS);
+    this.headers = HttpHeaders.of().acceptProtobuf().add(config.headers()).build();
     TransitAlertService transitAlertService = new TransitAlertServiceImpl(transitModel);
 
     var fuzzyTripMatcher = config.fuzzyTripMatching()
@@ -90,7 +75,7 @@ public String toString() {
   @Override
   protected void runPolling() {
     try {
-      InputStream data = HttpUtils.getData(URI.create(url), this.headers);
+      InputStream data = HttpUtils.getData(URI.create(url), this.headers.asMap());
       if (data == null) {
         throw new RuntimeException("Failed to get data from url " + url);
       }

src/main/java/org/opentripplanner/updater/alert/GtfsRealtimeAlertsUpdaterParameters.java

@@ -1,6 +1,7 @@
 package org.opentripplanner.updater.alert;
 
 import java.util.Map;
+import org.opentripplanner.updater.spi.HttpHeaders;
 import org.opentripplanner.updater.spi.PollingGraphUpdaterParameters;
 
 public record GtfsRealtimeAlertsUpdaterParameters(
@@ -10,10 +11,6 @@ public record GtfsRealtimeAlertsUpdaterParameters(
   int earlyStartSec,
   boolean fuzzyTripMatching,
   int frequencySec,
-  Map<String, String> headers
+  HttpHeaders headers
 )
-  implements PollingGraphUpdaterParameters {
-  public GtfsRealtimeAlertsUpdaterParameters {
-    headers = Map.copyOf(headers);
-  }
-}
+  implements PollingGraphUpdaterParameters {}