xds, rbac: build per route serverInterceptor for httpConfig (#8524)
diff --git a/xds/src/main/java/io/grpc/xds/FilterChainMatchingProtocolNegotiators.java b/xds/src/main/java/io/grpc/xds/FilterChainMatchingProtocolNegotiators.java
index 24cd4e9..b828b86 100644
--- a/xds/src/main/java/io/grpc/xds/FilterChainMatchingProtocolNegotiators.java
+++ b/xds/src/main/java/io/grpc/xds/FilterChainMatchingProtocolNegotiators.java
@@ -59,6 +59,7 @@
import java.util.Map;
import java.util.concurrent.Executor;
import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicReference;
import java.util.logging.Level;
import java.util.logging.Logger;
import javax.annotation.Nullable;
@@ -135,28 +136,29 @@
static final class FilterChainSelector {
public static final FilterChainSelector NO_FILTER_CHAIN = new FilterChainSelector(
- Collections.<FilterChain, ServerRoutingConfig>emptyMap(), null, null);
- private final Map<FilterChain, ServerRoutingConfig> routingConfigs;
+ Collections.<FilterChain, AtomicReference<ServerRoutingConfig>>emptyMap(),
+ null, new AtomicReference<ServerRoutingConfig>());
+ private final Map<FilterChain, AtomicReference<ServerRoutingConfig>> routingConfigs;
@Nullable
private final SslContextProviderSupplier defaultSslContextProviderSupplier;
@Nullable
- private final ServerRoutingConfig defaultRoutingConfig;
+ private final AtomicReference<ServerRoutingConfig> defaultRoutingConfig;
- FilterChainSelector(Map<FilterChain, ServerRoutingConfig> routingConfigs,
+ FilterChainSelector(Map<FilterChain, AtomicReference<ServerRoutingConfig>> routingConfigs,
@Nullable SslContextProviderSupplier defaultSslContextProviderSupplier,
- @Nullable ServerRoutingConfig defaultRoutingConfig) {
+ @Nullable AtomicReference<ServerRoutingConfig> defaultRoutingConfig) {
this.routingConfigs = checkNotNull(routingConfigs, "routingConfigs");
this.defaultSslContextProviderSupplier = defaultSslContextProviderSupplier;
- this.defaultRoutingConfig = defaultRoutingConfig;
+ this.defaultRoutingConfig = checkNotNull(defaultRoutingConfig, "defaultRoutingConfig");
}
@VisibleForTesting
- Map<FilterChain, ServerRoutingConfig> getRoutingConfigs() {
+ Map<FilterChain, AtomicReference<ServerRoutingConfig>> getRoutingConfigs() {
return routingConfigs;
}
@VisibleForTesting
- ServerRoutingConfig getDefaultRoutingConfig() {
+ AtomicReference<ServerRoutingConfig> getDefaultRoutingConfig() {
return defaultRoutingConfig;
}
@@ -189,7 +191,7 @@
return new SelectedConfig(
routingConfigs.get(selected), selected.getSslContextProviderSupplier());
}
- if (defaultRoutingConfig != null) {
+ if (defaultRoutingConfig.get() != null) {
return new SelectedConfig(defaultRoutingConfig, defaultSslContextProviderSupplier);
}
return null;
@@ -393,11 +395,11 @@
* The FilterChain level configuration.
*/
private static final class SelectedConfig {
- private final ServerRoutingConfig routingConfig;
+ private final AtomicReference<ServerRoutingConfig> routingConfig;
@Nullable
private final SslContextProviderSupplier sslContextProviderSupplier;
- private SelectedConfig(ServerRoutingConfig routingConfig,
+ private SelectedConfig(AtomicReference<ServerRoutingConfig> routingConfig,
@Nullable SslContextProviderSupplier sslContextProviderSupplier) {
this.routingConfig = checkNotNull(routingConfig, "routingConfig");
this.sslContextProviderSupplier = sslContextProviderSupplier;
diff --git a/xds/src/main/java/io/grpc/xds/XdsServerWrapper.java b/xds/src/main/java/io/grpc/xds/XdsServerWrapper.java
index 29821f2..e730150 100644
--- a/xds/src/main/java/io/grpc/xds/XdsServerWrapper.java
+++ b/xds/src/main/java/io/grpc/xds/XdsServerWrapper.java
@@ -22,6 +22,7 @@
import com.google.auto.value.AutoValue;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
import com.google.common.util.concurrent.SettableFuture;
import io.grpc.Attributes;
import io.grpc.InternalServerInterceptors;
@@ -87,8 +88,9 @@
}
});
- public static final Attributes.Key<ServerRoutingConfig> ATTR_SERVER_ROUTING_CONFIG =
- Attributes.Key.create("io.grpc.xds.ServerWrapper.serverRoutingConfig");
+ public static final Attributes.Key<AtomicReference<ServerRoutingConfig>>
+ ATTR_SERVER_ROUTING_CONFIG =
+ Attributes.Key.create("io.grpc.xds.ServerWrapper.serverRoutingConfig");
@VisibleForTesting
static final long RETRY_DELAY_NANOS = TimeUnit.MINUTES.toNanos(1);
@@ -346,6 +348,15 @@
@Nullable
private FilterChain defaultFilterChain;
private boolean stopped;
+ private final Map<FilterChain, AtomicReference<ServerRoutingConfig>> savedRdsRoutingConfigRef
+ = new HashMap<>();
+ private final ServerInterceptor noopInterceptor = new ServerInterceptor() {
+ @Override
+ public <ReqT, RespT> Listener<ReqT> interceptCall(ServerCall<ReqT, RespT> call,
+ Metadata headers, ServerCallHandler<ReqT, RespT> next) {
+ return next.startCall(call, headers);
+ }
+ };
private DiscoveryState(String resourceName) {
this.resourceName = checkNotNull(resourceName, "resourceName");
@@ -452,14 +463,16 @@
}
private void updateSelector() {
- Map<FilterChain, ServerRoutingConfig> filterChainRouting = new HashMap<>();
+ Map<FilterChain, AtomicReference<ServerRoutingConfig>> filterChainRouting = new HashMap<>();
+ savedRdsRoutingConfigRef.clear();
for (FilterChain filterChain: filterChains) {
filterChainRouting.put(filterChain, generateRoutingConfig(filterChain));
}
FilterChainSelector selector = new FilterChainSelector(
Collections.unmodifiableMap(filterChainRouting),
defaultFilterChain == null ? null : defaultFilterChain.getSslContextProviderSupplier(),
- defaultFilterChain == null ? null : generateRoutingConfig(defaultFilterChain));
+ defaultFilterChain == null ? new AtomicReference<ServerRoutingConfig>() :
+ generateRoutingConfig(defaultFilterChain));
List<SslContextProviderSupplier> toRelease = getSuppliersInUse();
filterChainSelectorManager.updateSelector(selector);
for (SslContextProviderSupplier e: toRelease) {
@@ -468,18 +481,84 @@
startDelegateServer();
}
- private ServerRoutingConfig generateRoutingConfig(FilterChain filterChain) {
+ private AtomicReference<ServerRoutingConfig> generateRoutingConfig(FilterChain filterChain) {
HttpConnectionManager hcm = filterChain.getHttpConnectionManager();
if (hcm.virtualHosts() != null) {
- return ServerRoutingConfig.create(hcm.httpFilterConfigs(),
- new AtomicReference<>(hcm.virtualHosts()));
+ ImmutableMap<Route, ServerInterceptor> interceptors = generatePerRouteInterceptors(
+ hcm.httpFilterConfigs(), hcm.virtualHosts());
+ return new AtomicReference<>(ServerRoutingConfig.create(hcm.virtualHosts(),interceptors));
} else {
RouteDiscoveryState rds = routeDiscoveryStates.get(hcm.rdsName());
checkNotNull(rds, "rds");
- return ServerRoutingConfig.create(hcm.httpFilterConfigs(), rds.savedVirtualHosts);
+ AtomicReference<ServerRoutingConfig> serverRoutingConfigRef = new AtomicReference<>();
+ if (rds.savedVirtualHosts != null) {
+ ImmutableMap<Route, ServerInterceptor> interceptors = generatePerRouteInterceptors(
+ hcm.httpFilterConfigs(), rds.savedVirtualHosts);
+ ServerRoutingConfig serverRoutingConfig =
+ ServerRoutingConfig.create(rds.savedVirtualHosts, interceptors);
+ serverRoutingConfigRef.set(serverRoutingConfig);
+ } else {
+ serverRoutingConfigRef.set(ServerRoutingConfig.FAILING_ROUTING_CONFIG);
+ }
+ savedRdsRoutingConfigRef.put(filterChain, serverRoutingConfigRef);
+ return serverRoutingConfigRef;
}
}
+ private ImmutableMap<Route, ServerInterceptor> generatePerRouteInterceptors(
+ List<NamedFilterConfig> namedFilterConfigs, List<VirtualHost> virtualHosts) {
+ ImmutableMap.Builder<Route, ServerInterceptor> perRouteInterceptors =
+ new ImmutableMap.Builder<>();
+ for (VirtualHost virtualHost : virtualHosts) {
+ for (Route route : virtualHost.routes()) {
+ List<ServerInterceptor> filterInterceptors = new ArrayList<>();
+ Map<String, FilterConfig> selectedOverrideConfigs =
+ new HashMap<>(virtualHost.filterConfigOverrides());
+ selectedOverrideConfigs.putAll(route.filterConfigOverrides());
+ for (NamedFilterConfig namedFilterConfig : namedFilterConfigs) {
+ FilterConfig filterConfig = namedFilterConfig.filterConfig;
+ Filter filter = filterRegistry.get(filterConfig.typeUrl());
+ if (filter instanceof ServerInterceptorBuilder) {
+ ServerInterceptor interceptor =
+ ((ServerInterceptorBuilder) filter).buildServerInterceptor(
+ filterConfig, selectedOverrideConfigs.get(namedFilterConfig.name));
+ if (interceptor != null) {
+ filterInterceptors.add(interceptor);
+ }
+ } else {
+ logger.log(Level.WARNING, "HttpFilterConfig(type URL: "
+ + filterConfig.typeUrl() + ") is not supported on server-side. "
+ + "Probably a bug at ClientXdsClient verification.");
+ }
+ }
+ ServerInterceptor interceptor = combineInterceptors(filterInterceptors);
+ perRouteInterceptors.put(route, interceptor);
+ }
+ }
+ return perRouteInterceptors.build();
+ }
+
+ private ServerInterceptor combineInterceptors(final List<ServerInterceptor> interceptors) {
+ if (interceptors.isEmpty()) {
+ return noopInterceptor;
+ }
+ if (interceptors.size() == 1) {
+ return interceptors.get(0);
+ }
+ return new ServerInterceptor() {
+ @Override
+ public <ReqT, RespT> Listener<ReqT> interceptCall(ServerCall<ReqT, RespT> call,
+ Metadata headers, ServerCallHandler<ReqT, RespT> next) {
+ // intercept forward
+ for (int i = interceptors.size() - 1; i >= 0; i--) {
+ next = InternalServerInterceptors.interceptCallHandlerCreate(
+ interceptors.get(i), next);
+ }
+ return next.startCall(call, headers);
+ }
+ };
+ }
+
private void handleConfigNotFound(StatusException exception) {
cleanUpRouteDiscoveryStates();
List<SslContextProviderSupplier> toRelease = getSuppliersInUse();
@@ -508,6 +587,7 @@
xdsClient.cancelRdsResourceWatch(rdsName, rdsState);
}
routeDiscoveryStates.clear();
+ savedRdsRoutingConfigRef.clear();
}
private List<SslContextProviderSupplier> getSuppliersInUse() {
@@ -544,8 +624,7 @@
private final class RouteDiscoveryState implements RdsResourceWatcher {
private final String resourceName;
- private AtomicReference<ImmutableList<VirtualHost>> savedVirtualHosts =
- new AtomicReference<>();
+ private ImmutableList<VirtualHost> savedVirtualHosts;
private boolean isPending = true;
private RouteDiscoveryState(String resourceName) {
@@ -560,7 +639,8 @@
if (!routeDiscoveryStates.containsKey(resourceName)) {
return;
}
- savedVirtualHosts.set(ImmutableList.copyOf(update.virtualHosts));
+ savedVirtualHosts = ImmutableList.copyOf(update.virtualHosts);
+ updateRdsRoutingConfig();
maybeUpdateSelector();
}
});
@@ -575,7 +655,8 @@
return;
}
logger.log(Level.WARNING, "Rds {0} unavailable", resourceName);
- savedVirtualHosts.set(null);
+ savedVirtualHosts = null;
+ updateRdsRoutingConfig();
maybeUpdateSelector();
}
});
@@ -596,6 +677,25 @@
});
}
+ private void updateRdsRoutingConfig() {
+ for (FilterChain filterChain : savedRdsRoutingConfigRef.keySet()) {
+ if (resourceName.equals(filterChain.getHttpConnectionManager().rdsName())) {
+ ServerRoutingConfig updatedRoutingConfig;
+ if (savedVirtualHosts == null) {
+ updatedRoutingConfig = ServerRoutingConfig.FAILING_ROUTING_CONFIG;
+ } else {
+ ImmutableMap<Route, ServerInterceptor> updatedInterceptors =
+ generatePerRouteInterceptors(
+ filterChain.getHttpConnectionManager().httpFilterConfigs(),
+ savedVirtualHosts);
+ updatedRoutingConfig = ServerRoutingConfig.create(savedVirtualHosts,
+ updatedInterceptors);
+ }
+ savedRdsRoutingConfigRef.get(filterChain).set(updatedRoutingConfig);
+ }
+ }
+ }
+
// Update the selector to use the most recently updated configs only after all rds have been
// discovered for the first time. Later changes on rds will be applied through virtual host
// list atomic ref.
@@ -632,18 +732,16 @@
@Override
public <ReqT, RespT> Listener<ReqT> interceptCall(ServerCall<ReqT, RespT> call,
Metadata headers, ServerCallHandler<ReqT, RespT> next) {
- ServerRoutingConfig routingConfig = call.getAttributes().get(ATTR_SERVER_ROUTING_CONFIG);
- if (routingConfig == null) {
- String errorMsg = "Missing xDS routing config.";
+ AtomicReference<ServerRoutingConfig> routingConfigRef =
+ call.getAttributes().get(ATTR_SERVER_ROUTING_CONFIG);
+ ServerRoutingConfig routingConfig = routingConfigRef == null ? null :
+ routingConfigRef.get();
+ if (routingConfig == null || routingConfig == ServerRoutingConfig.FAILING_ROUTING_CONFIG) {
+ String errorMsg = "Missing or broken xDS routing config: RDS config unavailable.";
call.close(Status.UNAVAILABLE.withDescription(errorMsg), new Metadata());
return new Listener<ReqT>() {};
}
- List<VirtualHost> virtualHosts = routingConfig.virtualHosts().get();
- if (virtualHosts == null) {
- String errorMsg = "Missing xDS routing config VirtualHosts due to RDS config unavailable.";
- call.close(Status.UNAVAILABLE.withDescription(errorMsg), new Metadata());
- return new Listener<ReqT>() {};
- }
+ List<VirtualHost> virtualHosts = routingConfig.virtualHosts();
VirtualHost virtualHost = RoutingUtils.findVirtualHostForHostName(
virtualHosts, call.getAuthority());
if (virtualHost == null) {
@@ -653,14 +751,11 @@
return new Listener<ReqT>() {};
}
Route selectedRoute = null;
- Map<String, FilterConfig> selectedOverrideConfigs =
- new HashMap<>(virtualHost.filterConfigOverrides());
MethodDescriptor<ReqT, RespT> method = call.getMethodDescriptor();
for (Route route : virtualHost.routes()) {
if (RoutingUtils.matchRoute(
route.routeMatch(), "/" + method.getFullMethodName(), headers, random)) {
selectedRoute = route;
- selectedOverrideConfigs.putAll(route.filterConfigOverrides());
break;
}
}
@@ -670,48 +765,12 @@
new Metadata());
return new ServerCall.Listener<ReqT>() {};
}
- List<ServerInterceptor> filterInterceptors = new ArrayList<>();
- for (NamedFilterConfig namedFilterConfig : routingConfig.httpFilterConfigs()) {
- FilterConfig filterConfig = namedFilterConfig.filterConfig;
- Filter filter = filterRegistry.get(filterConfig.typeUrl());
- if (filter instanceof ServerInterceptorBuilder) {
- ServerInterceptor interceptor =
- ((ServerInterceptorBuilder) filter).buildServerInterceptor(
- filterConfig, selectedOverrideConfigs.get(namedFilterConfig.name));
- if (interceptor != null) {
- filterInterceptors.add(interceptor);
- }
- } else {
- call.close(
- Status.UNAVAILABLE.withDescription("HttpFilterConfig(type URL: "
- + filterConfig.typeUrl() + ") is not supported on server-side."),
- new Metadata());
- return new Listener<ReqT>() {};
- }
+ ServerInterceptor routeInterceptor = noopInterceptor;
+ Map<Route, ServerInterceptor> perRouteInterceptors = routingConfig.interceptors();
+ if (perRouteInterceptors != null && perRouteInterceptors.get(selectedRoute) != null) {
+ routeInterceptor = perRouteInterceptors.get(selectedRoute);
}
- ServerInterceptor interceptor = combineInterceptors(filterInterceptors);
- return interceptor.interceptCall(call, headers, next);
- }
-
- private ServerInterceptor combineInterceptors(final List<ServerInterceptor> interceptors) {
- if (interceptors.isEmpty()) {
- return noopInterceptor;
- }
- if (interceptors.size() == 1) {
- return interceptors.get(0);
- }
- return new ServerInterceptor() {
- @Override
- public <ReqT, RespT> Listener<ReqT> interceptCall(ServerCall<ReqT, RespT> call,
- Metadata headers, ServerCallHandler<ReqT, RespT> next) {
- // intercept forward
- for (int i = interceptors.size() - 1; i >= 0; i--) {
- next = InternalServerInterceptors.interceptCallHandlerCreate(
- interceptors.get(i), next);
- }
- return next.startCall(call, headers);
- }
- };
+ return routeInterceptor.interceptCall(call, headers, next);
}
}
@@ -720,20 +779,24 @@
*/
@AutoValue
abstract static class ServerRoutingConfig {
- // Top level http filter configs.
- abstract ImmutableList<NamedFilterConfig> httpFilterConfigs();
+ @VisibleForTesting
+ static final ServerRoutingConfig FAILING_ROUTING_CONFIG = ServerRoutingConfig.create(
+ ImmutableList.<VirtualHost>of(), ImmutableMap.<Route, ServerInterceptor>of());
- abstract AtomicReference<ImmutableList<VirtualHost>> virtualHosts();
+ abstract ImmutableList<VirtualHost> virtualHosts();
+
+ // Prebuilt per route server interceptors from http filter configs.
+ abstract ImmutableMap<Route, ServerInterceptor> interceptors();
/**
* Server routing configuration.
* */
- public static ServerRoutingConfig create(List<NamedFilterConfig> httpFilterConfigs,
- AtomicReference<ImmutableList<VirtualHost>> virtualHosts) {
- checkNotNull(httpFilterConfigs, "httpFilterConfigs");
+ public static ServerRoutingConfig create(
+ ImmutableList<VirtualHost> virtualHosts,
+ ImmutableMap<Route, ServerInterceptor> interceptors) {
checkNotNull(virtualHosts, "virtualHosts");
- return new AutoValue_XdsServerWrapper_ServerRoutingConfig(
- ImmutableList.copyOf(httpFilterConfigs), virtualHosts);
+ checkNotNull(interceptors, "interceptors");
+ return new AutoValue_XdsServerWrapper_ServerRoutingConfig(virtualHosts, interceptors);
}
}
}
diff --git a/xds/src/test/java/io/grpc/xds/FilterChainMatchingProtocolNegotiatorsTest.java b/xds/src/test/java/io/grpc/xds/FilterChainMatchingProtocolNegotiatorsTest.java
index 5b7b1bf..32a7bc1 100644
--- a/xds/src/test/java/io/grpc/xds/FilterChainMatchingProtocolNegotiatorsTest.java
+++ b/xds/src/test/java/io/grpc/xds/FilterChainMatchingProtocolNegotiatorsTest.java
@@ -26,6 +26,7 @@
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
import com.google.common.util.concurrent.SettableFuture;
+import io.grpc.ServerInterceptor;
import io.grpc.internal.TestUtils.NoopChannelLogger;
import io.grpc.netty.GrpcHttp2ConnectionHandler;
import io.grpc.netty.InternalProtocolNegotiationEvent;
@@ -93,8 +94,12 @@
private static final String LOCAL_IP = "10.1.2.3"; // dest
private static final String REMOTE_IP = "10.4.2.3"; // source
private static final int PORT = 7000;
- private final ServerRoutingConfig noopConfig = ServerRoutingConfig.create(
- new ArrayList<NamedFilterConfig>(), new AtomicReference<ImmutableList<VirtualHost>>());
+ private final AtomicReference<ServerRoutingConfig> noopConfig = new AtomicReference<>(
+ ServerRoutingConfig.create(ImmutableList.<VirtualHost>of(),
+ ImmutableMap.<Route, ServerInterceptor>of()));
+ final SettableFuture<SslContextProviderSupplier> sslSet = SettableFuture.create();
+ final SettableFuture<AtomicReference<ServerRoutingConfig>> routingSettable =
+ SettableFuture.create();
@After
@SuppressWarnings("FutureReturnValueIgnored")
@@ -108,15 +113,14 @@
@Test
public void nofilterChainMatch_defaultSslContext() throws Exception {
- final SettableFuture<SslContextProviderSupplier> sslSet = SettableFuture.create();
- final SettableFuture<ServerRoutingConfig> routingSettable = SettableFuture.create();
ChannelHandler next = captureAttrHandler(sslSet, routingSettable);
when(mockDelegate.newHandler(grpcHandler)).thenReturn(next);
SslContextProviderSupplier defaultSsl = new SslContextProviderSupplier(createTls(),
tlsContextManager);
selectorManager.updateSelector(new FilterChainSelector(
- new HashMap<FilterChain, ServerRoutingConfig>(), defaultSsl, noopConfig));
+ new HashMap<FilterChain, AtomicReference<ServerRoutingConfig>>(),
+ defaultSsl, noopConfig));
FilterChainMatchingHandler filterChainMatchingHandler =
new FilterChainMatchingHandler(grpcHandler, selectorManager, mockDelegate);
setupChannel("172.168.1.1", "172.168.1.2", 80, filterChainMatchingHandler);
@@ -138,7 +142,8 @@
@Test
public void noFilterChainMatch_noDefaultSslContext() {
selectorManager.updateSelector(new FilterChainSelector(
- new HashMap<FilterChain, ServerRoutingConfig>(), null, null));
+ new HashMap<FilterChain, AtomicReference<ServerRoutingConfig>>(),
+ null, new AtomicReference<ServerRoutingConfig>()));
FilterChainMatchingHandler filterChainMatchingHandler =
new FilterChainMatchingHandler(grpcHandler, selectorManager, mockDelegate);
setupChannel("172.168.1.1", "172.168.2.2", 90, filterChainMatchingHandler);
@@ -156,7 +161,7 @@
ChannelHandler next = new ChannelInboundHandlerAdapter();
when(mockDelegate.newHandler(grpcHandler)).thenReturn(next);
selectorManager.updateSelector(new FilterChainSelector(
- new HashMap<FilterChain, ServerRoutingConfig>(), null, noopConfig));
+ new HashMap<FilterChain, AtomicReference<ServerRoutingConfig>>(), null, noopConfig));
FilterChainMatchingHandler filterChainMatchingHandler =
new FilterChainMatchingHandler(grpcHandler, selectorManager, mockDelegate);
setupChannel("172.168.1.1", "172.168.2.2", 90, filterChainMatchingHandler);
@@ -175,7 +180,7 @@
assertThat(msg).isNull();
selectorManager.updateSelector(new FilterChainSelector(
- new HashMap<FilterChain, ServerRoutingConfig>(), null, noopConfig));
+ new HashMap<FilterChain, AtomicReference<ServerRoutingConfig>>(), null, noopConfig));
assertThat(channel.readOutbound().getClass().getName())
.isEqualTo("io.grpc.netty.GracefulServerCloseCommand");
}
@@ -199,11 +204,9 @@
tlsContextManager);
selectorManager.updateSelector(new FilterChainSelector(ImmutableMap.of(filterChain, noopConfig),
- null, null));
+ null, new AtomicReference<ServerRoutingConfig>()));
FilterChainMatchingHandler filterChainMatchingHandler =
new FilterChainMatchingHandler(grpcHandler, selectorManager, mockDelegate);
- final SettableFuture<SslContextProviderSupplier> sslSet = SettableFuture.create();
- final SettableFuture<ServerRoutingConfig> routingSettable = SettableFuture.create();
ChannelHandler next = captureAttrHandler(sslSet, routingSettable);
when(mockDelegate.newHandler(grpcHandler)).thenReturn(next);
setupChannel(LOCAL_IP, REMOTE_IP, 15000, filterChainMatchingHandler);
@@ -243,8 +246,6 @@
FilterChainMatchingHandler filterChainMatchingHandler =
new FilterChainMatchingHandler(grpcHandler, selectorManager, mockDelegate);
- final SettableFuture<SslContextProviderSupplier> sslSet = SettableFuture.create();
- final SettableFuture<ServerRoutingConfig> routingSettable = SettableFuture.create();
ChannelHandler next = captureAttrHandler(sslSet, routingSettable);
when(mockDelegate.newHandler(grpcHandler)).thenReturn(next);
setupChannel(LOCAL_IP, REMOTE_IP, 15000, filterChainMatchingHandler);
@@ -281,17 +282,16 @@
tlsContextForDefaultFilterChain, tlsContextManager);
ServerRoutingConfig routingConfig = ServerRoutingConfig.create(
- new ArrayList<NamedFilterConfig>(), new AtomicReference<>(
- ImmutableList.of(createVirtualHost("virtual"))));
+ ImmutableList.of(createVirtualHost("virtual")),
+ ImmutableMap.<Route, ServerInterceptor>of());
selectorManager.updateSelector(new FilterChainSelector(
- ImmutableMap.of(filterChainWithDestPort, routingConfig),
+ ImmutableMap.of(filterChainWithDestPort,
+ new AtomicReference<ServerRoutingConfig>(routingConfig)),
defaultFilterChain.getSslContextProviderSupplier(), noopConfig));
FilterChainMatchingHandler filterChainMatchingHandler =
new FilterChainMatchingHandler(grpcHandler, selectorManager, mockDelegate);
- final SettableFuture<SslContextProviderSupplier> sslSet = SettableFuture.create();
- final SettableFuture<ServerRoutingConfig> routingSettable = SettableFuture.create();
ChannelHandler next = captureAttrHandler(sslSet, routingSettable);
when(mockDelegate.newHandler(grpcHandler)).thenReturn(next);
setupChannel(LOCAL_IP, REMOTE_IP, 15000, filterChainMatchingHandler);
@@ -333,8 +333,6 @@
FilterChainMatchingHandler filterChainMatchingHandler =
new FilterChainMatchingHandler(grpcHandler, selectorManager, mockDelegate);
- final SettableFuture<SslContextProviderSupplier> sslSet = SettableFuture.create();
- final SettableFuture<ServerRoutingConfig> routingSettable = SettableFuture.create();
ChannelHandler next = captureAttrHandler(sslSet, routingSettable);
when(mockDelegate.newHandler(grpcHandler)).thenReturn(next);
setupChannel(LOCAL_IP, REMOTE_IP, 15000, filterChainMatchingHandler);
@@ -377,8 +375,6 @@
FilterChainMatchingHandler filterChainMatchingHandler =
new FilterChainMatchingHandler(grpcHandler, selectorManager, mockDelegate);
- final SettableFuture<SslContextProviderSupplier> sslSet = SettableFuture.create();
- final SettableFuture<ServerRoutingConfig> routingSettable = SettableFuture.create();
ChannelHandler next = captureAttrHandler(sslSet, routingSettable);
when(mockDelegate.newHandler(grpcHandler)).thenReturn(next);
setupChannel(LOCAL_IP, REMOTE_IP, 15000, filterChainMatchingHandler);
@@ -417,12 +413,11 @@
selectorManager.updateSelector(new FilterChainSelector(
ImmutableMap.of(filterChain0Length, noopConfig),
- defaultFilterChain.getSslContextProviderSupplier(), null));
+ defaultFilterChain.getSslContextProviderSupplier(),
+ new AtomicReference<ServerRoutingConfig>()));
FilterChainMatchingHandler filterChainMatchingHandler =
new FilterChainMatchingHandler(grpcHandler, selectorManager, mockDelegate);
- final SettableFuture<SslContextProviderSupplier> sslSet = SettableFuture.create();
- final SettableFuture<ServerRoutingConfig> routingSettable = SettableFuture.create();
ChannelHandler next = captureAttrHandler(sslSet, routingSettable);
when(mockDelegate.newHandler(grpcHandler)).thenReturn(next);
setupChannel(LOCAL_IP, REMOTE_IP, 15000, filterChainMatchingHandler);
@@ -480,8 +475,6 @@
FilterChainMatchingHandler filterChainMatchingHandler =
new FilterChainMatchingHandler(grpcHandler, selectorManager, mockDelegate);
- final SettableFuture<SslContextProviderSupplier> sslSet = SettableFuture.create();
- final SettableFuture<ServerRoutingConfig> routingSettable = SettableFuture.create();
ChannelHandler next = captureAttrHandler(sslSet, routingSettable);
when(mockDelegate.newHandler(grpcHandler)).thenReturn(next);
setupChannel(LOCAL_IP, REMOTE_IP, 15000, filterChainMatchingHandler);
@@ -538,8 +531,6 @@
FilterChainMatchingHandler filterChainMatchingHandler =
new FilterChainMatchingHandler(grpcHandler, selectorManager, mockDelegate);
- final SettableFuture<SslContextProviderSupplier> sslSet = SettableFuture.create();
- final SettableFuture<ServerRoutingConfig> routingSettable = SettableFuture.create();
ChannelHandler next = captureAttrHandler(sslSet, routingSettable);
when(mockDelegate.newHandler(grpcHandler)).thenReturn(next);
setupChannel(LOCAL_IP, REMOTE_IP, 15000, filterChainMatchingHandler);
@@ -596,8 +587,6 @@
FilterChainMatchingHandler filterChainMatchingHandler =
new FilterChainMatchingHandler(grpcHandler, selectorManager, mockDelegate);
- final SettableFuture<SslContextProviderSupplier> sslSet = SettableFuture.create();
- final SettableFuture<ServerRoutingConfig> routingSettable = SettableFuture.create();
ChannelHandler next = captureAttrHandler(sslSet, routingSettable);
when(mockDelegate.newHandler(grpcHandler)).thenReturn(next);
@@ -658,8 +647,6 @@
FilterChainMatchingHandler filterChainMatchingHandler =
new FilterChainMatchingHandler(grpcHandler, selectorManager, mockDelegate);
- final SettableFuture<SslContextProviderSupplier> sslSet = SettableFuture.create();
- final SettableFuture<ServerRoutingConfig> routingSettable = SettableFuture.create();
ChannelHandler next = captureAttrHandler(sslSet, routingSettable);
when(mockDelegate.newHandler(grpcHandler)).thenReturn(next);
setupChannel(LOCAL_IP, REMOTE_IP, 15000, filterChainMatchingHandler);
@@ -700,8 +687,7 @@
FilterChainMatchingHandler filterChainMatchingHandler =
new FilterChainMatchingHandler(grpcHandler, selectorManager, mockDelegate);
- final SettableFuture<SslContextProviderSupplier> sslSet = SettableFuture.create();
- final SettableFuture<ServerRoutingConfig> routingSettable = SettableFuture.create();
+
ChannelHandler next = captureAttrHandler(sslSet, routingSettable);
when(mockDelegate.newHandler(grpcHandler)).thenReturn(next);
setupChannel(LOCAL_IP, REMOTE_IP, 15000, filterChainMatchingHandler);
@@ -714,8 +700,6 @@
@Test
public void sourceTypeLocal() throws Exception {
- final SettableFuture<SslContextProviderSupplier> sslSet = SettableFuture.create();
- final SettableFuture<ServerRoutingConfig> routingSettable = SettableFuture.create();
ChannelHandler next = captureAttrHandler(sslSet, routingSettable);
when(mockDelegate.newHandler(grpcHandler)).thenReturn(next);
EnvoyServerProtoData.DownstreamTlsContext tlsContextMatch =
@@ -755,8 +739,6 @@
@Test
public void sourcePrefixRange_moreSpecificWith2Wins()
throws Exception {
- final SettableFuture<SslContextProviderSupplier> sslSet = SettableFuture.create();
- final SettableFuture<ServerRoutingConfig> routingSettable = SettableFuture.create();
ChannelHandler next = captureAttrHandler(sslSet, routingSettable);
when(mockDelegate.newHandler(grpcHandler)).thenReturn(next);
@@ -817,7 +799,6 @@
@Test
public void sourcePrefixRange_2Matchers_expectException()
throws UnknownHostException {
- final SettableFuture<SslContextProviderSupplier> sslSet = SettableFuture.create();
ChannelHandler next = new ChannelInboundHandlerAdapter() {
@Override
public void userEventTriggered(ChannelHandlerContext ctx, Object evt) {
@@ -932,8 +913,6 @@
FilterChainMatchingHandler filterChainMatchingHandler =
new FilterChainMatchingHandler(grpcHandler, selectorManager, mockDelegate);
- final SettableFuture<SslContextProviderSupplier> sslSet = SettableFuture.create();
- final SettableFuture<ServerRoutingConfig> routingSettable = SettableFuture.create();
ChannelHandler next = captureAttrHandler(sslSet, routingSettable);
when(mockDelegate.newHandler(grpcHandler)).thenReturn(next);
setupChannel(LOCAL_IP, REMOTE_IP, 15000, filterChainMatchingHandler);
@@ -1074,7 +1053,7 @@
EnvoyServerProtoData.FilterChain defaultFilterChain = new EnvoyServerProtoData.FilterChain(
"filter-chain-7", null, HTTP_CONNECTION_MANAGER, null, tlsContextManager);
- Map<FilterChain, ServerRoutingConfig> map = new HashMap<>();
+ Map<FilterChain, AtomicReference<ServerRoutingConfig>> map = new HashMap<>();
map.put(filterChain1, randomConfig("1"));
map.put(filterChain2, randomConfig("2"));
map.put(filterChain3, randomConfig("3"));
@@ -1087,8 +1066,6 @@
FilterChainMatchingHandler filterChainMatchingHandler =
new FilterChainMatchingHandler(grpcHandler, selectorManager, mockDelegate);
- final SettableFuture<SslContextProviderSupplier> sslSet = SettableFuture.create();
- final SettableFuture<ServerRoutingConfig> routingSettable = SettableFuture.create();
ChannelHandler next = captureAttrHandler(sslSet, routingSettable);
when(mockDelegate.newHandler(grpcHandler)).thenReturn(next);
setupChannel(LOCAL_IP, REMOTE_IP, 15000, filterChainMatchingHandler);
@@ -1161,8 +1138,6 @@
FilterChainMatchingHandler filterChainMatchingHandler =
new FilterChainMatchingHandler(grpcHandler, selectorManager, mockDelegate);
- final SettableFuture<SslContextProviderSupplier> sslSet = SettableFuture.create();
- final SettableFuture<ServerRoutingConfig> routingSettable = SettableFuture.create();
ChannelHandler next = captureAttrHandler(sslSet, routingSettable);
when(mockDelegate.newHandler(grpcHandler)).thenReturn(next);
setupChannel(LOCAL_IP, REMOTE_IP, 15000, filterChainMatchingHandler);
@@ -1186,10 +1161,11 @@
ImmutableMap.<String, FilterConfig>of());
}
- private static ServerRoutingConfig randomConfig(String domain) {
- return ServerRoutingConfig.create(
- new ArrayList<NamedFilterConfig>(), new AtomicReference<>(
- ImmutableList.of(createVirtualHost(domain))));
+ private static AtomicReference<ServerRoutingConfig> randomConfig(String domain) {
+ return new AtomicReference<>(
+ ServerRoutingConfig.create(ImmutableList.of(createVirtualHost(domain)),
+ ImmutableMap.<Route, ServerInterceptor>of())
+ );
}
private EnvoyServerProtoData.DownstreamTlsContext createTls() {
@@ -1218,7 +1194,7 @@
private static ChannelHandler captureAttrHandler(
final SettableFuture<SslContextProviderSupplier> sslSet,
- final SettableFuture<ServerRoutingConfig> routingSettable) {
+ final SettableFuture<AtomicReference<ServerRoutingConfig>> routingSettable) {
return new ChannelInboundHandlerAdapter() {
@Override
public void userEventTriggered(ChannelHandlerContext ctx, Object evt) {
diff --git a/xds/src/test/java/io/grpc/xds/FilterChainSelectorManagerTest.java b/xds/src/test/java/io/grpc/xds/FilterChainSelectorManagerTest.java
index d7b883f..a3a2218 100644
--- a/xds/src/test/java/io/grpc/xds/FilterChainSelectorManagerTest.java
+++ b/xds/src/test/java/io/grpc/xds/FilterChainSelectorManagerTest.java
@@ -19,8 +19,9 @@
import static com.google.common.truth.Truth.assertThat;
import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+import io.grpc.ServerInterceptor;
import io.grpc.xds.EnvoyServerProtoData.FilterChain;
-import io.grpc.xds.Filter.NamedFilterConfig;
import io.grpc.xds.FilterChainMatchingProtocolNegotiators.FilterChainMatchingHandler.FilterChainSelector;
import io.grpc.xds.FilterChainSelectorManager.Closer;
import io.grpc.xds.XdsServerWrapper.ServerRoutingConfig;
@@ -33,13 +34,15 @@
@RunWith(JUnit4.class)
public final class FilterChainSelectorManagerTest {
private FilterChainSelectorManager manager = new FilterChainSelectorManager();
- private ServerRoutingConfig noopConfig = ServerRoutingConfig.create(
- Collections.<NamedFilterConfig>emptyList(),
- new AtomicReference<ImmutableList<VirtualHost>>());
+ private AtomicReference<ServerRoutingConfig> noopConfig = new AtomicReference<>(
+ ServerRoutingConfig.create(ImmutableList.<VirtualHost>of(),
+ ImmutableMap.<VirtualHost.Route, ServerInterceptor>of()));
private FilterChainSelector selector1 = new FilterChainSelector(
- Collections.<FilterChain,ServerRoutingConfig>emptyMap(), null, null);
+ Collections.<FilterChain,AtomicReference<ServerRoutingConfig>>emptyMap(),
+ null, new AtomicReference<ServerRoutingConfig>());
private FilterChainSelector selector2 = new FilterChainSelector(
- Collections.<FilterChain,ServerRoutingConfig>emptyMap(), null, noopConfig);
+ Collections.<FilterChain,AtomicReference<ServerRoutingConfig>>emptyMap(),
+ null, noopConfig);
private CounterRunnable runnable1 = new CounterRunnable();
private CounterRunnable runnable2 = new CounterRunnable();
diff --git a/xds/src/test/java/io/grpc/xds/XdsServerWrapperTest.java b/xds/src/test/java/io/grpc/xds/XdsServerWrapperTest.java
index d442136..c109bb4 100644
--- a/xds/src/test/java/io/grpc/xds/XdsServerWrapperTest.java
+++ b/xds/src/test/java/io/grpc/xds/XdsServerWrapperTest.java
@@ -104,6 +104,8 @@
private FakeXdsClient xdsClient = new FakeXdsClient();
private FilterRegistry filterRegistry = FilterRegistry.getDefaultRegistry();
private XdsServerWrapper xdsServerWrapper;
+ private ServerRoutingConfig noopConfig = ServerRoutingConfig.create(
+ ImmutableList.<VirtualHost>of(), ImmutableMap.<Route, ServerInterceptor>of());
@Before
public void setup() {
@@ -380,9 +382,9 @@
assertThat(selectorManager.getSelectorToUpdateSelector().getRoutingConfigs().size())
.isEqualTo(1);
ServerRoutingConfig realConfig =
- selectorManager.getSelectorToUpdateSelector().getRoutingConfigs().get(filterChain);
- assertThat(realConfig.virtualHosts().get()).isEqualTo(httpConnectionManager.virtualHosts());
- assertThat(realConfig.httpFilterConfigs()).isEqualTo(httpConnectionManager.httpFilterConfigs());
+ selectorManager.getSelectorToUpdateSelector().getRoutingConfigs().get(filterChain).get();
+ assertThat(realConfig.virtualHosts()).isEqualTo(httpConnectionManager.virtualHosts());
+ assertThat(realConfig.interceptors()).isEqualTo(ImmutableMap.of());
verify(listener).onServing();
verify(mockServer).start();
}
@@ -429,26 +431,21 @@
start.get(5000, TimeUnit.MILLISECONDS);
verify(mockServer).start();
ServerRoutingConfig realConfig =
- selectorManager.getSelectorToUpdateSelector().getRoutingConfigs().get(f0);
- assertThat(realConfig.virtualHosts().get()).isEqualTo(
+ selectorManager.getSelectorToUpdateSelector().getRoutingConfigs().get(f0).get();
+ assertThat(realConfig.virtualHosts()).isEqualTo(
Collections.singletonList(createVirtualHost("virtual-host-0")));
- assertThat(realConfig.httpFilterConfigs()).isEqualTo(
- f0.getHttpConnectionManager().httpFilterConfigs());
+ assertThat(realConfig.interceptors()).isEqualTo(ImmutableMap.of());
assertThat(selectorManager.getSelectorToUpdateSelector().getRoutingConfigs().size())
.isEqualTo(2);
- realConfig = selectorManager.getSelectorToUpdateSelector().getRoutingConfigs().get(f2);
- assertThat(realConfig.virtualHosts().get()).isEqualTo(
+ realConfig = selectorManager.getSelectorToUpdateSelector().getRoutingConfigs().get(f2).get();
+ assertThat(realConfig.virtualHosts()).isEqualTo(
Collections.singletonList(createVirtualHost("virtual-host-1")));
- assertThat(realConfig.httpFilterConfigs()).isEqualTo(
- f2.getHttpConnectionManager().httpFilterConfigs());
- realConfig = selectorManager.getSelectorToUpdateSelector().getDefaultRoutingConfig();
- assertThat(realConfig.virtualHosts().get()).isEqualTo(
+ assertThat(realConfig.interceptors()).isEqualTo(ImmutableMap.of());
+ realConfig = selectorManager.getSelectorToUpdateSelector().getDefaultRoutingConfig().get();
+ assertThat(realConfig.virtualHosts()).isEqualTo(
Collections.singletonList(createVirtualHost("virtual-host-2")));
- assertThat(realConfig.httpFilterConfigs()).isEqualTo(
- f3.getHttpConnectionManager().httpFilterConfigs());
assertThat(selectorManager.getSelectorToUpdateSelector().getDefaultSslContextProviderSupplier())
- .isEqualTo(
- f3.getSslContextProviderSupplier());
+ .isEqualTo(f3.getSslContextProviderSupplier());
}
@Test
@@ -481,22 +478,20 @@
start.get(5000, TimeUnit.MILLISECONDS);
verify(mockServer, times(1)).start();
ServerRoutingConfig realConfig =
- selectorManager.getSelectorToUpdateSelector().getRoutingConfigs().get(f0);
- assertThat(realConfig.virtualHosts().get()).isEqualTo(
+ selectorManager.getSelectorToUpdateSelector().getRoutingConfigs().get(f0).get();
+ assertThat(realConfig.virtualHosts()).isEqualTo(
Collections.singletonList(createVirtualHost("virtual-host-0")));
- assertThat(realConfig.httpFilterConfigs()).isEqualTo(
- f0.getHttpConnectionManager().httpFilterConfigs());
- realConfig = selectorManager.getSelectorToUpdateSelector().getRoutingConfigs().get(f1);
- assertThat(realConfig.virtualHosts().get()).isEqualTo(
- Collections.singletonList(createVirtualHost("virtual-host-0")));
- assertThat(realConfig.httpFilterConfigs()).isEqualTo(
- f1.getHttpConnectionManager().httpFilterConfigs());
+ assertThat(realConfig.interceptors()).isEqualTo(ImmutableMap.of());
- realConfig = selectorManager.getSelectorToUpdateSelector().getDefaultRoutingConfig();
- assertThat(realConfig.virtualHosts().get()).isEqualTo(
+ realConfig = selectorManager.getSelectorToUpdateSelector().getRoutingConfigs().get(f1).get();
+ assertThat(realConfig.virtualHosts()).isEqualTo(
Collections.singletonList(createVirtualHost("virtual-host-0")));
- assertThat(realConfig.httpFilterConfigs()).isEqualTo(
- f2.getHttpConnectionManager().httpFilterConfigs());
+ assertThat(realConfig.interceptors()).isEqualTo(ImmutableMap.of());
+
+ realConfig = selectorManager.getSelectorToUpdateSelector().getDefaultRoutingConfig().get();
+ assertThat(realConfig.virtualHosts()).isEqualTo(
+ Collections.singletonList(createVirtualHost("virtual-host-0")));
+ assertThat(realConfig.interceptors()).isEqualTo(ImmutableMap.of());
assertThat(selectorManager.getSelectorToUpdateSelector().getDefaultSslContextProviderSupplier())
.isSameInstanceAs(f2.getSslContextProviderSupplier());
@@ -513,25 +508,22 @@
assertThat(selectorManager.getSelectorToUpdateSelector().getRoutingConfigs().size())
.isEqualTo(2);
- realConfig = selectorManager.getSelectorToUpdateSelector().getRoutingConfigs().get(f5);
- assertThat(realConfig.virtualHosts().get()).isEqualTo(
+ realConfig = selectorManager.getSelectorToUpdateSelector().getRoutingConfigs().get(f5).get();
+ assertThat(realConfig.virtualHosts()).isEqualTo(
Collections.singletonList(createVirtualHost("virtual-host-1")));
- assertThat(realConfig.httpFilterConfigs()).isEqualTo(
- f5.getHttpConnectionManager().httpFilterConfigs());
- realConfig = selectorManager.getSelectorToUpdateSelector().getRoutingConfigs().get(f3);
- assertThat(realConfig.virtualHosts().get()).isEqualTo(
+ assertThat(realConfig.interceptors()).isEqualTo(ImmutableMap.of());
+ realConfig = selectorManager.getSelectorToUpdateSelector().getRoutingConfigs().get(f3).get();
+ assertThat(realConfig.virtualHosts()).isEqualTo(
Collections.singletonList(createVirtualHost("virtual-host-0")));
- assertThat(realConfig.httpFilterConfigs()).isEqualTo(
- f3.getHttpConnectionManager().httpFilterConfigs());
+ assertThat(realConfig.interceptors()).isEqualTo(ImmutableMap.of());
- realConfig = selectorManager.getSelectorToUpdateSelector().getDefaultRoutingConfig();
- assertThat(realConfig.virtualHosts().get()).isEqualTo(
+ realConfig = selectorManager.getSelectorToUpdateSelector().getDefaultRoutingConfig().get();
+ assertThat(realConfig.virtualHosts()).isEqualTo(
Collections.singletonList(createVirtualHost("virtual-host-1")));
- assertThat(realConfig.httpFilterConfigs()).isEqualTo(
- f4.getHttpConnectionManager().httpFilterConfigs());
+ assertThat(realConfig.interceptors()).isEqualTo(ImmutableMap.of());
+
assertThat(selectorManager.getSelectorToUpdateSelector().getDefaultSslContextProviderSupplier())
- .isSameInstanceAs(
- f4.getSslContextProviderSupplier());
+ .isSameInstanceAs(f4.getSslContextProviderSupplier());
verify(mockServer, times(1)).start();
xdsServerWrapper.shutdown();
verify(mockServer, times(1)).shutdown();
@@ -567,35 +559,31 @@
assertThat(selectorManager.getSelectorToUpdateSelector().getRoutingConfigs().size())
.isEqualTo(2);
ServerRoutingConfig realConfig =
- selectorManager.getSelectorToUpdateSelector().getRoutingConfigs().get(f1);
- assertThat(realConfig.virtualHosts().get()).isNull();
- assertThat(realConfig.httpFilterConfigs()).isEqualTo(
- f1.getHttpConnectionManager().httpFilterConfigs());
- realConfig = selectorManager.getSelectorToUpdateSelector().getRoutingConfigs().get(f0);
- assertThat(realConfig.virtualHosts().get()).isEqualTo(hcmVirtual.virtualHosts());
- assertThat(realConfig.httpFilterConfigs()).isEqualTo(
- f0.getHttpConnectionManager().httpFilterConfigs());
+ selectorManager.getSelectorToUpdateSelector().getRoutingConfigs().get(f1).get();
+ assertThat(realConfig.virtualHosts()).isEmpty();
+ assertThat(realConfig.interceptors()).isEmpty();
+
+ realConfig = selectorManager.getSelectorToUpdateSelector().getRoutingConfigs().get(f0).get();
+ assertThat(realConfig.virtualHosts()).isEqualTo(hcmVirtual.virtualHosts());
+ assertThat(realConfig.interceptors()).isEqualTo(ImmutableMap.of());
xdsClient.deliverRdsUpdate("r0",
Collections.singletonList(createVirtualHost("virtual-host-1")));
- realConfig = selectorManager.getSelectorToUpdateSelector().getRoutingConfigs().get(f1);
- assertThat(realConfig.virtualHosts().get()).isEqualTo(
+ realConfig = selectorManager.getSelectorToUpdateSelector().getRoutingConfigs().get(f1).get();
+ assertThat(realConfig.virtualHosts()).isEqualTo(
Collections.singletonList(createVirtualHost("virtual-host-1")));
- assertThat(realConfig.httpFilterConfigs()).isEqualTo(
- f1.getHttpConnectionManager().httpFilterConfigs());
+ assertThat(realConfig.interceptors()).isEqualTo(ImmutableMap.of());
xdsClient.rdsWatchers.get("r0").onError(Status.CANCELLED);
- realConfig = selectorManager.getSelectorToUpdateSelector().getRoutingConfigs().get(f1);
- assertThat(realConfig.virtualHosts().get()).isEqualTo(
+ realConfig = selectorManager.getSelectorToUpdateSelector().getRoutingConfigs().get(f1).get();
+ assertThat(realConfig.virtualHosts()).isEqualTo(
Collections.singletonList(createVirtualHost("virtual-host-1")));
- assertThat(realConfig.httpFilterConfigs()).isEqualTo(
- f1.getHttpConnectionManager().httpFilterConfigs());
+ assertThat(realConfig.interceptors()).isEqualTo(ImmutableMap.of());
xdsClient.rdsWatchers.get("r0").onResourceDoesNotExist("r0");
- realConfig = selectorManager.getSelectorToUpdateSelector().getRoutingConfigs().get(f1);
- assertThat(realConfig.virtualHosts().get()).isNull();
- assertThat(realConfig.httpFilterConfigs()).isEqualTo(
- f1.getHttpConnectionManager().httpFilterConfigs());
+ realConfig = selectorManager.getSelectorToUpdateSelector().getRoutingConfigs().get(f1).get();
+ assertThat(realConfig.virtualHosts()).isEmpty();
+ assertThat(realConfig.interceptors()).isEmpty();
}
@Test
@@ -648,11 +636,11 @@
assertThat(selectorManager.getSelectorToUpdateSelector().getRoutingConfigs().size())
.isEqualTo(1);
ServerRoutingConfig realConfig =
- selectorManager.getSelectorToUpdateSelector().getRoutingConfigs().get(filterChain1);
- assertThat(realConfig.virtualHosts().get()).isEqualTo(
+ selectorManager.getSelectorToUpdateSelector().getRoutingConfigs().get(filterChain1).get();
+ assertThat(realConfig.virtualHosts()).isEqualTo(
Collections.singletonList(createVirtualHost("virtual-host-1")));
- assertThat(realConfig.httpFilterConfigs()).isEqualTo(
- filterChain1.getHttpConnectionManager().httpFilterConfigs());
+ assertThat(realConfig.interceptors()).isEqualTo(ImmutableMap.of());
+
// xds update after start
xdsClient.deliverRdsUpdate("rds",
Collections.singletonList(createVirtualHost("virtual-host-2")));
@@ -664,11 +652,11 @@
assertThat(selectorManager.getSelectorToUpdateSelector().getRoutingConfigs().size())
.isEqualTo(1);
realConfig = selectorManager.getSelectorToUpdateSelector().getRoutingConfigs()
- .get(filterChain1);
- assertThat(realConfig.virtualHosts().get()).isEqualTo(
+ .get(filterChain1).get();
+ assertThat(realConfig.virtualHosts()).isEqualTo(
Collections.singletonList(createVirtualHost("virtual-host-2")));
- assertThat(realConfig.httpFilterConfigs()).isEqualTo(
- filterChain1.getHttpConnectionManager().httpFilterConfigs());
+ assertThat(realConfig.interceptors()).isEqualTo(ImmutableMap.of());
+
assertThat(sslSupplier1.isShutdown()).isFalse();
// not serving after serving
@@ -705,11 +693,11 @@
assertThat(selectorManager.getSelectorToUpdateSelector().getRoutingConfigs().size())
.isEqualTo(1);
realConfig = selectorManager.getSelectorToUpdateSelector().getRoutingConfigs()
- .get(filterChain2);
- assertThat(realConfig.virtualHosts().get()).isEqualTo(
+ .get(filterChain2).get();
+ assertThat(realConfig.virtualHosts()).isEqualTo(
Collections.singletonList(createVirtualHost("virtual-host-1")));
- assertThat(realConfig.httpFilterConfigs()).isEqualTo(
- filterChain2.getHttpConnectionManager().httpFilterConfigs());
+ assertThat(realConfig.interceptors()).isEqualTo(ImmutableMap.of());
+
assertThat(executor.numPendingTasks()).isEqualTo(1);
xdsClient.ldsWatcher.onResourceDoesNotExist(ldsResource);
verify(mockServer, times(4)).shutdown();
@@ -733,11 +721,11 @@
assertThat(selectorManager.getSelectorToUpdateSelector().getRoutingConfigs().size())
.isEqualTo(1);
realConfig = selectorManager.getSelectorToUpdateSelector().getRoutingConfigs()
- .get(filterChain3);
- assertThat(realConfig.virtualHosts().get()).isEqualTo(
+ .get(filterChain3).get();
+ assertThat(realConfig.virtualHosts()).isEqualTo(
Collections.singletonList(createVirtualHost("virtual-host-1")));
- assertThat(realConfig.httpFilterConfigs()).isEqualTo(
- filterChain3.getHttpConnectionManager().httpFilterConfigs());
+ assertThat(realConfig.interceptors()).isEqualTo(ImmutableMap.of());
+
xdsServerWrapper.shutdown();
verify(mockServer, times(5)).shutdown();
assertThat(sslSupplier3.isShutdown()).isTrue();
@@ -747,9 +735,9 @@
@Test
@SuppressWarnings("unchecked")
- public void interceptor_notServerInterceptor() throws Exception {
+ public void interceptor_success() throws Exception {
ArgumentCaptor<ConfigApplyingInterceptor> interceptorCaptor =
- ArgumentCaptor.forClass(ConfigApplyingInterceptor.class);
+ ArgumentCaptor.forClass(ConfigApplyingInterceptor.class);
final SettableFuture<Server> start = SettableFuture.create();
Executors.newSingleThreadExecutor().execute(new Runnable() {
@Override
@@ -764,26 +752,36 @@
xdsClient.ldsResource.get(5, TimeUnit.SECONDS);
verify(mockBuilder).intercept(interceptorCaptor.capture());
ConfigApplyingInterceptor interceptor = interceptorCaptor.getValue();
- ServerRoutingConfig routingConfig = createRoutingConfig("/FooService/barMethod",
- "foo.google.com", "filter-type-url");
+ RouteMatch routeMatch =
+ RouteMatch.create(
+ PathMatcher.fromPath("/FooService/barMethod", true),
+ Collections.<HeaderMatcher>emptyList(), null);
+ Route route = Route.forAction(routeMatch, null,
+ ImmutableMap.<String, FilterConfig>of());
+ VirtualHost virtualHost = VirtualHost.create(
+ "v1", Collections.singletonList("foo.google.com"), Arrays.asList(route),
+ ImmutableMap.<String, FilterConfig>of());
+ final List<Integer> interceptorTrace = new ArrayList<>();
+ ServerInterceptor interceptor0 = new ServerInterceptor() {
+ @Override
+ public <ReqT, RespT> ServerCall.Listener<ReqT> interceptCall(ServerCall<ReqT, RespT> call,
+ Metadata headers, ServerCallHandler<ReqT, RespT> next) {
+ interceptorTrace.add(0);
+ return next.startCall(call, headers);
+ }
+ };
+ ServerRoutingConfig realConfig = ServerRoutingConfig.create(
+ ImmutableList.of(virtualHost), ImmutableMap.of(route, interceptor0));
ServerCall<Void, Void> serverCall = mock(ServerCall.class);
- when(serverCall.getAttributes()).thenReturn(
- Attributes.newBuilder().set(ATTR_SERVER_ROUTING_CONFIG, routingConfig).build());
when(serverCall.getMethodDescriptor()).thenReturn(createMethod("FooService/barMethod"));
+ when(serverCall.getAttributes()).thenReturn(
+ Attributes.newBuilder().set(ATTR_SERVER_ROUTING_CONFIG,
+ new AtomicReference<>(realConfig)).build());
when(serverCall.getAuthority()).thenReturn("foo.google.com");
-
- Filter filter = mock(Filter.class);
- when(filter.typeUrls()).thenReturn(new String[]{"filter-type-url"});
- filterRegistry.register(filter);
ServerCallHandler<Void, Void> next = mock(ServerCallHandler.class);
interceptor.interceptCall(serverCall, new Metadata(), next);
- verify(next, never()).startCall(any(ServerCall.class), any(Metadata.class));
- ArgumentCaptor<Status> statusCaptor = ArgumentCaptor.forClass(Status.class);
- verify(serverCall).close(statusCaptor.capture(), any(Metadata.class));
- Status status = statusCaptor.getValue();
- assertThat(status.getCode()).isEqualTo(Status.UNAVAILABLE.getCode());
- assertThat(status.getDescription()).isEqualTo(
- "HttpFilterConfig(type URL: filter-type-url) is not supported on server-side.");
+ verify(next).startCall(eq(serverCall), any(Metadata.class));
+ assertThat(interceptorTrace).isEqualTo(Arrays.asList(0));
}
@Test
@@ -809,7 +807,8 @@
"foo.google.com", "filter-type-url");
ServerCall<Void, Void> serverCall = mock(ServerCall.class);
when(serverCall.getAttributes()).thenReturn(
- Attributes.newBuilder().set(ATTR_SERVER_ROUTING_CONFIG, routingConfig).build());
+ Attributes.newBuilder().set(ATTR_SERVER_ROUTING_CONFIG,
+ new AtomicReference<>(routingConfig)).build());
when(serverCall.getAuthority()).thenReturn("not-match.google.com");
Filter filter = mock(Filter.class);
@@ -848,7 +847,8 @@
"foo.google.com", "filter-type-url");
ServerCall<Void, Void> serverCall = mock(ServerCall.class);
when(serverCall.getAttributes()).thenReturn(
- Attributes.newBuilder().set(ATTR_SERVER_ROUTING_CONFIG, routingConfig).build());
+ Attributes.newBuilder()
+ .set(ATTR_SERVER_ROUTING_CONFIG, new AtomicReference<>(routingConfig)).build());
when(serverCall.getMethodDescriptor()).thenReturn(createMethod("NotMatchMethod"));
when(serverCall.getAuthority()).thenReturn("foo.google.com");
@@ -884,12 +884,11 @@
xdsClient.ldsResource.get(5, TimeUnit.SECONDS);
verify(mockBuilder).intercept(interceptorCaptor.capture());
ConfigApplyingInterceptor interceptor = interceptorCaptor.getValue();
- ServerRoutingConfig failingConfig = ServerRoutingConfig.create(
- ImmutableList.<NamedFilterConfig>of(), new AtomicReference<ImmutableList<VirtualHost>>()
- );
ServerCall<Void, Void> serverCall = mock(ServerCall.class);
+
when(serverCall.getAttributes()).thenReturn(
- Attributes.newBuilder().set(ATTR_SERVER_ROUTING_CONFIG, failingConfig).build());
+ Attributes.newBuilder().set(ATTR_SERVER_ROUTING_CONFIG,
+ new AtomicReference<>(ServerRoutingConfig.FAILING_ROUTING_CONFIG)).build());
ServerCallHandler<Void, Void> next = mock(ServerCallHandler.class);
interceptor.interceptCall(serverCall, new Metadata(), next);
@@ -899,14 +898,12 @@
Status status = statusCaptor.getValue();
assertThat(status.getCode()).isEqualTo(Status.UNAVAILABLE.getCode());
assertThat(status.getDescription()).isEqualTo(
- "Missing xDS routing config VirtualHosts due to RDS config unavailable.");
+ "Missing or broken xDS routing config: RDS config unavailable.");
}
@Test
@SuppressWarnings("unchecked")
- public void interceptors() throws Exception {
- ArgumentCaptor<ConfigApplyingInterceptor> interceptorCaptor =
- ArgumentCaptor.forClass(ConfigApplyingInterceptor.class);
+ public void buildInterceptor_inline() throws Exception {
final SettableFuture<Server> start = SettableFuture.create();
Executors.newSingleThreadExecutor().execute(new Runnable() {
@Override
@@ -919,14 +916,12 @@
}
});
xdsClient.ldsResource.get(5, TimeUnit.SECONDS);
- verify(mockBuilder).intercept(interceptorCaptor.capture());
- final ConfigApplyingInterceptor interceptor = interceptorCaptor.getValue();
RouteMatch routeMatch =
- RouteMatch.create(
- PathMatcher.fromPath("/FooService/barMethod", true),
- Collections.<HeaderMatcher>emptyList(), null);
+ RouteMatch.create(
+ PathMatcher.fromPath("/FooService/barMethod", true),
+ Collections.<HeaderMatcher>emptyList(), null);
Filter filter = mock(Filter.class, withSettings()
- .extraInterfaces(ServerInterceptorBuilder.class));
+ .extraInterfaces(ServerInterceptorBuilder.class));
when(filter.typeUrls()).thenReturn(new String[]{"filter-type-url"});
filterRegistry.register(filter);
FilterConfig f0 = mock(FilterConfig.class);
@@ -936,7 +931,7 @@
ServerInterceptor interceptor0 = new ServerInterceptor() {
@Override
public <ReqT, RespT> ServerCall.Listener<ReqT> interceptCall(ServerCall<ReqT, RespT> call,
- Metadata headers, ServerCallHandler<ReqT, RespT> next) {
+ Metadata headers, ServerCallHandler<ReqT, RespT> next) {
interceptorTrace.add(0);
return next.startCall(call, headers);
}
@@ -949,55 +944,130 @@
return next.startCall(call, headers);
}
};
-
+ when(((ServerInterceptorBuilder)filter).buildServerInterceptor(f0, null))
+ .thenReturn(interceptor0);
+ when(((ServerInterceptorBuilder)filter).buildServerInterceptor(f0, f0Override))
+ .thenReturn(interceptor1);
+ Route route = Route.forAction(routeMatch, null,
+ ImmutableMap.<String, FilterConfig>of());
VirtualHost virtualHost = VirtualHost.create(
- "v1", Collections.singletonList("foo.google.com"),
- Arrays.asList(Route.forAction(routeMatch, null,
- ImmutableMap.<String, FilterConfig>of())),
- ImmutableMap.of("filter-config-name-0", f0Override));
- ServerRoutingConfig routingConfig = ServerRoutingConfig.create(
- Arrays.asList(new NamedFilterConfig("filter-config-name-0", f0),
- new NamedFilterConfig("filter-config-name-1", f0)),
- new AtomicReference<>(ImmutableList.<VirtualHost>of(virtualHost))
- );
+ "v1", Collections.singletonList("foo.google.com"), Arrays.asList(route),
+ ImmutableMap.of("filter-config-name-0", f0Override));
+ HttpConnectionManager hcmVirtual = HttpConnectionManager.forVirtualHosts(
+ 0L, Collections.singletonList(virtualHost),
+ Arrays.asList(new NamedFilterConfig("filter-config-name-0", f0),
+ new NamedFilterConfig("filter-config-name-1", f0)));
+ EnvoyServerProtoData.FilterChain filterChain = createFilterChain("filter-chain-0", hcmVirtual);
+ xdsClient.deliverLdsUpdate(Collections.singletonList(filterChain), null);
+ start.get(5000, TimeUnit.MILLISECONDS);
+ verify(mockServer).start();
+ assertThat(selectorManager.getSelectorToUpdateSelector().getRoutingConfigs().size())
+ .isEqualTo(1);
+ ServerInterceptor realInterceptor = selectorManager.getSelectorToUpdateSelector()
+ .getRoutingConfigs().get(filterChain).get().interceptors().get(route);
+ assertThat(realInterceptor).isNotNull();
+
ServerCall<Void, Void> serverCall = mock(ServerCall.class);
ServerCallHandler<Void, Void> mockNext = mock(ServerCallHandler.class);
final ServerCall.Listener<Void> listener = new ServerCall.Listener<Void>() {};
when(mockNext.startCall(any(ServerCall.class), any(Metadata.class))).thenReturn(listener);
- when(serverCall.getAttributes()).thenReturn(
- Attributes.newBuilder().set(ATTR_SERVER_ROUTING_CONFIG, routingConfig).build());
- when(serverCall.getMethodDescriptor()).thenReturn(createMethod("FooService/barMethod"));
- when(serverCall.getAuthority()).thenReturn("foo.google.com");
-
- when(((ServerInterceptorBuilder)filter).buildServerInterceptor(f0, f0Override))
- .thenReturn(null);
- when(((ServerInterceptorBuilder)filter).buildServerInterceptor(f0, null))
- .thenReturn(null);
- ServerCall.Listener<Void> configApplyingInterceptorListener =
- interceptor.interceptCall(serverCall, new Metadata(), mockNext);
- assertThat(configApplyingInterceptorListener).isSameInstanceAs(listener);
+ realInterceptor.interceptCall(serverCall, new Metadata(), mockNext);
+ assertThat(interceptorTrace).isEqualTo(Arrays.asList(1, 0));
verify(mockNext).startCall(eq(serverCall), any(Metadata.class));
- assertThat(interceptorTrace).isEqualTo(Arrays.asList());
+ }
- when(((ServerInterceptorBuilder)filter).buildServerInterceptor(f0, f0Override))
- .thenReturn(null);
+ @Test
+ @SuppressWarnings("unchecked")
+ public void buildInterceptor_rds() throws Exception {
+ final SettableFuture<Server> start = SettableFuture.create();
+ Executors.newSingleThreadExecutor().execute(new Runnable() {
+ @Override
+ public void run() {
+ try {
+ start.set(xdsServerWrapper.start());
+ } catch (Exception ex) {
+ start.setException(ex);
+ }
+ }
+ });
+ xdsClient.ldsResource.get(5, TimeUnit.SECONDS);
+
+ Filter filter = mock(Filter.class, withSettings()
+ .extraInterfaces(ServerInterceptorBuilder.class));
+ when(filter.typeUrls()).thenReturn(new String[]{"filter-type-url"});
+ filterRegistry.register(filter);
+ FilterConfig f0 = mock(FilterConfig.class);
+ FilterConfig f0Override = mock(FilterConfig.class);
+ when(f0.typeUrl()).thenReturn("filter-type-url");
+ final List<Integer> interceptorTrace = new ArrayList<>();
+ ServerInterceptor interceptor0 = new ServerInterceptor() {
+ @Override
+ public <ReqT, RespT> ServerCall.Listener<ReqT> interceptCall(ServerCall<ReqT, RespT> call,
+ Metadata headers, ServerCallHandler<ReqT, RespT> next) {
+ interceptorTrace.add(0);
+ return next.startCall(call, headers);
+ }
+ };
+ ServerInterceptor interceptor1 = new ServerInterceptor() {
+ @Override
+ public <ReqT, RespT> ServerCall.Listener<ReqT> interceptCall(ServerCall<ReqT, RespT> call,
+ Metadata headers, ServerCallHandler<ReqT, RespT> next) {
+ interceptorTrace.add(1);
+ return next.startCall(call, headers);
+ }
+ };
when(((ServerInterceptorBuilder)filter).buildServerInterceptor(f0, null))
- .thenReturn(interceptor0);
- configApplyingInterceptorListener = interceptor.interceptCall(
- serverCall, new Metadata(), mockNext);
- assertThat(configApplyingInterceptorListener).isSameInstanceAs(listener);
+ .thenReturn(interceptor0);
+ when(((ServerInterceptorBuilder)filter).buildServerInterceptor(f0, f0Override))
+ .thenReturn(interceptor1);
+ RouteMatch routeMatch =
+ RouteMatch.create(
+ PathMatcher.fromPath("/FooService/barMethod", true),
+ Collections.<HeaderMatcher>emptyList(), null);
+
+ HttpConnectionManager rdsHcm = HttpConnectionManager.forRdsName(0L, "r0",
+ Arrays.asList(new NamedFilterConfig("filter-config-name-0", f0),
+ new NamedFilterConfig("filter-config-name-1", f0)));
+ EnvoyServerProtoData.FilterChain filterChain = createFilterChain("filter-chain-0", rdsHcm);
+ xdsClient.deliverLdsUpdate(Collections.singletonList(filterChain), null);
+ Route route = Route.forAction(routeMatch, null,
+ ImmutableMap.<String, FilterConfig>of());
+ VirtualHost virtualHost = VirtualHost.create(
+ "v1", Collections.singletonList("foo.google.com"), Arrays.asList(route),
+ ImmutableMap.of("filter-config-name-0", f0Override));
+ xdsClient.rdsCount.await(5, TimeUnit.SECONDS);
+ xdsClient.deliverRdsUpdate("r0", Collections.singletonList(virtualHost));
+ start.get(5000, TimeUnit.MILLISECONDS);
+ verify(mockServer).start();
+ assertThat(selectorManager.getSelectorToUpdateSelector().getRoutingConfigs().size())
+ .isEqualTo(1);
+ ServerInterceptor realInterceptor = selectorManager.getSelectorToUpdateSelector()
+ .getRoutingConfigs().get(filterChain).get().interceptors().get(route);
+ assertThat(realInterceptor).isNotNull();
+
+ ServerCall<Void, Void> serverCall = mock(ServerCall.class);
+ ServerCallHandler<Void, Void> mockNext = mock(ServerCallHandler.class);
+ final ServerCall.Listener<Void> listener = new ServerCall.Listener<Void>() {};
+ when(mockNext.startCall(any(ServerCall.class), any(Metadata.class))).thenReturn(listener);
+ realInterceptor.interceptCall(serverCall, new Metadata(), mockNext);
+ assertThat(interceptorTrace).isEqualTo(Arrays.asList(1, 0));
+ verify(mockNext).startCall(eq(serverCall), any(Metadata.class));
+
+ virtualHost = VirtualHost.create(
+ "v1", Collections.singletonList("foo.google.com"), Arrays.asList(route),
+ ImmutableMap.<String, FilterConfig>of());
+ xdsClient.deliverRdsUpdate("r0", Collections.singletonList(virtualHost));
+ realInterceptor = selectorManager.getSelectorToUpdateSelector().getRoutingConfigs()
+ .get(filterChain).get().interceptors().get(route);
+ assertThat(realInterceptor).isNotNull();
+ interceptorTrace.clear();
+ realInterceptor.interceptCall(serverCall, new Metadata(), mockNext);
+ assertThat(interceptorTrace).isEqualTo(Arrays.asList(0, 0));
verify(mockNext, times(2)).startCall(eq(serverCall), any(Metadata.class));
- assertThat(interceptorTrace).isEqualTo(Arrays.asList(0));
- when(((ServerInterceptorBuilder)filter).buildServerInterceptor(f0, f0Override))
- .thenReturn(interceptor0);
- when(((ServerInterceptorBuilder)filter).buildServerInterceptor(f0, null))
- .thenReturn(interceptor1);
- configApplyingInterceptorListener = interceptor.interceptCall(
- serverCall, new Metadata(), mockNext);
- assertThat(configApplyingInterceptorListener).isSameInstanceAs(listener);
- verify(mockNext, times(3)).startCall(eq(serverCall), any(Metadata.class));
- assertThat(interceptorTrace).isEqualTo(Arrays.asList(0, 0, 1));
+ xdsClient.rdsWatchers.get("r0").onResourceDoesNotExist("r0");
+ assertThat(selectorManager.getSelectorToUpdateSelector().getRoutingConfigs()
+ .get(filterChain).get()).isEqualTo(noopConfig);
}
private static FilterChain createFilterChain(String name, HttpConnectionManager hcm) {
@@ -1012,8 +1082,12 @@
}
private static HttpConnectionManager createRds(String name) {
+ return createRds(name, null);
+ }
+
+ private static HttpConnectionManager createRds(String name, FilterConfig filterConfig) {
return HttpConnectionManager.forRdsName(0L, name,
- Arrays.asList(new NamedFilterConfig("named-config-" + name, null)));
+ Arrays.asList(new NamedFilterConfig("named-config-" + name, filterConfig)));
}
private static EnvoyServerProtoData.FilterChainMatch createMatch() {
@@ -1041,9 +1115,8 @@
Collections.<String, FilterConfig>emptyMap());
FilterConfig f0 = mock(FilterConfig.class);
when(f0.typeUrl()).thenReturn(filterType);
- return ServerRoutingConfig.create(
- Arrays.asList(new NamedFilterConfig("filter-config-name-0", f0)),
- new AtomicReference<>(ImmutableList.<VirtualHost>of(virtualHost))
+ return ServerRoutingConfig.create(ImmutableList.<VirtualHost>of(virtualHost),
+ ImmutableMap.<Route, ServerInterceptor>of()
);
}
