From bfdbd34f4dee79e068ed353bce4d31045908b2d8 Mon Sep 17 00:00:00 2001 From: DavidLiu506 <85367145+DavidLiu506@users.noreply.github.com> Date: Thu, 7 Jul 2022 14:22:50 -0700 Subject: [PATCH] Add Arion gateway (#748) --- kubernetes/services/dpm_manager.yaml | 5 + lib/pom.xml | 8 +- schema/pom.xml | 10 +- schema/proto3/gateway.proto | 10 ++ schema/proto3/goalstateprovisioner.proto | 10 ++ schema/proto3/neighbor.proto | 3 + services/data_plane_manager/pom.xml | 6 +- .../alcor/dataplane/cache/ArionWingCache.java | 101 ++++++++++++++++ .../client/grpc/DataPlaneClientImplV2.java | 69 ++++++++--- .../controller/ArionGatewayController.java | 78 +++++++++++++ .../alcor/dataplane/entity/ArionGroup.java | 32 +++++ .../alcor/dataplane/entity/ArionWing.java | 88 ++++++++++++++ .../service/impl/ArionWingService.java | 110 ++++++++++++++++++ .../service/impl/DpmServiceImplV2.java | 10 ++ .../service/impl/NeighborService.java | 13 ++- .../src/main/resources/application.properties | 4 + .../web/entity/gateway/ArionWingInfo.java | 34 ++++++ 17 files changed, 564 insertions(+), 27 deletions(-) create mode 100644 services/data_plane_manager/src/main/java/com/futurewei/alcor/dataplane/cache/ArionWingCache.java create mode 100644 services/data_plane_manager/src/main/java/com/futurewei/alcor/dataplane/controller/ArionGatewayController.java create mode 100644 services/data_plane_manager/src/main/java/com/futurewei/alcor/dataplane/entity/ArionGroup.java create mode 100644 services/data_plane_manager/src/main/java/com/futurewei/alcor/dataplane/entity/ArionWing.java create mode 100644 services/data_plane_manager/src/main/java/com/futurewei/alcor/dataplane/service/impl/ArionWingService.java create mode 100644 web/src/main/java/com/futurewei/alcor/web/entity/gateway/ArionWingInfo.java diff --git a/kubernetes/services/dpm_manager.yaml b/kubernetes/services/dpm_manager.yaml index 62778111d..07291cbb7 100644 --- a/kubernetes/services/dpm_manager.yaml +++ b/kubernetes/services/dpm_manager.yaml @@ -61,6 +61,11 @@ data: host.ip.to.group.topic.map=group-topic1:192.168.131.131,10.10.10.11 group-topic2:192.168.131.131,11.11.11.12 group.topic.to.multicast.topic.map=multicast-topic1:group-topic1,group-topic3 multicast-topic2:group-topic2,group-topic4 + + arionGateway.enabled = false + arionMaster.server = 127.0.0.1 + arionMaster.port = 9090 + zetaGateway.enabled=false zetaGateway.node.mac=e0:97:96:02:45:53 microservices.node.service.url=http://nodemanager-service.default.svc.cluster.local:9007/nodes diff --git a/lib/pom.xml b/lib/pom.xml index fe404374a..206b6ce59 100644 --- a/lib/pom.xml +++ b/lib/pom.xml @@ -81,22 +81,22 @@ Copyright(c) 2020 Futurewei Cloud io.grpc grpc-netty-shaded - 1.23.0 + 1.42.2 io.grpc grpc-protobuf - 1.23.0 + 1.42.2 io.grpc grpc-stub - 1.23.0 + 1.42.2 io.grpc protoc-gen-grpc-java - 1.23.0 + 1.42.2 pom diff --git a/schema/pom.xml b/schema/pom.xml index 69932c164..698197372 100644 --- a/schema/pom.xml +++ b/schema/pom.xml @@ -36,22 +36,22 @@ io.grpc grpc-netty-shaded - 1.23.0 + 1.42.2 io.grpc grpc-protobuf - 1.23.0 + 1.42.2 io.grpc grpc-stub - 1.23.0 + 1.42.2 io.grpc protoc-gen-grpc-java - 1.23.0 + 1.42.2 pom @@ -62,7 +62,7 @@ io.grpc grpc-testing - 1.23.0 + 1.42.2 test diff --git a/schema/proto3/gateway.proto b/schema/proto3/gateway.proto index 327d38e0b..2614ae168 100644 --- a/schema/proto3/gateway.proto +++ b/schema/proto3/gateway.proto @@ -29,6 +29,7 @@ enum GatewayType { TGW = 2; // Transit Gateway IGW = 3; // Internet Gateway NGW = 4; // NAT Gateway + ARION = 5; } message GatewayConfiguration { @@ -50,8 +51,17 @@ message GatewayConfiguration { uint32 port_inband_operation = 1; } + message arion { + string vpc_id = 1; + uint32 vni = 2; + string subnet_id = 3; + // port for in-band (same NIC channel) operation + uint32 port_inband_operation = 4; + } + oneof extra_info { zeta zeta_info = 6; + arion arion_info = 7; } } diff --git a/schema/proto3/goalstateprovisioner.proto b/schema/proto3/goalstateprovisioner.proto index b2d27f0f4..4bb16c36c 100644 --- a/schema/proto3/goalstateprovisioner.proto +++ b/schema/proto3/goalstateprovisioner.proto @@ -22,6 +22,7 @@ option java_package = "com.futurewei.alcor.schema"; import "common.proto"; import "goalstate.proto"; +import "neighbor.proto"; service GoalStateProvisioner { @@ -49,6 +50,8 @@ service GoalStateProvisioner { rpc RequestGoalStates (HostRequest) returns (HostRequestReply) { } + rpc PushGoalstates (NeighborRulesRequest) returns (GoalStateOperationReply) { + } } message HostRequest { @@ -83,6 +86,13 @@ message HostRequestReply { uint32 total_operation_time = 3; } +message NeighborRulesRequest { + uint32 format_version = 1; + string request_id = 2; + + repeated NeighborState neigborstates = 3; +} + message GoalStateOperationReply { uint32 format_version = 1; diff --git a/schema/proto3/neighbor.proto b/schema/proto3/neighbor.proto index f72d8cf33..d60c6b691 100644 --- a/schema/proto3/neighbor.proto +++ b/schema/proto3/neighbor.proto @@ -43,6 +43,9 @@ message NeighborConfiguration { NeighborType neighbor_type = 1; string subnet_id = 2; string ip_address = 3; + string arion_group = 4; + uint32 tunnel_id = 5; + string mac_address = 6; } message AllowAddressPair { diff --git a/services/data_plane_manager/pom.xml b/services/data_plane_manager/pom.xml index 69a287c7d..b2018081f 100644 --- a/services/data_plane_manager/pom.xml +++ b/services/data_plane_manager/pom.xml @@ -86,7 +86,11 @@ Copyright(c) 2020 Futurewei Cloud junit-jupiter test - + + com.github.ishugaliy + allgood-consistent-hash + 1.0.0 + org.springframework.boot spring-boot-starter-test diff --git a/services/data_plane_manager/src/main/java/com/futurewei/alcor/dataplane/cache/ArionWingCache.java b/services/data_plane_manager/src/main/java/com/futurewei/alcor/dataplane/cache/ArionWingCache.java new file mode 100644 index 000000000..e39cc156d --- /dev/null +++ b/services/data_plane_manager/src/main/java/com/futurewei/alcor/dataplane/cache/ArionWingCache.java @@ -0,0 +1,101 @@ +/* +MIT License +Copyright(c) 2020 Futurewei Cloud + + Permission is hereby granted, + free of charge, to any person obtaining a copy of this software and associated documentation files(the "Software"), to deal in the Software without restriction, + including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and / or sell copies of the Software, and to permit persons + to whom the Software is furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +*/ +package com.futurewei.alcor.dataplane.cache; + +import com.futurewei.alcor.common.db.CacheException; +import com.futurewei.alcor.common.db.CacheFactory; +import com.futurewei.alcor.common.db.ICache; +import com.futurewei.alcor.common.stats.DurationStatistics; +import com.futurewei.alcor.dataplane.entity.ArionGroup; +import com.futurewei.alcor.dataplane.entity.ArionWing; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.context.annotation.ComponentScan; +import org.springframework.stereotype.Repository; + +import java.util.Collection; +import java.util.Map; +import java.util.Set; + +@Repository +@ComponentScan(value="com.futurewei.alcor.common.db") +public class ArionWingCache { + + // arionWingCache store Arion wing meta data. key is Arion Wing hash code and value is Arion wing meta data. + private ICache arionWingCache; + + // arionWingGroupCache store Arion group meta data. key is Arion wing group name, value is Arion group meta data. + private ICache arionWingGroupCache; + private CacheFactory cacheFactory; + + @Autowired + public ArionWingCache(CacheFactory cacheFactory) { + this.cacheFactory = cacheFactory; + arionWingCache = cacheFactory.getCache(ArionWing.class); + arionWingGroupCache = cacheFactory.getCache(ArionGroup.class); + } + + @DurationStatistics + public ArionWing getArionWing (String resourceId) throws CacheException { + return arionWingCache.get(resourceId); + } + + @DurationStatistics + public Collection getArionWings () throws CacheException { + return arionWingCache.getAll().values(); + } + + @DurationStatistics + public Map getAllSubnetPorts(Map queryParams) throws CacheException { + return arionWingCache.getAll(queryParams); + } + + @DurationStatistics + public Collection getArionWings (Set keys) throws CacheException { + return arionWingCache.getAll(keys).values(); + } + + @DurationStatistics + public void insertArionWing (ArionWing arionWing) throws CacheException { + arionWingCache.put(String.valueOf(arionWing.hashCode()), arionWing); + } + + @DurationStatistics + public void deleteArionWing (String resourceId) throws CacheException { + arionWingCache.remove(resourceId); + } + + @DurationStatistics + public Object getArionGroup (String resourceId) throws CacheException { + return arionWingGroupCache.get(resourceId); + } + + @DurationStatistics + public void insertArionGroup (String resourceId) throws CacheException { + System.out.println("Insert arion group: " + resourceId); + arionWingGroupCache.put(resourceId, new ArionGroup(resourceId)); + } + + @DurationStatistics + public void deleteArionGroup (String resourceId) throws CacheException { + arionWingGroupCache.remove(resourceId); + } + + @DurationStatistics + public Map getAllArionGroup () throws CacheException { + return arionWingGroupCache.getAll(); + } + +} diff --git a/services/data_plane_manager/src/main/java/com/futurewei/alcor/dataplane/client/grpc/DataPlaneClientImplV2.java b/services/data_plane_manager/src/main/java/com/futurewei/alcor/dataplane/client/grpc/DataPlaneClientImplV2.java index c51796262..2db8628ea 100644 --- a/services/data_plane_manager/src/main/java/com/futurewei/alcor/dataplane/client/grpc/DataPlaneClientImplV2.java +++ b/services/data_plane_manager/src/main/java/com/futurewei/alcor/dataplane/client/grpc/DataPlaneClientImplV2.java @@ -59,6 +59,15 @@ public class DataPlaneClientImplV2 implements DataPlaneClient monitorHosts; + @Value("${arionGateway.enabled:false}") + private boolean arionGatwayEnabled; + + @Value("${arionGateway.server:127.0.0.1}") + private String arionMasterServer; + + @Value("${arionGateway.port:9090}") + private int arionMasterPort; + @Value("${microservices.connectTimeout:300}") private String connectTimeout; @@ -72,6 +81,9 @@ public List sendGoalStates(List unicastGoalStates) t for (UnicastGoalStateV2 unicastGoalState : unicastGoalStates) { goalStateBuilder = getGoalState(goalStateBuilder, unicastGoalState); } + if (arionGatwayEnabled) { + doSendGoalStateToArionMaster(goalStateBuilder); + } doSendGoalState(goalStateBuilder.build(), finishLatch, results); if (!finishLatch.await(Integer.parseInt(connectTimeout), TimeUnit.SECONDS)) { @@ -149,8 +161,8 @@ public List sendGoalStates(List unicastGoalStates, M return null; } - private GrpcChannelStub createGrpcChannelStub(String hostIp) { - ManagedChannel channel = ManagedChannelBuilder.forAddress(hostIp, this.hostAgentPort) + private GrpcChannelStub createGrpcChannelStub(String hostIp, int port) { + ManagedChannel channel = ManagedChannelBuilder.forAddress(hostIp, port) .usePlaintext() .keepAliveWithoutCalls(true) .keepAliveTime(Long.MAX_VALUE, TimeUnit.SECONDS) @@ -160,30 +172,31 @@ private GrpcChannelStub createGrpcChannelStub(String hostIp) { return new GrpcChannelStub(channel, asyncStub); } - private GrpcChannelStub getOrCreateGrpcChannel(String hostIp) { - if (!this.hostIpGrpcChannelStubMap.containsKey(hostIp)) { - this.hostIpGrpcChannelStubMap.put(hostIp, createGrpcChannelStubArrayList(hostIp)); + private GrpcChannelStub getOrCreateGrpcChannel(String hostIp, Integer port) { + if (!this.hostIpGrpcChannelStubMap.containsKey(hostIp + port)) { + + this.hostIpGrpcChannelStubMap.put(hostIp + port, createGrpcChannelStubArrayList(hostIp, port)); LOG.info("[getOrCreateGrpcChannel] Created a channel and stub to host IP: " + hostIp); } int usingChannelWithThisIndex = ThreadLocalRandom.current().nextInt(0, numberOfGrpcChannelPerHost); - ManagedChannel chan = this.hostIpGrpcChannelStubMap.get(hostIp).get(usingChannelWithThisIndex).channel; + ManagedChannel chan = this.hostIpGrpcChannelStubMap.get(hostIp + port).get(usingChannelWithThisIndex).channel; //checks the channel status, reconnects if the channel is IDLE ConnectivityState channelState = chan.getState(true); if (channelState != ConnectivityState.READY && channelState != ConnectivityState.CONNECTING && channelState != ConnectivityState.IDLE) { - GrpcChannelStub newChannelStub = createGrpcChannelStub(hostIp); - this.hostIpGrpcChannelStubMap.get(hostIp).set(usingChannelWithThisIndex, newChannelStub); + GrpcChannelStub newChannelStub = createGrpcChannelStub(hostIp, port); + this.hostIpGrpcChannelStubMap.get(hostIp + port).set(usingChannelWithThisIndex, newChannelStub); LOG.info("[getOrCreateGrpcChannel] Replaced a channel and stub to host IP: " + hostIp); } LOG.info("[getOrCreateGrpcChannel] Using channel and stub index " + usingChannelWithThisIndex + " to host IP: " + hostIp); - return this.hostIpGrpcChannelStubMap.get(hostIp).get(usingChannelWithThisIndex); + return this.hostIpGrpcChannelStubMap.get(hostIp + port).get(usingChannelWithThisIndex); } - private ArrayList createGrpcChannelStubArrayList(String hostIp) { + private ArrayList createGrpcChannelStubArrayList(String hostIp, int port) { long start = System.currentTimeMillis(); ArrayList arr = new ArrayList<>(); for (int i = 0; i < numberOfGrpcChannelPerHost; i++) { - GrpcChannelStub channelStub = createGrpcChannelStub(hostIp); + GrpcChannelStub channelStub = createGrpcChannelStub(hostIp, port); // Using Linkerd load balance //warmUpChannelStub(channelStub, hostIp); arr.add(channelStub); @@ -232,10 +245,34 @@ public void onCompleted() { return; } + private String doSendGoalStateToArionMaster (Goalstate.GoalStateV2.Builder goalStateV2) { + GrpcChannelStub channelStub = getOrCreateGrpcChannel(arionMasterServer, arionMasterPort); + GoalStateProvisionerGrpc.GoalStateProvisionerStub asyncStub = channelStub.stub; + var neighborStateRequestBuilder = Goalstateprovisioner.NeighborRulesRequest.newBuilder(); + neighborStateRequestBuilder.addAllNeigborstates(goalStateV2.getNeighborStatesMap().values()); + asyncStub.pushGoalstates(neighborStateRequestBuilder.build(), new StreamObserver() { + @Override + public void onNext(Goalstateprovisioner.GoalStateOperationReply goalStateOperationReply) { + LOG.info("Get response: " + goalStateOperationReply.toString()); + } + + @Override + public void onError(Throwable throwable) { + LOG.info(throwable.getMessage()); + } + + @Override + public void onCompleted() { + + } + }); + return null; + } + private String doSendGoalState(Goalstate.GoalStateV2 goalStateV2, CountDownLatch finishLatch, List replies) { String hostIp = netwconfigmanagerGrpcServiceUrl; long start = System.currentTimeMillis(); - GrpcChannelStub channelStub = getOrCreateGrpcChannel(hostIp); + GrpcChannelStub channelStub = getOrCreateGrpcChannel(hostIp, hostAgentPort); long chan_established = System.currentTimeMillis(); LOG.info("[doSendGoalState] Established channel, elapsed Time in milli seconds: " + (chan_established - start)); GoalStateProvisionerGrpc.GoalStateProvisionerStub asyncStub = channelStub.stub; @@ -371,10 +408,10 @@ private Goalstate.GoalStateV2.Builder getGoalState(Goalstate.GoalStateV2.Builder if (goalStateBuilder.containsRouterStates(unicastGoalStateV2.getHostIp() + "/" + entry.getKey())) { - Router.RouterConfiguration.Builder routerConfigurationBuilder = goalStateBuilder.getRouterStatesMap().get(unicastGoalStateV2.getHostIp() + "/" + entry.getKey()).getConfiguration().toBuilder(); - routerConfigurationBuilder.addAllSubnetRoutingTables(entry.getValue().getConfiguration().getSubnetRoutingTablesList()); - Router.RouterState.Builder routerStateBuilder = goalStateBuilder.getRouterStatesMap().get(unicastGoalStateV2.getHostIp() + "/" + entry.getKey()).toBuilder(); - routerStateBuilder.setConfiguration(routerConfigurationBuilder); + Router.RouterConfiguration.Builder routerConfigurationBuilder = goalStateBuilder.getRouterStatesMap().get(unicastGoalStateV2.getHostIp() + "/" + entry.getKey()).getConfiguration().toBuilder(); + routerConfigurationBuilder.addAllSubnetRoutingTables(entry.getValue().getConfiguration().getSubnetRoutingTablesList()); + Router.RouterState.Builder routerStateBuilder = goalStateBuilder.getRouterStatesMap().get(unicastGoalStateV2.getHostIp() + "/" + entry.getKey()).toBuilder(); + routerStateBuilder.setConfiguration(routerConfigurationBuilder); } else { goalStateBuilder.putRouterStates(unicastGoalStateV2.getHostIp() + "/" + entry.getKey(), entry.getValue()); } diff --git a/services/data_plane_manager/src/main/java/com/futurewei/alcor/dataplane/controller/ArionGatewayController.java b/services/data_plane_manager/src/main/java/com/futurewei/alcor/dataplane/controller/ArionGatewayController.java new file mode 100644 index 000000000..19e4a1d3f --- /dev/null +++ b/services/data_plane_manager/src/main/java/com/futurewei/alcor/dataplane/controller/ArionGatewayController.java @@ -0,0 +1,78 @@ +/* +MIT License +Copyright(c) 2020 Futurewei Cloud + + Permission is hereby granted, + free of charge, to any person obtaining a copy of this software and associated documentation files(the "Software"), to deal in the Software without restriction, + including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and / or sell copies of the Software, and to permit persons + to whom the Software is furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +*/ +package com.futurewei.alcor.dataplane.controller; + +import com.futurewei.alcor.common.stats.DurationStatistics; +import com.futurewei.alcor.dataplane.entity.ArionWing; +import com.futurewei.alcor.dataplane.service.impl.ArionWingService; +import com.futurewei.alcor.web.entity.gateway.ArionWingInfo; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.context.annotation.ComponentScan; +import org.springframework.http.HttpStatus; +import org.springframework.web.bind.annotation.*; + + +/* +This controller provide api for Arion master input group and Arion wing information. +It used for Arion wing Consistent Hash. Could be deprecated once Arion Master could be made self-contained + */ + +@Slf4j +@RestController +@ComponentScan(value = "com.futurewei.alcor.common.stats") +public class ArionGatewayController { + + @Autowired + private ArionWingService arionWingService; + + @PostMapping({"/arionwing"}) + @ResponseStatus(HttpStatus.CREATED) + @DurationStatistics + public ArionWingInfo createGateway(@RequestBody ArionWingInfo arionWingInfo) throws Exception { + arionWingService.createArionWing(new ArionWing(arionWingInfo.getGroup(), arionWingInfo.getIp(), arionWingInfo.getMac(), arionWingInfo.getVni())); + return arionWingInfo; + } + + @PutMapping({"/arionwing/{resource_id}"}) + @DurationStatistics + public ArionWingInfo updateGateway(@PathVariable String resource_id, @RequestBody ArionWingInfo arionWingInfo) throws Exception { + + arionWingService.updateArionWing(new ArionWing(arionWingInfo.getGroup(), arionWingInfo.getIp(), arionWingInfo.getMac(), arionWingInfo.getVni())); + return arionWingInfo; + } + + @DeleteMapping({"/arionwing/{resource_id}"}) + @DurationStatistics + public void deleteGateway(@PathVariable String resource_id) throws Exception { + arionWingService.deleteArionWing(resource_id); + } + + + @PostMapping({"/ariongroup/{group_id}"}) + @ResponseStatus(HttpStatus.CREATED) + @DurationStatistics + public String createGatewayGroup(@PathVariable String group_id) throws Exception { + arionWingService.createArionWingGroup(group_id); + return group_id; + } + + @DeleteMapping({"/ariongroup/{group_id}"}) + @DurationStatistics + public void deleteGatewayGroup(@PathVariable String group_id) throws Exception { + arionWingService.deleteArionWingGroup(group_id); + } +} diff --git a/services/data_plane_manager/src/main/java/com/futurewei/alcor/dataplane/entity/ArionGroup.java b/services/data_plane_manager/src/main/java/com/futurewei/alcor/dataplane/entity/ArionGroup.java new file mode 100644 index 000000000..3971c6d45 --- /dev/null +++ b/services/data_plane_manager/src/main/java/com/futurewei/alcor/dataplane/entity/ArionGroup.java @@ -0,0 +1,32 @@ +/* +MIT License +Copyright(c) 2020 Futurewei Cloud + + Permission is hereby granted, + free of charge, to any person obtaining a copy of this software and associated documentation files(the "Software"), to deal in the Software without restriction, + including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and / or sell copies of the Software, and to permit persons + to whom the Software is furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +*/ +package com.futurewei.alcor.dataplane.entity; + +public class ArionGroup { + String groupName; + + public ArionGroup(String groupName) { + this.groupName = groupName; + } + + public String getGroupName() { + return groupName; + } + + public void setGroupName(String groupName) { + this.groupName = groupName; + } +} diff --git a/services/data_plane_manager/src/main/java/com/futurewei/alcor/dataplane/entity/ArionWing.java b/services/data_plane_manager/src/main/java/com/futurewei/alcor/dataplane/entity/ArionWing.java new file mode 100644 index 000000000..02d4ef880 --- /dev/null +++ b/services/data_plane_manager/src/main/java/com/futurewei/alcor/dataplane/entity/ArionWing.java @@ -0,0 +1,88 @@ +/* +MIT License +Copyright(c) 2020 Futurewei Cloud + + Permission is hereby granted, + free of charge, to any person obtaining a copy of this software and associated documentation files(the "Software"), to deal in the Software without restriction, + including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and / or sell copies of the Software, and to permit persons + to whom the Software is furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +*/ +package com.futurewei.alcor.dataplane.entity; + +import org.ishugaliy.allgood.consistent.hash.annotation.Generated; +import org.ishugaliy.allgood.consistent.hash.node.Node; + +import java.util.Objects; +import java.util.StringJoiner; + +public class ArionWing implements Node { + private String group; + private String ip; + private String mac; + private int port; + + public ArionWing() { + + } + + public ArionWing(String group, String ip, String mac, int port) { + this.group = group != null ? group : ""; + this.mac = mac; + this.ip = ip != null ? ip : ""; + this.port = port; + } + + public String getGroup() { + return group; + } + + public String getIp() { + return ip; + } + + public String getMac() {return mac; } + + public int getPort() { + return port; + } + + @Override + public String getKey() { + return getGroup(); + } + + @Override + @Generated + public boolean equals(Object o) { + if (this == o) return true; + if (!(o instanceof ArionWing)) return false; + ArionWing that = (ArionWing) o; + return getPort() == that.getPort() && + Objects.equals(getGroup(), that.getGroup()) && + Objects.equals(getIp(), that.getIp()) && + Objects.equals(getMac(), that.getMac()); + } + + @Override + @Generated + public int hashCode() { + return Objects.hash(getGroup(), getIp(), getMac(), getPort()); + } + + @Override + @Generated + public String toString() { + return new StringJoiner(", ", ArionWing.class.getSimpleName() + "[", "]") + .add("dc='" + getGroup() + "'") + .add("ip='" + getIp() + "'") + .add("ip='" + getMac() + "'") + .add("port=" + getPort()) + .toString(); + } +} diff --git a/services/data_plane_manager/src/main/java/com/futurewei/alcor/dataplane/service/impl/ArionWingService.java b/services/data_plane_manager/src/main/java/com/futurewei/alcor/dataplane/service/impl/ArionWingService.java new file mode 100644 index 000000000..a0c44f9b7 --- /dev/null +++ b/services/data_plane_manager/src/main/java/com/futurewei/alcor/dataplane/service/impl/ArionWingService.java @@ -0,0 +1,110 @@ +package com.futurewei.alcor.dataplane.service.impl; + +import com.futurewei.alcor.common.db.CacheException; +import com.futurewei.alcor.dataplane.cache.ArionWingCache; +import com.futurewei.alcor.dataplane.entity.ArionWing; +import com.futurewei.alcor.dataplane.entity.UnicastGoalState; +import com.futurewei.alcor.dataplane.entity.UnicastGoalStateV2; +import com.futurewei.alcor.schema.Gateway; +import com.futurewei.alcor.web.entity.dataplane.InternalSubnetEntity; +import com.futurewei.alcor.web.entity.dataplane.v2.NetworkConfiguration; +import org.ishugaliy.allgood.consistent.hash.ConsistentHash; +import org.ishugaliy.allgood.consistent.hash.HashRing; +import org.ishugaliy.allgood.consistent.hash.hasher.DefaultHasher; +import org.ishugaliy.allgood.consistent.hash.node.SimpleNode; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; + +import java.util.*; +import java.util.stream.Collectors; + + +@Service +public class ArionWingService { + + @Autowired + private ArionWingCache arionWingCache; + + private ConsistentHash ring; + + public ArionWingService () { + ring = HashRing.newBuilder() + .name("file_cache_hash_ring") // set hash ring name + .hasher(DefaultHasher.METRO_HASH) // hash function to distribute partitions + .partitionRate(10) // number of partitions per node + .nodes(Arrays.asList()) // initial nodes set + .build(); + } + + public void buildArionGatewayState (NetworkConfiguration networkConfiguration, UnicastGoalStateV2 unicastGoalStateV2) throws CacheException { + for (InternalSubnetEntity internalSubnetEntity : networkConfiguration.getSubnets()) { + Gateway.GatewayState.Builder gatewayStateBuilder = Gateway.GatewayState.newBuilder(); + int vni = internalSubnetEntity.getTunnelId().intValue(); + String subnet = internalSubnetEntity.getCidr(); + String key = String.valueOf(vni) + "-" + subnet; + getArionWings(); + Optional group = ring.locate(key); + Map queryParams = new HashMap<>(); + Object[] value = new Object[1]; + value[0] = group.get().getKey(); + queryParams.put("group", value); + Collection arionWings = arionWingCache.getAllSubnetPorts(queryParams).values(); + for (ArionWing arionWing : arionWings) { + Gateway.GatewayConfiguration.destination.Builder builder = Gateway.GatewayConfiguration.destination.newBuilder(); + builder.setIpAddress(arionWing.getIp()); + builder.setMacAddress(arionWing.getMac()); + gatewayStateBuilder.getConfigurationBuilder().addDestinations(builder); + } + gatewayStateBuilder.getConfigurationBuilder().getArionInfoBuilder().setVni(vni); + gatewayStateBuilder.getConfigurationBuilder().getArionInfoBuilder().setSubnetId(subnet); + unicastGoalStateV2.getGoalStateBuilder().putGatewayStates(key, gatewayStateBuilder.build()); + } + } + + public String getArionGroup (int vni, String subnet) { + String key = String.valueOf(vni) + "-" + subnet; + Optional group = ring.locate(key); + return group.get().getKey(); + } + + + public void getArionWings () throws CacheException { + Set keys = new HashSet<>(); + if (!ring.getNodes().isEmpty()) { + keys = ring.getNodes().stream().map(item -> item.getKey()).collect(Collectors.toSet()); + } + Set keysInCache = arionWingCache.getAllArionGroup().values().stream().map(item -> item.getGroupName()).collect(Collectors.toSet()); + if (!keys.equals(keysInCache)) { + keys.retainAll(keysInCache); + for (SimpleNode node : ring.getNodes()) { + if (!keys.contains(node.getKey())) { + ring.remove(node); + } + } + keysInCache.removeAll(keys); + for (var key : keysInCache) { + ring.add(SimpleNode.of(key)); + } + } + } + + public void createArionWingGroup (String resourcdId) throws CacheException { + arionWingCache.insertArionGroup(resourcdId); + } + + public void deleteArionWingGroup (String resourcdId) throws CacheException { + arionWingCache.deleteArionGroup(resourcdId); + } + + public void createArionWing(ArionWing arionWing) throws CacheException { + arionWingCache.insertArionWing(arionWing); + } + + public void updateArionWing(ArionWing arionWing) throws CacheException { + arionWingCache.insertArionWing(arionWing); + } + + public void deleteArionWing (String resourceId) throws CacheException { + arionWingCache.deleteArionWing(resourceId); + } +} diff --git a/services/data_plane_manager/src/main/java/com/futurewei/alcor/dataplane/service/impl/DpmServiceImplV2.java b/services/data_plane_manager/src/main/java/com/futurewei/alcor/dataplane/service/impl/DpmServiceImplV2.java index 39cd3c7ca..98a6ca0e6 100644 --- a/services/data_plane_manager/src/main/java/com/futurewei/alcor/dataplane/service/impl/DpmServiceImplV2.java +++ b/services/data_plane_manager/src/main/java/com/futurewei/alcor/dataplane/service/impl/DpmServiceImplV2.java @@ -53,6 +53,9 @@ public class DpmServiceImplV2 implements DpmService { @Value("${zetaGateway.enabled:false}") private boolean zetaGatwayEnabled; + @Value("${arionGateway.enabled:false}") + private boolean arionGatwayEnabled; + @Autowired private ZetaGatewayClient zetaGatewayClient; @@ -104,6 +107,9 @@ public class DpmServiceImplV2 implements DpmService { @Autowired private PortHostInfoCache portHostInfoCache; + @Autowired + private ArionWingService arionWingService; + @Autowired private DpmServiceImplV2(Config globalConfig) { this.goalStateMessageVersion = globalConfig.goalStateMessageVersion; @@ -141,6 +147,10 @@ private UnicastGoalStateV2 buildUnicastGoalState(NetworkConfiguration networkCon neighborService.buildNeighborStatesL3(networkConfig, unicastGoalState, multicastGoalState); } + if (arionGatwayEnabled) { + arionWingService.buildArionGatewayState(networkConfig, unicastGoalState); + } + unicastGoalState.setGoalState(unicastGoalState.getGoalStateBuilder().build()); unicastGoalState.setGoalStateBuilder(null); multicastGoalState.setGoalState(multicastGoalState.getGoalStateBuilder().build()); diff --git a/services/data_plane_manager/src/main/java/com/futurewei/alcor/dataplane/service/impl/NeighborService.java b/services/data_plane_manager/src/main/java/com/futurewei/alcor/dataplane/service/impl/NeighborService.java index c80116a01..785027d34 100644 --- a/services/data_plane_manager/src/main/java/com/futurewei/alcor/dataplane/service/impl/NeighborService.java +++ b/services/data_plane_manager/src/main/java/com/futurewei/alcor/dataplane/service/impl/NeighborService.java @@ -16,7 +16,6 @@ free of charge, to any person obtaining a copy of this software and associated d package com.futurewei.alcor.dataplane.service.impl; import com.futurewei.alcor.common.db.CacheException; -import com.futurewei.alcor.common.db.CacheFactory; import com.futurewei.alcor.dataplane.cache.NeighborCache; import com.futurewei.alcor.dataplane.cache.PortHostInfoCache; import com.futurewei.alcor.dataplane.cache.SubnetPortsCacheV2; @@ -36,6 +35,7 @@ free of charge, to any person obtaining a copy of this software and associated d import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Service; import java.util.*; @@ -60,6 +60,12 @@ public class NeighborService extends ResourceService { @Autowired private RouterService routerService; + @Autowired + private ArionWingService arionWingService; + + @Value("${arionGateway.enabled:false}") + private boolean arionGatwayEnabled; + private static String NEIGHBOR_STATE_L2_PREFIX = "L2/"; private static String NEIGHBOR_STATE_L3_PREFIX = "L3/"; @@ -106,6 +112,11 @@ public Neighbor.NeighborState buildNeighborState(NeighborEntry.NeighborType type fixedIpBuilder.setSubnetId(portHostInfo.getSubnetId()); fixedIpBuilder.setIpAddress(portHostInfo.getPortIp()); fixedIpBuilder.setNeighborType(neighborType); + if (arionGatwayEnabled) { + var subnetEntity = subnetPortsCache.getSubnetPorts(portHostInfo.getSubnetId()); + fixedIpBuilder.setArionGroup(arionWingService.getArionGroup(subnetEntity.getTunnelId().intValue(), subnetEntity.getCidr())); + fixedIpBuilder.setTunnelId(subnetEntity.getTunnelId().intValue()); + } neighborConfigBuilder.addFixedIps(fixedIpBuilder.build()); //TODO:setAllowAddressPairs //neighborConfigBuilder.setAllowAddressPairs(); diff --git a/services/data_plane_manager/src/main/resources/application.properties b/services/data_plane_manager/src/main/resources/application.properties index 5c00a8c19..287df8bb7 100644 --- a/services/data_plane_manager/src/main/resources/application.properties +++ b/services/data_plane_manager/src/main/resources/application.properties @@ -60,6 +60,10 @@ zetaGateway.enabled=false zetaGateway.check.timeout=30 zetaGateway.check.interval=2 +arionGateway.enabled = true +arionMaster.server = 127.0.0.1 +arionMaster.port = 9090 + #####Spring health##### management.health.redis.enabled=false diff --git a/web/src/main/java/com/futurewei/alcor/web/entity/gateway/ArionWingInfo.java b/web/src/main/java/com/futurewei/alcor/web/entity/gateway/ArionWingInfo.java new file mode 100644 index 000000000..f5ef740f8 --- /dev/null +++ b/web/src/main/java/com/futurewei/alcor/web/entity/gateway/ArionWingInfo.java @@ -0,0 +1,34 @@ +/* +MIT License +Copyright(c) 2020 Futurewei Cloud + + Permission is hereby granted, + free of charge, to any person obtaining a copy of this software and associated documentation files(the "Software"), to deal in the Software without restriction, + including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and / or sell copies of the Software, and to permit persons + to whom the Software is furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +*/ +package com.futurewei.alcor.web.entity.gateway; + +import com.fasterxml.jackson.annotation.JsonProperty; +import lombok.Data; + +@Data +public class ArionWingInfo { + private String group; + private String ip; + private String mac; + private int vni; + + public ArionWingInfo(String group, String ip, String mac, int vni) { + this.group = group; + this.ip = ip; + this.mac = mac; + this.vni = vni; + } +}