Fix Static NAT/Port Forwarding when VM NIC is not the default

api/src/main/java/com/cloud/agent/api/to/StaticNatRuleTO.java

@@ -28,6 +28,7 @@
 
 public class StaticNatRuleTO extends FirewallRuleTO {
     String dstIp;
+    boolean shouldApplyCrossNetworkSnat = false;
 
     protected StaticNatRuleTO() {
     }
@@ -79,4 +80,12 @@ public String getDstIp() {
         return dstIp;
     }
 
+    public boolean isShouldApplyCrossNetworkSnat() {
+        return shouldApplyCrossNetworkSnat;
+    }
+
+    public void setShouldApplyCrossNetworkSnat(boolean shouldApplyCrossNetworkSnat) {
+        this.shouldApplyCrossNetworkSnat = shouldApplyCrossNetworkSnat;
+    }
+
 }

core/src/main/java/com/cloud/agent/resource/virtualnetwork/facade/SetStaticNatRulesConfigItem.java

@@ -39,7 +39,8 @@ public List<ConfigItem> generateConfig(final NetworkElementCommand cmd) {
 
         final LinkedList<StaticNatRule> rules = new LinkedList<>();
         for (final StaticNatRuleTO rule : command.getRules()) {
-            final StaticNatRule staticNatRule = new StaticNatRule(rule.revoked(), rule.getProtocol(), rule.getSrcIp(), rule.getStringSrcPortRange(), rule.getDstIp());
+            final StaticNatRule staticNatRule = new StaticNatRule(rule.revoked(), rule.getProtocol(), rule.getSrcIp(),
+                    rule.getStringSrcPortRange(), rule.getDstIp(), rule.isShouldApplyCrossNetworkSnat());
             rules.add(staticNatRule);
         }
         final StaticNatRules staticNatRules = new StaticNatRules(rules);

core/src/main/java/com/cloud/agent/resource/virtualnetwork/model/StaticNatRule.java

@@ -25,18 +25,25 @@ public class StaticNatRule {
     private String sourceIpAddress;
     private String sourcePortRange;
     private String destinationIpAddress;
+    private boolean shouldApplyCrossNetworkSnat = false;
 
     public StaticNatRule() {
         // Empty constructor for (de)serialization
     }
 
     public StaticNatRule(boolean revoke, String protocol, String sourceIpAddress, String sourcePortRange, String destinationIpAddress) {
+        this(revoke, protocol, sourceIpAddress, sourcePortRange, destinationIpAddress, false);
+    }
+
+    public StaticNatRule(boolean revoke, String protocol, String sourceIpAddress, String sourcePortRange,
+                         String destinationIpAddress, boolean shouldApplyCrossNetworkSnat) {
         super();
         this.revoke = revoke;
         this.protocol = protocol;
         this.sourceIpAddress = sourceIpAddress;
         this.sourcePortRange = sourcePortRange;
         this.destinationIpAddress = destinationIpAddress;
+        this.shouldApplyCrossNetworkSnat = shouldApplyCrossNetworkSnat;
     }
 
     public boolean isRevoke() {
@@ -79,4 +86,12 @@ public void setDestinationIpAddress(String destinationIpAddress) {
         this.destinationIpAddress = destinationIpAddress;
     }
 
+    public boolean isShouldApplyCrossNetworkSnat() {
+        return shouldApplyCrossNetworkSnat;
+    }
+
+    public void setShouldApplyCrossNetworkSnat(boolean shouldApplyCrossNetworkSnat) {
+        this.shouldApplyCrossNetworkSnat = shouldApplyCrossNetworkSnat;
+    }
+
 }

server/src/main/java/com/cloud/network/router/CommandSetupHelper.java

@@ -450,6 +450,7 @@ public void createApplyStaticNatRulesCommands(final List<? extends StaticNatRule
             for (final StaticNatRule rule : rules) {
                 final IpAddress sourceIp = _networkModel.getIp(rule.getSourceIpAddressId());
                 final StaticNatRuleTO ruleTO = new StaticNatRuleTO(rule, null, sourceIp.getAddress().addr(), rule.getDestIpAddress());
+                ruleTO.setShouldApplyCrossNetworkSnat(requiresReturnPathSnat(guestNetworkId, rule.getDestIpAddress()));
                 rulesTO.add(ruleTO);
             }
         }
@@ -463,6 +464,25 @@ public void createApplyStaticNatRulesCommands(final List<? extends StaticNatRule
         cmds.addCommand(cmd);
     }
 
+    /**
+     * This method determines whether to use network-wide SNAT or NIC aware SNAT
+     * @param networkId
+     * @param destinationIp
+     * @return
+     */
+    private boolean requiresReturnPathSnat(final long networkId, final String destinationIp) {
+        if (!VirtualNetworkApplianceManager.NicSnatEnabled.value()) {
+            return false;
+        }
+
+        final NicVO destinationNic = _nicDao.findByIp4AddressAndNetworkId(destinationIp, networkId);
+        if (destinationNic == null) {
+            logger.debug("Unable to find destination NIC for ip [{}] in network [{}], assuming default NIC.", destinationIp, networkId);
+            return false;
+        }
+        return !destinationNic.isDefaultNic();
+    }
+
     public void createApplyFirewallRulesCommands(final List<? extends FirewallRule> rules, final VirtualRouter router, final Commands cmds,
             final Long guestNetworkId, final Long vpcId) {
         final List<FirewallRuleTO> rulesTO = new ArrayList<>();
@@ -729,6 +749,7 @@ public void createApplyStaticNatCommands(final List<? extends StaticNat> rules,
                 final IpAddress sourceIp = _networkModel.getIp(rule.getSourceIpAddressId());
                 final StaticNatRuleTO ruleTO = new StaticNatRuleTO(0, sourceIp.getAddress().addr(), null, null, rule.getDestIpAddress(), null, null, null, rule.isForRevoke(),
                         false);
+                ruleTO.setShouldApplyCrossNetworkSnat(requiresReturnPathSnat(guestNetworkId, rule.getDestIpAddress()));
                 rulesTO.add(ruleTO);
             }
         }

server/src/main/java/com/cloud/network/router/VirtualNetworkApplianceManager.java

@@ -50,6 +50,7 @@ public interface VirtualNetworkApplianceManager extends Manager, VirtualNetworkA
     String RouterHealthChecksResultFetchIntervalCK = "router.health.checks.results.fetch.interval";
     String RouterHealthChecksFailuresToRecreateVrCK = "router.health.checks.failures.to.recreate.vr";
     String RemoveControlIpOnStopCK = "systemvm.release.control.ip.on.stop";
+    String UseNicAwareSnat = "network.nic.snat.enabled";
 
     ConfigKey<String> RouterTemplateXen = new ConfigKey<>(String.class, RouterTemplateXenCK, "Advanced", "SystemVM Template (XenServer)",
             "Name of the default router template on Xenserver.", true, ConfigKey.Scope.Zone, null);
@@ -131,6 +132,17 @@ public interface VirtualNetworkApplianceManager extends Manager, VirtualNetworkA
     ConfigKey<Boolean> RemoveControlIpOnStop = new ConfigKey<>(Boolean.class, RemoveControlIpOnStopCK, "Advanced", "true",
             "on stopping routers and system VMs the IP will be released to preserve IPv4 space.", true, ConfigKey.Scope.Zone, null);
 
+    ConfigKey<Boolean> NicSnatEnabled = new ConfigKey<>(Boolean.class,
+            UseNicAwareSnat,
+            "Advanced",
+            "false",
+            "When enabled, Virtual Router overwrites the SNAT source IP in POSTROUTING for traffic exiting " +
+                    "non-default NICs. The SNAT source is selected based on the egress interface to preserve return path " +
+                    "consistency in multi-NIC configurations.",
+            true,
+            ConfigKey.Scope.Global,
+            null);
+
     int DEFAULT_ROUTER_VM_RAMSIZE = 256;            // 256M
     int DEFAULT_ROUTER_CPU_MHZ = 500;                // 500 MHz
     boolean USE_POD_VLAN = false;

server/src/main/java/com/cloud/network/router/VirtualNetworkApplianceManagerImpl.java

@@ -3384,7 +3384,8 @@ public ConfigKey<?>[] getConfigKeys() {
                 ExposeDnsAndBootpServer,
                 RouterLogrotateFrequency,
                 RemoveControlIpOnStop,
-                VirtualRouterUserData
+                VirtualRouterUserData,
+                NicSnatEnabled
         };
     }
 

systemvm/debian/opt/cloud/bin/configure.py

@@ -1573,7 +1573,7 @@ def forward_vr(self, rule):
               )
         fw4 = "-j SNAT --to-source %s -A POSTROUTING -s %s -d %s/32 -o %s -p %s -m %s --dport %s" % \
               (
-                self.getGuestIp(),
+                self.getGuestIpByIp(rule['internal_ip']),
                 self.getNetworkByIp(rule['internal_ip']),
                 rule['internal_ip'],
                 internal_fwinterface,
@@ -1688,11 +1688,20 @@ def processStaticNatRule(self, rule):
         self.fw.append(["filter", "",
                         "-A FORWARD -i %s -o eth0  -d %s  -m state  --state NEW -j ACCEPT " % (device, rule["internal_ip"])])
 
-        # Configure the hairpin snat
-        self.fw.append(["nat", "front", "-A POSTROUTING -s %s -d %s -j SNAT -o %s --to-source %s" %
+        # Configure the hairpin snat for default nic or nic-aware snat for non-default
+        apply_cross_network_snat = rule.get("should_apply_cross_network_snat", False)
+        if apply_cross_network_snat:
+            internal_device = self.getDeviceByIp(rule["internal_ip"])
+            internal_vr_ip = self.getGuestIpByIp(rule["internal_ip"])
+            if internal_device and internal_vr_ip and internal_device != device:
+                self.fw.append(["nat", "front",
+                                "-A POSTROUTING -o %s -d %s/32 -j SNAT --to-source %s" % (internal_device, rule["internal_ip"], internal_vr_ip)])
+        else:
+            self.fw.append(["nat", "front", "-A POSTROUTING -s %s -d %s -j SNAT -o %s --to-source %s" %
                         (self.getNetworkByIp(rule['internal_ip']), rule["internal_ip"], self.getDeviceByIp(rule["internal_ip"]), self.getGuestIpByIp(rule["internal_ip"]))])
 
 
+
 class IpTablesExecutor:
 
     config = None

systemvm/debian/opt/cloud/bin/cs_forwardingrules.py

@@ -27,6 +27,8 @@ def merge(dbag, rules):
         newrule = dict()
         newrule["public_ip"] = source_ip
         newrule["internal_ip"] = destination_ip
+        if "should_apply_cross_network_snat" in rule:
+            newrule["should_apply_cross_network_snat"] = rule["should_apply_cross_network_snat"]
 
         if rules["type"] == "staticnatrules":
             newrule["type"] = "staticnat"