Flxcld 2859 management server async job issue

.gitignore

@@ -99,6 +99,7 @@ tools/marvin/marvin/cloudstackAPI/*
 unittest
 venv
 waf-*
+.envrc
 
 # this ignores _all files starting with '.'. Don't do that!
 #.*

framework/jobs/src/main/java/org/apache/cloudstack/framework/jobs/impl/AsyncJobManagerImpl.java

@@ -1153,7 +1153,7 @@ public void doInTransactionWithoutResult(TransactionStatus status) {
                     final List<AsyncJobVO> jobs = _jobDao.getResetJobs(msid);
                     for (final AsyncJobVO job : jobs) {
                         if (logger.isDebugEnabled()) {
-                            logger.debug("Cancel left-over job-" + job.getId());
+                            logger.debug("Cancel left-over job-{} for msid {}", job.getId(), msid);
                         }
                         cleanupResources(job);
                         job.setStatus(JobInfo.Status.FAILED);

utils/src/main/java/org/apache/cloudstack/utils/identity/ManagementServerNode.java

@@ -22,6 +22,7 @@
 import java.net.InetAddress;
 import java.nio.charset.StandardCharsets;
 import java.security.MessageDigest;
+import java.security.NoSuchAlgorithmException;
 
 import com.cloud.utils.component.AdapterBase;
 import com.cloud.utils.component.ComponentLifecycle;
@@ -46,73 +47,117 @@ public class ManagementServerNode extends AdapterBase implements SystemIntegrity
 
     private static final String FQDN_ENV_VAR = "CLOUDSTACK_MSID_FROM_FQDN";
     private static final String FQDN_SYS_PROP = "cloudstack.msid.from.fqdn";
+    private static final String IDENTITY_ENV_VAR = "CLOUDSTACK_MSID_IDENTITY";
+    private static final String IDENTITY_SYS_PROP = "cloudstack.msid.identity";
+    private static final String HOSTNAME_ENV_VAR = "HOSTNAME";
+    private static final String POD_NAMESPACE_ENV_VAR = "POD_NAMESPACE";
+
+    // op_lock.mac is varchar(17) and holds the msid, so the id must stay within the 48-bit MAC address range.
+    private static final int MSID_BYTES = 6;
 
     private static String s_nodeIdSource;
-    private static Exception s_initError;
     private static final long s_nodeId = initNodeId();
 
     private static long initNodeId() {
-        if (isFqdnModeEnabled()) {
-            return generateIdFromFqdn();
+        if (isTruthy(System.getenv(FQDN_ENV_VAR)) || isTruthy(System.getProperty(FQDN_SYS_PROP))) {
+            return generateIdFromStableIdentity();
         }
+
         s_nodeIdSource = "mac-address";
         return MacAddress.getMacAddress().toLong();
     }
 
-    private static boolean isFqdnModeEnabled() {
-        return isTruthy(System.getenv(FQDN_ENV_VAR)) || isTruthy(System.getProperty(FQDN_SYS_PROP));
-    }
+    static String resolveNodeIdentity(String explicitIdentity, String hostnameEnv, String podNamespaceEnv,
+            String detectedHostName, String canonicalHostName) {
+        String configuredIdentity = trimToNull(explicitIdentity);
+        if (configuredIdentity != null) {
+            return configuredIdentity;
+        }
 
-    private static boolean isTruthy(String value) {
-        if (value == null) {
-            return false;
+        String hostName = trimToNull(hostnameEnv);
+        if (hostName != null) {
+            String podNamespace = trimToNull(podNamespaceEnv);
+            return podNamespace == null ? hostName : hostName + "." + podNamespace;
         }
-        String trimmed = value.trim();
-        return "true".equalsIgnoreCase(trimmed) || "1".equals(trimmed) || "yes".equalsIgnoreCase(trimmed);
+
+        String detected = trimToNull(detectedHostName);
+        String canonical = trimToNull(canonicalHostName);
+        if (detected != null && canonical != null && !detected.equals(canonical)) {
+            return detected + "|" + canonical;
+        }
+
+        return canonical != null ? canonical : detected;
     }
 
-    /**
-     * Derives a stable node id from a SHA-256 hash of the local FQDN.
-     *
-     * <p>On failure it records the cause and returns {@code 0} (an invalid id) rather than
-     * silently reverting to an unstable MAC-based id. The invalid id makes {@link #check()}
-     * fail the system-integrity check, which stops startup cleanly via {@link #start()}
-     * instead of raising an {@code ExceptionInInitializerError} from static initialization.
-     *
-     * @return a positive, non-zero 48-bit id, or {@code 0} if it cannot be derived
-     */
-    private static long generateIdFromFqdn() {
+    static long hashNodeIdentity(String nodeIdentity) {
         try {
-            String fqdn = InetAddress.getLocalHost().getCanonicalHostName();
-            s_nodeIdSource = "fqdn:" + fqdn;
             MessageDigest digest = MessageDigest.getInstance("SHA-256");
-            byte[] hash = digest.digest(fqdn.getBytes(StandardCharsets.UTF_8));
+            byte[] hash = digest.digest(nodeIdentity.getBytes(StandardCharsets.UTF_8));
             long id = 0;
-            for (int i = 0; i < 6; i++) {
-                id = (id << 8) | (hash[i] & 0xFF);
+            for (int i = 0; i < MSID_BYTES; i++) {
+                id = (id << 8) | (hash[i] & 0xFFL);
             }
-            // Ensure positive and non-zero
-            id = id & 0x7FFFFFFFFFFFFFFFL;
-            if (id == 0) {
-                id = 1;
+
+            return id == 0 ? 1 : id;
+        } catch (NoSuchAlgorithmException e) {
+            throw new CloudRuntimeException("SHA-256 algorithm not available for management server ID generation", e);
+        }
+    }
+
+    private static long generateIdFromStableIdentity() {
+        try {
+            InetAddress localHost = InetAddress.getLocalHost();
+            String nodeIdentity = resolveNodeIdentity(
+                    firstNonBlank(System.getProperty(IDENTITY_SYS_PROP), System.getenv(IDENTITY_ENV_VAR)),
+                    System.getenv(HOSTNAME_ENV_VAR),
+                    System.getenv(POD_NAMESPACE_ENV_VAR),
+                    localHost.getHostName(),
+                    localHost.getCanonicalHostName());
+
+            if (nodeIdentity == null) {
+                throw new CloudRuntimeException("Unable to resolve a stable management server identity");
             }
-            return id;
+
+            s_nodeIdSource = "identity:" + nodeIdentity;
+            return hashNodeIdentity(nodeIdentity);
+        } catch (CloudRuntimeException e) {
+            throw e;
         } catch (Exception e) {
-            s_nodeIdSource = "fqdn-error";
-            s_initError = e;
-            return 0;
+            throw new CloudRuntimeException("Unable to generate management server ID from host identity", e);
         }
     }
 
+    private static String firstNonBlank(String first, String second) {
+        String value = trimToNull(first);
+        return value != null ? value : trimToNull(second);
+    }
+
+    private static boolean isTruthy(String value) {
+        if (value == null) {
+            return false;
+        }
+
+        String trimmed = value.trim();
+        return "true".equalsIgnoreCase(trimmed) || "1".equals(trimmed) || "yes".equalsIgnoreCase(trimmed);
+    }
+
+    private static String trimToNull(String value) {
+        if (value == null) {
+            return null;
+        }
+
+        String trimmed = value.trim();
+        return trimmed.isEmpty() ? null : trimmed;
+    }
+
     public ManagementServerNode() {
         setRunLevel(ComponentLifecycle.RUN_LEVEL_FRAMEWORK_BOOTSTRAP);
     }
 
     @Override
     public void check() {
         if (s_nodeId <= 0) {
-            throw new CloudRuntimeException(
-                    "Unable to derive the management server node id (source: " + s_nodeIdSource + ")", s_initError);
+            throw new CloudRuntimeException("Unable to get the management server node id");
         }
     }
 
@@ -124,10 +169,11 @@ public static long getManagementServerId() {
     public boolean start() {
         try {
             check();
-        } catch (CloudRuntimeException e) {
-            logger.error("System integrity check failed for the management server node id", e);
-            throw e;
+        } catch (Exception e) {
+            logger.error("System integrity check exception", e);
+            System.exit(1);
         }
+
         logger.info("Management server node id: {} (source: {})", s_nodeId, s_nodeIdSource);
         return true;
     }